tools/objtool/check.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Copyright (C) 2015-2017 Josh Poimboeuf <[email protected]>
   4  */
   5
   6 #include <string.h>
   7 #include <stdlib.h>
   8 #include <inttypes.h>
   9 #include <sys/mman.h>
  10
  11 #include <objtool/builtin.h>
  12 #include <objtool/cfi.h>
  13 #include <objtool/arch.h>
  14 #include <objtool/check.h>
  15 #include <objtool/special.h>
  16 #include <objtool/warn.h>
  17 #include <objtool/endianness.h>
  18
  19 #include <linux/objtool_types.h>
  20 #include <linux/hashtable.h>
  21 #include <linux/kernel.h>
  22 #include <linux/static_call_types.h>
  23 #include <linux/string.h>
  24
  25 struct alternative {
  26         struct alternative *next;
  27         struct instruction *insn;
  28         bool skip_orig;
  29 };
  30
  31 static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
  32
  33 static struct cfi_init_state initial_func_cfi;
  34 static struct cfi_state init_cfi;
  35 static struct cfi_state func_cfi;
  36 static struct cfi_state force_undefined_cfi;
  37
  38 struct instruction *find_insn(struct objtool_file *file,
  39                               struct section *sec, unsigned long offset)
  40 {
  41         struct instruction *insn;
  42
  43         hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
  44                 if (insn->sec == sec && insn->offset == offset)
  45                         return insn;
  46         }
  47
  48         return NULL;
  49 }
  50
  51 struct instruction *next_insn_same_sec(struct objtool_file *file,
  52                                        struct instruction *insn)
  53 {
  54         if (insn->idx == INSN_CHUNK_MAX)
  55                 return find_insn(file, insn->sec, insn->offset + insn->len);
  56
  57         insn++;
  58         if (!insn->len)
  59                 return NULL;
  60
  61         return insn;
  62 }
  63
  64 static struct instruction *next_insn_same_func(struct objtool_file *file,
  65                                                struct instruction *insn)
  66 {
  67         struct instruction *next = next_insn_same_sec(file, insn);
  68         struct symbol *func = insn_func(insn);
  69
  70         if (!func)
  71                 return NULL;
  72
  73         if (next && insn_func(next) == func)
  74                 return next;
  75
  76         /* Check if we're already in the subfunction: */
  77         if (func == func->cfunc)
  78                 return NULL;
  79
  80         /* Move to the subfunction: */
  81         return find_insn(file, func->cfunc->sec, func->cfunc->offset);
  82 }
  83
  84 static struct instruction *prev_insn_same_sec(struct objtool_file *file,
  85                                               struct instruction *insn)
  86 {
  87         if (insn->idx == 0) {
  88                 if (insn->prev_len)
  89                         return find_insn(file, insn->sec, insn->offset - insn->prev_len);
  90                 return NULL;
  91         }
  92
  93         return insn - 1;
  94 }
  95
  96 static struct instruction *prev_insn_same_sym(struct objtool_file *file,
  97                                               struct instruction *insn)
  98 {
  99         struct instruction *prev = prev_insn_same_sec(file, insn);
 100
 101         if (prev && insn_func(prev) == insn_func(insn))
 102                 return prev;
 103
 104         return NULL;
 105 }
 106
 107 #define for_each_insn(file, insn)                                       \
 108         for (struct section *__sec, *__fake = (struct section *)1;      \
 109              __fake; __fake = NULL)                                     \
 110                 for_each_sec(file, __sec)                               \
 111                         sec_for_each_insn(file, __sec, insn)
 112
 113 #define func_for_each_insn(file, func, insn)                            \
 114         for (insn = find_insn(file, func->sec, func->offset);           \
 115              insn;                                                      \
 116              insn = next_insn_same_func(file, insn))
 117
 118 #define sym_for_each_insn(file, sym, insn)                              \
 119         for (insn = find_insn(file, sym->sec, sym->offset);             \
 120              insn && insn->offset < sym->offset + sym->len;             \
 121              insn = next_insn_same_sec(file, insn))
 122
 123 #define sym_for_each_insn_continue_reverse(file, sym, insn)             \
 124         for (insn = prev_insn_same_sec(file, insn);                     \
 125              insn && insn->offset >= sym->offset;                       \
 126              insn = prev_insn_same_sec(file, insn))
 127
 128 #define sec_for_each_insn_from(file, insn)                              \
 129         for (; insn; insn = next_insn_same_sec(file, insn))
 130
 131 #define sec_for_each_insn_continue(file, insn)                          \
 132         for (insn = next_insn_same_sec(file, insn); insn;               \
 133              insn = next_insn_same_sec(file, insn))
 134
 135 static inline struct symbol *insn_call_dest(struct instruction *insn)
 136 {
 137         if (insn->type == INSN_JUMP_DYNAMIC ||
 138             insn->type == INSN_CALL_DYNAMIC)
 139                 return NULL;
 140
 141         return insn->_call_dest;
 142 }
 143
 144 static inline struct reloc *insn_jump_table(struct instruction *insn)
 145 {
 146         if (insn->type == INSN_JUMP_DYNAMIC ||
 147             insn->type == INSN_CALL_DYNAMIC)
 148                 return insn->_jump_table;
 149
 150         return NULL;
 151 }
 152
 153 static bool is_jump_table_jump(struct instruction *insn)
 154 {
 155         struct alt_group *alt_group = insn->alt_group;
 156
 157         if (insn_jump_table(insn))
 158                 return true;
 159
 160         /* Retpoline alternative for a jump table? */
 161         return alt_group && alt_group->orig_group &&
 162                insn_jump_table(alt_group->orig_group->first_insn);
 163 }
 164
 165 static bool is_sibling_call(struct instruction *insn)
 166 {
 167         /*
 168          * Assume only STT_FUNC calls have jump-tables.
 169          */
 170         if (insn_func(insn)) {
 171                 /* An indirect jump is either a sibling call or a jump to a table. */
 172                 if (insn->type == INSN_JUMP_DYNAMIC)
 173                         return !is_jump_table_jump(insn);
 174         }
 175
 176         /* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */
 177         return (is_static_jump(insn) && insn_call_dest(insn));
 178 }
 179
 180 /*
 181  * This checks to see if the given function is a "noreturn" function.
 182  *
 183  * For global functions which are outside the scope of this object file, we
 184  * have to keep a manual list of them.
 185  *
 186  * For local functions, we have to detect them manually by simply looking for
 187  * the lack of a return instruction.
 188  */
 189 static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
 190                                 int recursion)
 191 {
 192         int i;
 193         struct instruction *insn;
 194         bool empty = true;
 195
 196 #define NORETURN(func) __stringify(func),
 197         static const char * const global_noreturns[] = {
 198 #include "noreturns.h"
 199         };
 200 #undef NORETURN
 201
 202         if (!func)
 203                 return false;
 204
 205         if (func->bind == STB_GLOBAL || func->bind == STB_WEAK)
 206                 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
 207                         if (!strcmp(func->name, global_noreturns[i]))
 208                                 return true;
 209
 210         if (func->bind == STB_WEAK)
 211                 return false;
 212
 213         if (!func->len)
 214                 return false;
 215
 216         insn = find_insn(file, func->sec, func->offset);
 217         if (!insn || !insn_func(insn))
 218                 return false;
 219
 220         func_for_each_insn(file, func, insn) {
 221                 empty = false;
 222
 223                 if (insn->type == INSN_RETURN)
 224                         return false;
 225         }
 226
 227         if (empty)
 228                 return false;
 229
 230         /*
 231          * A function can have a sibling call instead of a return.  In that
 232          * case, the function's dead-end status depends on whether the target
 233          * of the sibling call returns.
 234          */
 235         func_for_each_insn(file, func, insn) {
 236                 if (is_sibling_call(insn)) {
 237                         struct instruction *dest = insn->jump_dest;
 238
 239                         if (!dest)
 240                                 /* sibling call to another file */
 241                                 return false;
 242
 243                         /* local sibling call */
 244                         if (recursion == 5) {
 245                                 /*
 246                                  * Infinite recursion: two functions have
 247                                  * sibling calls to each other.  This is a very
 248                                  * rare case.  It means they aren't dead ends.
 249                                  */
 250                                 return false;
 251                         }
 252
 253                         return __dead_end_function(file, insn_func(dest), recursion+1);
 254                 }
 255         }
 256
 257         return true;
 258 }
 259
 260 static bool dead_end_function(struct objtool_file *file, struct symbol *func)
 261 {
 262         return __dead_end_function(file, func, 0);
 263 }
 264
 265 static void init_cfi_state(struct cfi_state *cfi)
 266 {
 267         int i;
 268
 269         for (i = 0; i < CFI_NUM_REGS; i++) {
 270                 cfi->regs[i].base = CFI_UNDEFINED;
 271                 cfi->vals[i].base = CFI_UNDEFINED;
 272         }
 273         cfi->cfa.base = CFI_UNDEFINED;
 274         cfi->drap_reg = CFI_UNDEFINED;
 275         cfi->drap_offset = -1;
 276 }
 277
 278 static void init_insn_state(struct objtool_file *file, struct insn_state *state,
 279                             struct section *sec)
 280 {
 281         memset(state, 0, sizeof(*state));
 282         init_cfi_state(&state->cfi);
 283
 284         /*
 285          * We need the full vmlinux for noinstr validation, otherwise we can
 286          * not correctly determine insn_call_dest(insn)->sec (external symbols
 287          * do not have a section).
 288          */
 289         if (opts.link && opts.noinstr && sec)
 290                 state->noinstr = sec->noinstr;
 291 }
 292
 293 static struct cfi_state *cfi_alloc(void)
 294 {
 295         struct cfi_state *cfi = calloc(1, sizeof(struct cfi_state));
 296         if (!cfi) {
 297                 WARN("calloc failed");
 298                 exit(1);
 299         }
 300         nr_cfi++;
 301         return cfi;
 302 }
 303
 304 static int cfi_bits;
 305 static struct hlist_head *cfi_hash;
 306
 307 static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2)
 308 {
 309         return memcmp((void *)cfi1 + sizeof(cfi1->hash),
 310                       (void *)cfi2 + sizeof(cfi2->hash),
 311                       sizeof(struct cfi_state) - sizeof(struct hlist_node));
 312 }
 313
 314 static inline u32 cfi_key(struct cfi_state *cfi)
 315 {
 316         return jhash((void *)cfi + sizeof(cfi->hash),
 317                      sizeof(*cfi) - sizeof(cfi->hash), 0);
 318 }
 319
 320 static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi)
 321 {
 322         struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
 323         struct cfi_state *obj;
 324
 325         hlist_for_each_entry(obj, head, hash) {
 326                 if (!cficmp(cfi, obj)) {
 327                         nr_cfi_cache++;
 328                         return obj;
 329                 }
 330         }
 331
 332         obj = cfi_alloc();
 333         *obj = *cfi;
 334         hlist_add_head(&obj->hash, head);
 335
 336         return obj;
 337 }
 338
 339 static void cfi_hash_add(struct cfi_state *cfi)
 340 {
 341         struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
 342
 343         hlist_add_head(&cfi->hash, head);
 344 }
 345
 346 static void *cfi_hash_alloc(unsigned long size)
 347 {
 348         cfi_bits = max(10, ilog2(size));
 349         cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
 350                         PROT_READ|PROT_WRITE,
 351                         MAP_PRIVATE|MAP_ANON, -1, 0);
 352         if (cfi_hash == (void *)-1L) {
 353                 WARN("mmap fail cfi_hash");
 354                 cfi_hash = NULL;
 355         }  else if (opts.stats) {
 356                 printf("cfi_bits: %d\n", cfi_bits);
 357         }
 358
 359         return cfi_hash;
 360 }
 361
 362 static unsigned long nr_insns;
 363 static unsigned long nr_insns_visited;
 364
 365 /*
 366  * Call the arch-specific instruction decoder for all the instructions and add
 367  * them to the global instruction list.
 368  */
 369 static int decode_instructions(struct objtool_file *file)
 370 {
 371         struct section *sec;
 372         struct symbol *func;
 373         unsigned long offset;
 374         struct instruction *insn;
 375         int ret;
 376
 377         for_each_sec(file, sec) {
 378                 struct instruction *insns = NULL;
 379                 u8 prev_len = 0;
 380                 u8 idx = 0;
 381
 382                 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
 383                         continue;
 384
 385                 if (strcmp(sec->name, ".altinstr_replacement") &&
 386                     strcmp(sec->name, ".altinstr_aux") &&
 387                     strncmp(sec->name, ".discard.", 9))
 388                         sec->text = true;
 389
 390                 if (!strcmp(sec->name, ".noinstr.text") ||
 391                     !strcmp(sec->name, ".entry.text") ||
 392                     !strcmp(sec->name, ".cpuidle.text") ||
 393                     !strncmp(sec->name, ".text..__x86.", 13))
 394                         sec->noinstr = true;
 395
 396                 /*
 397                  * .init.text code is ran before userspace and thus doesn't
 398                  * strictly need retpolines, except for modules which are
 399                  * loaded late, they very much do need retpoline in their
 400                  * .init.text
 401                  */
 402                 if (!strcmp(sec->name, ".init.text") && !opts.module)
 403                         sec->init = true;
 404
 405                 for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
 406                         if (!insns || idx == INSN_CHUNK_MAX) {
 407                                 insns = calloc(sizeof(*insn), INSN_CHUNK_SIZE);
 408                                 if (!insns) {
 409                                         WARN("malloc failed");
 410                                         return -1;
 411                                 }
 412                                 idx = 0;
 413                         } else {
 414                                 idx++;
 415                         }
 416                         insn = &insns[idx];
 417                         insn->idx = idx;
 418
 419                         INIT_LIST_HEAD(&insn->call_node);
 420                         insn->sec = sec;
 421                         insn->offset = offset;
 422                         insn->prev_len = prev_len;
 423
 424                         ret = arch_decode_instruction(file, sec, offset,
 425                                                       sec->sh.sh_size - offset,
 426                                                       insn);
 427                         if (ret)
 428                                 return ret;
 429
 430                         prev_len = insn->len;
 431
 432                         /*
 433                          * By default, "ud2" is a dead end unless otherwise
 434                          * annotated, because GCC 7 inserts it for certain
 435                          * divide-by-zero cases.
 436                          */
 437                         if (insn->type == INSN_BUG)
 438                                 insn->dead_end = true;
 439
 440                         hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
 441                         nr_insns++;
 442                 }
 443
 444 //              printf("%s: last chunk used: %d\n", sec->name, (int)idx);
 445
 446                 sec_for_each_sym(sec, func) {
 447                         if (func->type != STT_NOTYPE && func->type != STT_FUNC)
 448                                 continue;
 449
 450                         if (func->offset == sec->sh.sh_size) {
 451                                 /* Heuristic: likely an "end" symbol */
 452                                 if (func->type == STT_NOTYPE)
 453                                         continue;
 454                                 WARN("%s(): STT_FUNC at end of section",
 455                                      func->name);
 456                                 return -1;
 457                         }
 458
 459                         if (func->embedded_insn || func->alias != func)
 460                                 continue;
 461
 462                         if (!find_insn(file, sec, func->offset)) {
 463                                 WARN("%s(): can't find starting instruction",
 464                                      func->name);
 465                                 return -1;
 466                         }
 467
 468                         sym_for_each_insn(file, func, insn) {
 469                                 insn->sym = func;
 470                                 if (func->type == STT_FUNC &&
 471                                     insn->type == INSN_ENDBR &&
 472                                     list_empty(&insn->call_node)) {
 473                                         if (insn->offset == func->offset) {
 474                                                 list_add_tail(&insn->call_node, &file->endbr_list);
 475                                                 file->nr_endbr++;
 476                                         } else {
 477                                                 file->nr_endbr_int++;
 478                                         }
 479                                 }
 480                         }
 481                 }
 482         }
 483
 484         if (opts.stats)
 485                 printf("nr_insns: %lu\n", nr_insns);
 486
 487         return 0;
 488 }
 489
 490 /*
 491  * Read the pv_ops[] .data table to find the static initialized values.
 492  */
 493 static int add_pv_ops(struct objtool_file *file, const char *symname)
 494 {
 495         struct symbol *sym, *func;
 496         unsigned long off, end;
 497         struct reloc *reloc;
 498         int idx;
 499
 500         sym = find_symbol_by_name(file->elf, symname);
 501         if (!sym)
 502                 return 0;
 503
 504         off = sym->offset;
 505         end = off + sym->len;
 506         for (;;) {
 507                 reloc = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off);
 508                 if (!reloc)
 509                         break;
 510
 511                 func = reloc->sym;
 512                 if (func->type == STT_SECTION)
 513                         func = find_symbol_by_offset(reloc->sym->sec,
 514                                                      reloc_addend(reloc));
 515
 516                 idx = (reloc_offset(reloc) - sym->offset) / sizeof(unsigned long);
 517
 518                 objtool_pv_add(file, idx, func);
 519
 520                 off = reloc_offset(reloc) + 1;
 521                 if (off > end)
 522                         break;
 523         }
 524
 525         return 0;
 526 }
 527
 528 /*
 529  * Allocate and initialize file->pv_ops[].
 530  */
 531 static int init_pv_ops(struct objtool_file *file)
 532 {
 533         static const char *pv_ops_tables[] = {
 534                 "pv_ops",
 535                 "xen_cpu_ops",
 536                 "xen_irq_ops",
 537                 "xen_mmu_ops",
 538                 NULL,
 539         };
 540         const char *pv_ops;
 541         struct symbol *sym;
 542         int idx, nr;
 543
 544         if (!opts.noinstr)
 545                 return 0;
 546
 547         file->pv_ops = NULL;
 548
 549         sym = find_symbol_by_name(file->elf, "pv_ops");
 550         if (!sym)
 551                 return 0;
 552
 553         nr = sym->len / sizeof(unsigned long);
 554         file->pv_ops = calloc(sizeof(struct pv_state), nr);
 555         if (!file->pv_ops)
 556                 return -1;
 557
 558         for (idx = 0; idx < nr; idx++)
 559                 INIT_LIST_HEAD(&file->pv_ops[idx].targets);
 560
 561         for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++)
 562                 add_pv_ops(file, pv_ops);
 563
 564         return 0;
 565 }
 566
 567 static struct instruction *find_last_insn(struct objtool_file *file,
 568                                           struct section *sec)
 569 {
 570         struct instruction *insn = NULL;
 571         unsigned int offset;
 572         unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0;
 573
 574         for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--)
 575                 insn = find_insn(file, sec, offset);
 576
 577         return insn;
 578 }
 579
 580 /*
 581  * Mark "ud2" instructions and manually annotated dead ends.
 582  */
 583 static int add_dead_ends(struct objtool_file *file)
 584 {
 585         struct section *rsec;
 586         struct reloc *reloc;
 587         struct instruction *insn;
 588         uint64_t offset;
 589
 590         /*
 591          * Check for manually annotated dead ends.
 592          */
 593         rsec = find_section_by_name(file->elf, ".rela.discard.unreachable");
 594         if (!rsec)
 595                 goto reachable;
 596
 597         for_each_reloc(rsec, reloc) {
 598                 if (reloc->sym->type == STT_SECTION) {
 599                         offset = reloc_addend(reloc);
 600                 } else if (reloc->sym->local_label) {
 601                         offset = reloc->sym->offset;
 602                 } else {
 603                         WARN("unexpected relocation symbol type in %s", rsec->name);
 604                         return -1;
 605                 }
 606
 607                 insn = find_insn(file, reloc->sym->sec, offset);
 608                 if (insn)
 609                         insn = prev_insn_same_sec(file, insn);
 610                 else if (offset == reloc->sym->sec->sh.sh_size) {
 611                         insn = find_last_insn(file, reloc->sym->sec);
 612                         if (!insn) {
 613                                 WARN("can't find unreachable insn at %s+0x%" PRIx64,
 614                                      reloc->sym->sec->name, offset);
 615                                 return -1;
 616                         }
 617                 } else {
 618                         WARN("can't find unreachable insn at %s+0x%" PRIx64,
 619                              reloc->sym->sec->name, offset);
 620                         return -1;
 621                 }
 622
 623                 insn->dead_end = true;
 624         }
 625
 626 reachable:
 627         /*
 628          * These manually annotated reachable checks are needed for GCC 4.4,
 629          * where the Linux unreachable() macro isn't supported.  In that case
 630          * GCC doesn't know the "ud2" is fatal, so it generates code as if it's
 631          * not a dead end.
 632          */
 633         rsec = find_section_by_name(file->elf, ".rela.discard.reachable");
 634         if (!rsec)
 635                 return 0;
 636
 637         for_each_reloc(rsec, reloc) {
 638                 if (reloc->sym->type == STT_SECTION) {
 639                         offset = reloc_addend(reloc);
 640                 } else if (reloc->sym->local_label) {
 641                         offset = reloc->sym->offset;
 642                 } else {
 643                         WARN("unexpected relocation symbol type in %s", rsec->name);
 644                         return -1;
 645                 }
 646
 647                 insn = find_insn(file, reloc->sym->sec, offset);
 648                 if (insn)
 649                         insn = prev_insn_same_sec(file, insn);
 650                 else if (offset == reloc->sym->sec->sh.sh_size) {
 651                         insn = find_last_insn(file, reloc->sym->sec);
 652                         if (!insn) {
 653                                 WARN("can't find reachable insn at %s+0x%" PRIx64,
 654                                      reloc->sym->sec->name, offset);
 655                                 return -1;
 656                         }
 657                 } else {
 658                         WARN("can't find reachable insn at %s+0x%" PRIx64,
 659                              reloc->sym->sec->name, offset);
 660                         return -1;
 661                 }
 662
 663                 insn->dead_end = false;
 664         }
 665
 666         return 0;
 667 }
 668
 669 static int create_static_call_sections(struct objtool_file *file)
 670 {
 671         struct static_call_site *site;
 672         struct section *sec;
 673         struct instruction *insn;
 674         struct symbol *key_sym;
 675         char *key_name, *tmp;
 676         int idx;
 677
 678         sec = find_section_by_name(file->elf, ".static_call_sites");
 679         if (sec) {
 680                 INIT_LIST_HEAD(&file->static_call_list);
 681                 WARN("file already has .static_call_sites section, skipping");
 682                 return 0;
 683         }
 684
 685         if (list_empty(&file->static_call_list))
 686                 return 0;
 687
 688         idx = 0;
 689         list_for_each_entry(insn, &file->static_call_list, call_node)
 690                 idx++;
 691
 692         sec = elf_create_section_pair(file->elf, ".static_call_sites",
 693                                       sizeof(*site), idx, idx * 2);
 694         if (!sec)
 695                 return -1;
 696
 697         /* Allow modules to modify the low bits of static_call_site::key */
 698         sec->sh.sh_flags |= SHF_WRITE;
 699
 700         idx = 0;
 701         list_for_each_entry(insn, &file->static_call_list, call_node) {
 702
 703                 /* populate reloc for 'addr' */
 704                 if (!elf_init_reloc_text_sym(file->elf, sec,
 705                                              idx * sizeof(*site), idx * 2,
 706                                              insn->sec, insn->offset))
 707                         return -1;
 708
 709                 /* find key symbol */
 710                 key_name = strdup(insn_call_dest(insn)->name);
 711                 if (!key_name) {
 712                         perror("strdup");
 713                         return -1;
 714                 }
 715                 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
 716                             STATIC_CALL_TRAMP_PREFIX_LEN)) {
 717                         WARN("static_call: trampoline name malformed: %s", key_name);
 718                         free(key_name);
 719                         return -1;
 720                 }
 721                 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
 722                 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
 723
 724                 key_sym = find_symbol_by_name(file->elf, tmp);
 725                 if (!key_sym) {
 726                         if (!opts.module) {
 727                                 WARN("static_call: can't find static_call_key symbol: %s", tmp);
 728                                 free(key_name);
 729                                 return -1;
 730                         }
 731
 732                         /*
 733                          * For modules(), the key might not be exported, which
 734                          * means the module can make static calls but isn't
 735                          * allowed to change them.
 736                          *
 737                          * In that case we temporarily set the key to be the
 738                          * trampoline address.  This is fixed up in
 739                          * static_call_add_module().
 740                          */
 741                         key_sym = insn_call_dest(insn);
 742                 }
 743                 free(key_name);
 744
 745                 /* populate reloc for 'key' */
 746                 if (!elf_init_reloc_data_sym(file->elf, sec,
 747                                              idx * sizeof(*site) + 4,
 748                                              (idx * 2) + 1, key_sym,
 749                                              is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
 750                         return -1;
 751
 752                 idx++;
 753         }
 754
 755         return 0;
 756 }
 757
 758 static int create_retpoline_sites_sections(struct objtool_file *file)
 759 {
 760         struct instruction *insn;
 761         struct section *sec;
 762         int idx;
 763
 764         sec = find_section_by_name(file->elf, ".retpoline_sites");
 765         if (sec) {
 766                 WARN("file already has .retpoline_sites, skipping");
 767                 return 0;
 768         }
 769
 770         idx = 0;
 771         list_for_each_entry(insn, &file->retpoline_call_list, call_node)
 772                 idx++;
 773
 774         if (!idx)
 775                 return 0;
 776
 777         sec = elf_create_section_pair(file->elf, ".retpoline_sites",
 778                                       sizeof(int), idx, idx);
 779         if (!sec)
 780                 return -1;
 781
 782         idx = 0;
 783         list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
 784
 785                 if (!elf_init_reloc_text_sym(file->elf, sec,
 786                                              idx * sizeof(int), idx,
 787                                              insn->sec, insn->offset))
 788                         return -1;
 789
 790                 idx++;
 791         }
 792
 793         return 0;
 794 }
 795
 796 static int create_return_sites_sections(struct objtool_file *file)
 797 {
 798         struct instruction *insn;
 799         struct section *sec;
 800         int idx;
 801
 802         sec = find_section_by_name(file->elf, ".return_sites");
 803         if (sec) {
 804                 WARN("file already has .return_sites, skipping");
 805                 return 0;
 806         }
 807
 808         idx = 0;
 809         list_for_each_entry(insn, &file->return_thunk_list, call_node)
 810                 idx++;
 811
 812         if (!idx)
 813                 return 0;
 814
 815         sec = elf_create_section_pair(file->elf, ".return_sites",
 816                                       sizeof(int), idx, idx);
 817         if (!sec)
 818                 return -1;
 819
 820         idx = 0;
 821         list_for_each_entry(insn, &file->return_thunk_list, call_node) {
 822
 823                 if (!elf_init_reloc_text_sym(file->elf, sec,
 824                                              idx * sizeof(int), idx,
 825                                              insn->sec, insn->offset))
 826                         return -1;
 827
 828                 idx++;
 829         }
 830
 831         return 0;
 832 }
 833
 834 static int create_ibt_endbr_seal_sections(struct objtool_file *file)
 835 {
 836         struct instruction *insn;
 837         struct section *sec;
 838         int idx;
 839
 840         sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
 841         if (sec) {
 842                 WARN("file already has .ibt_endbr_seal, skipping");
 843                 return 0;
 844         }
 845
 846         idx = 0;
 847         list_for_each_entry(insn, &file->endbr_list, call_node)
 848                 idx++;
 849
 850         if (opts.stats) {
 851                 printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
 852                 printf("ibt: ENDBR inside functions:  %d\n", file->nr_endbr_int);
 853                 printf("ibt: superfluous ENDBR:       %d\n", idx);
 854         }
 855
 856         if (!idx)
 857                 return 0;
 858
 859         sec = elf_create_section_pair(file->elf, ".ibt_endbr_seal",
 860                                       sizeof(int), idx, idx);
 861         if (!sec)
 862                 return -1;
 863
 864         idx = 0;
 865         list_for_each_entry(insn, &file->endbr_list, call_node) {
 866
 867                 int *site = (int *)sec->data->d_buf + idx;
 868                 struct symbol *sym = insn->sym;
 869                 *site = 0;
 870
 871                 if (opts.module && sym && sym->type == STT_FUNC &&
 872                     insn->offset == sym->offset &&
 873                     (!strcmp(sym->name, "init_module") ||
 874                      !strcmp(sym->name, "cleanup_module")))
 875                         WARN("%s(): not an indirect call target", sym->name);
 876
 877                 if (!elf_init_reloc_text_sym(file->elf, sec,
 878                                              idx * sizeof(int), idx,
 879                                              insn->sec, insn->offset))
 880                         return -1;
 881
 882                 idx++;
 883         }
 884
 885         return 0;
 886 }
 887
 888 static int create_cfi_sections(struct objtool_file *file)
 889 {
 890         struct section *sec;
 891         struct symbol *sym;
 892         int idx;
 893
 894         sec = find_section_by_name(file->elf, ".cfi_sites");
 895         if (sec) {
 896                 INIT_LIST_HEAD(&file->call_list);
 897                 WARN("file already has .cfi_sites section, skipping");
 898                 return 0;
 899         }
 900
 901         idx = 0;
 902         for_each_sym(file, sym) {
 903                 if (sym->type != STT_FUNC)
 904                         continue;
 905
 906                 if (strncmp(sym->name, "__cfi_", 6))
 907                         continue;
 908
 909                 idx++;
 910         }
 911
 912         sec = elf_create_section_pair(file->elf, ".cfi_sites",
 913                                       sizeof(unsigned int), idx, idx);
 914         if (!sec)
 915                 return -1;
 916
 917         idx = 0;
 918         for_each_sym(file, sym) {
 919                 if (sym->type != STT_FUNC)
 920                         continue;
 921
 922                 if (strncmp(sym->name, "__cfi_", 6))
 923                         continue;
 924
 925                 if (!elf_init_reloc_text_sym(file->elf, sec,
 926                                              idx * sizeof(unsigned int), idx,
 927                                              sym->sec, sym->offset))
 928                         return -1;
 929
 930                 idx++;
 931         }
 932
 933         return 0;
 934 }
 935
 936 static int create_mcount_loc_sections(struct objtool_file *file)
 937 {
 938         size_t addr_size = elf_addr_size(file->elf);
 939         struct instruction *insn;
 940         struct section *sec;
 941         int idx;
 942
 943         sec = find_section_by_name(file->elf, "__mcount_loc");
 944         if (sec) {
 945                 INIT_LIST_HEAD(&file->mcount_loc_list);
 946                 WARN("file already has __mcount_loc section, skipping");
 947                 return 0;
 948         }
 949
 950         if (list_empty(&file->mcount_loc_list))
 951                 return 0;
 952
 953         idx = 0;
 954         list_for_each_entry(insn, &file->mcount_loc_list, call_node)
 955                 idx++;
 956
 957         sec = elf_create_section_pair(file->elf, "__mcount_loc", addr_size,
 958                                       idx, idx);
 959         if (!sec)
 960                 return -1;
 961
 962         sec->sh.sh_addralign = addr_size;
 963
 964         idx = 0;
 965         list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
 966
 967                 struct reloc *reloc;
 968
 969                 reloc = elf_init_reloc_text_sym(file->elf, sec, idx * addr_size, idx,
 970                                                insn->sec, insn->offset);
 971                 if (!reloc)
 972                         return -1;
 973
 974                 set_reloc_type(file->elf, reloc, addr_size == 8 ? R_ABS64 : R_ABS32);
 975
 976                 idx++;
 977         }
 978
 979         return 0;
 980 }
 981
 982 static int create_direct_call_sections(struct objtool_file *file)
 983 {
 984         struct instruction *insn;
 985         struct section *sec;
 986         int idx;
 987
 988         sec = find_section_by_name(file->elf, ".call_sites");
 989         if (sec) {
 990                 INIT_LIST_HEAD(&file->call_list);
 991                 WARN("file already has .call_sites section, skipping");
 992                 return 0;
 993         }
 994
 995         if (list_empty(&file->call_list))
 996                 return 0;
 997
 998         idx = 0;
 999         list_for_each_entry(insn, &file->call_list, call_node)
1000                 idx++;
1001
1002         sec = elf_create_section_pair(file->elf, ".call_sites",
1003                                       sizeof(unsigned int), idx, idx);
1004         if (!sec)
1005                 return -1;
1006
1007         idx = 0;
1008         list_for_each_entry(insn, &file->call_list, call_node) {
1009
1010                 if (!elf_init_reloc_text_sym(file->elf, sec,
1011                                              idx * sizeof(unsigned int), idx,
1012                                              insn->sec, insn->offset))
1013                         return -1;
1014
1015                 idx++;
1016         }
1017
1018         return 0;
1019 }
1020
1021 /*
1022  * Warnings shouldn't be reported for ignored functions.
1023  */
1024 static void add_ignores(struct objtool_file *file)
1025 {
1026         struct instruction *insn;
1027         struct section *rsec;
1028         struct symbol *func;
1029         struct reloc *reloc;
1030
1031         rsec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
1032         if (!rsec)
1033                 return;
1034
1035         for_each_reloc(rsec, reloc) {
1036                 switch (reloc->sym->type) {
1037                 case STT_FUNC:
1038                         func = reloc->sym;
1039                         break;
1040
1041                 case STT_SECTION:
1042                         func = find_func_by_offset(reloc->sym->sec, reloc_addend(reloc));
1043                         if (!func)
1044                                 continue;
1045                         break;
1046
1047                 default:
1048                         WARN("unexpected relocation symbol type in %s: %d",
1049                              rsec->name, reloc->sym->type);
1050                         continue;
1051                 }
1052
1053                 func_for_each_insn(file, func, insn)
1054                         insn->ignore = true;
1055         }
1056 }
1057
1058 /*
1059  * This is a whitelist of functions that is allowed to be called with AC set.
1060  * The list is meant to be minimal and only contains compiler instrumentation
1061  * ABI and a few functions used to implement *_{to,from}_user() functions.
1062  *
1063  * These functions must not directly change AC, but may PUSHF/POPF.
1064  */
1065 static const char *uaccess_safe_builtin[] = {
1066         /* KASAN */
1067         "kasan_report",
1068         "kasan_check_range",
1069         /* KASAN out-of-line */
1070         "__asan_loadN_noabort",
1071         "__asan_load1_noabort",
1072         "__asan_load2_noabort",
1073         "__asan_load4_noabort",
1074         "__asan_load8_noabort",
1075         "__asan_load16_noabort",
1076         "__asan_storeN_noabort",
1077         "__asan_store1_noabort",
1078         "__asan_store2_noabort",
1079         "__asan_store4_noabort",
1080         "__asan_store8_noabort",
1081         "__asan_store16_noabort",
1082         "__kasan_check_read",
1083         "__kasan_check_write",
1084         /* KASAN in-line */
1085         "__asan_report_load_n_noabort",
1086         "__asan_report_load1_noabort",
1087         "__asan_report_load2_noabort",
1088         "__asan_report_load4_noabort",
1089         "__asan_report_load8_noabort",
1090         "__asan_report_load16_noabort",
1091         "__asan_report_store_n_noabort",
1092         "__asan_report_store1_noabort",
1093         "__asan_report_store2_noabort",
1094         "__asan_report_store4_noabort",
1095         "__asan_report_store8_noabort",
1096         "__asan_report_store16_noabort",
1097         /* KCSAN */
1098         "__kcsan_check_access",
1099         "__kcsan_mb",
1100         "__kcsan_wmb",
1101         "__kcsan_rmb",
1102         "__kcsan_release",
1103         "kcsan_found_watchpoint",
1104         "kcsan_setup_watchpoint",
1105         "kcsan_check_scoped_accesses",
1106         "kcsan_disable_current",
1107         "kcsan_enable_current_nowarn",
1108         /* KCSAN/TSAN */
1109         "__tsan_func_entry",
1110         "__tsan_func_exit",
1111         "__tsan_read_range",
1112         "__tsan_write_range",
1113         "__tsan_read1",
1114         "__tsan_read2",
1115         "__tsan_read4",
1116         "__tsan_read8",
1117         "__tsan_read16",
1118         "__tsan_write1",
1119         "__tsan_write2",
1120         "__tsan_write4",
1121         "__tsan_write8",
1122         "__tsan_write16",
1123         "__tsan_read_write1",
1124         "__tsan_read_write2",
1125         "__tsan_read_write4",
1126         "__tsan_read_write8",
1127         "__tsan_read_write16",
1128         "__tsan_volatile_read1",
1129         "__tsan_volatile_read2",
1130         "__tsan_volatile_read4",
1131         "__tsan_volatile_read8",
1132         "__tsan_volatile_read16",
1133         "__tsan_volatile_write1",
1134         "__tsan_volatile_write2",
1135         "__tsan_volatile_write4",
1136         "__tsan_volatile_write8",
1137         "__tsan_volatile_write16",
1138         "__tsan_atomic8_load",
1139         "__tsan_atomic16_load",
1140         "__tsan_atomic32_load",
1141         "__tsan_atomic64_load",
1142         "__tsan_atomic8_store",
1143         "__tsan_atomic16_store",
1144         "__tsan_atomic32_store",
1145         "__tsan_atomic64_store",
1146         "__tsan_atomic8_exchange",
1147         "__tsan_atomic16_exchange",
1148         "__tsan_atomic32_exchange",
1149         "__tsan_atomic64_exchange",
1150         "__tsan_atomic8_fetch_add",
1151         "__tsan_atomic16_fetch_add",
1152         "__tsan_atomic32_fetch_add",
1153         "__tsan_atomic64_fetch_add",
1154         "__tsan_atomic8_fetch_sub",
1155         "__tsan_atomic16_fetch_sub",
1156         "__tsan_atomic32_fetch_sub",
1157         "__tsan_atomic64_fetch_sub",
1158         "__tsan_atomic8_fetch_and",
1159         "__tsan_atomic16_fetch_and",
1160         "__tsan_atomic32_fetch_and",
1161         "__tsan_atomic64_fetch_and",
1162         "__tsan_atomic8_fetch_or",
1163         "__tsan_atomic16_fetch_or",
1164         "__tsan_atomic32_fetch_or",
1165         "__tsan_atomic64_fetch_or",
1166         "__tsan_atomic8_fetch_xor",
1167         "__tsan_atomic16_fetch_xor",
1168         "__tsan_atomic32_fetch_xor",
1169         "__tsan_atomic64_fetch_xor",
1170         "__tsan_atomic8_fetch_nand",
1171         "__tsan_atomic16_fetch_nand",
1172         "__tsan_atomic32_fetch_nand",
1173         "__tsan_atomic64_fetch_nand",
1174         "__tsan_atomic8_compare_exchange_strong",
1175         "__tsan_atomic16_compare_exchange_strong",
1176         "__tsan_atomic32_compare_exchange_strong",
1177         "__tsan_atomic64_compare_exchange_strong",
1178         "__tsan_atomic8_compare_exchange_weak",
1179         "__tsan_atomic16_compare_exchange_weak",
1180         "__tsan_atomic32_compare_exchange_weak",
1181         "__tsan_atomic64_compare_exchange_weak",
1182         "__tsan_atomic8_compare_exchange_val",
1183         "__tsan_atomic16_compare_exchange_val",
1184         "__tsan_atomic32_compare_exchange_val",
1185         "__tsan_atomic64_compare_exchange_val",
1186         "__tsan_atomic_thread_fence",
1187         "__tsan_atomic_signal_fence",
1188         "__tsan_unaligned_read16",
1189         "__tsan_unaligned_write16",
1190         /* KCOV */
1191         "write_comp_data",
1192         "check_kcov_mode",
1193         "__sanitizer_cov_trace_pc",
1194         "__sanitizer_cov_trace_const_cmp1",
1195         "__sanitizer_cov_trace_const_cmp2",
1196         "__sanitizer_cov_trace_const_cmp4",
1197         "__sanitizer_cov_trace_const_cmp8",
1198         "__sanitizer_cov_trace_cmp1",
1199         "__sanitizer_cov_trace_cmp2",
1200         "__sanitizer_cov_trace_cmp4",
1201         "__sanitizer_cov_trace_cmp8",
1202         "__sanitizer_cov_trace_switch",
1203         /* KMSAN */
1204         "kmsan_copy_to_user",
1205         "kmsan_disable_current",
1206         "kmsan_enable_current",
1207         "kmsan_report",
1208         "kmsan_unpoison_entry_regs",
1209         "kmsan_unpoison_memory",
1210         "__msan_chain_origin",
1211         "__msan_get_context_state",
1212         "__msan_instrument_asm_store",
1213         "__msan_metadata_ptr_for_load_1",
1214         "__msan_metadata_ptr_for_load_2",
1215         "__msan_metadata_ptr_for_load_4",
1216         "__msan_metadata_ptr_for_load_8",
1217         "__msan_metadata_ptr_for_load_n",
1218         "__msan_metadata_ptr_for_store_1",
1219         "__msan_metadata_ptr_for_store_2",
1220         "__msan_metadata_ptr_for_store_4",
1221         "__msan_metadata_ptr_for_store_8",
1222         "__msan_metadata_ptr_for_store_n",
1223         "__msan_poison_alloca",
1224         "__msan_warning",
1225         /* UBSAN */
1226         "ubsan_type_mismatch_common",
1227         "__ubsan_handle_type_mismatch",
1228         "__ubsan_handle_type_mismatch_v1",
1229         "__ubsan_handle_shift_out_of_bounds",
1230         "__ubsan_handle_load_invalid_value",
1231         /* STACKLEAK */
1232         "stackleak_track_stack",
1233         /* misc */
1234         "csum_partial_copy_generic",
1235         "copy_mc_fragile",
1236         "copy_mc_fragile_handle_tail",
1237         "copy_mc_enhanced_fast_string",
1238         "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
1239         "rep_stos_alternative",
1240         "rep_movs_alternative",
1241         "__copy_user_nocache",
1242         NULL
1243 };
1244
1245 static void add_uaccess_safe(struct objtool_file *file)
1246 {
1247         struct symbol *func;
1248         const char **name;
1249
1250         if (!opts.uaccess)
1251                 return;
1252
1253         for (name = uaccess_safe_builtin; *name; name++) {
1254                 func = find_symbol_by_name(file->elf, *name);
1255                 if (!func)
1256                         continue;
1257
1258                 func->uaccess_safe = true;
1259         }
1260 }
1261
1262 /*
1263  * FIXME: For now, just ignore any alternatives which add retpolines.  This is
1264  * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
1265  * But it at least allows objtool to understand the control flow *around* the
1266  * retpoline.
1267  */
1268 static int add_ignore_alternatives(struct objtool_file *file)
1269 {
1270         struct section *rsec;
1271         struct reloc *reloc;
1272         struct instruction *insn;
1273
1274         rsec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
1275         if (!rsec)
1276                 return 0;
1277
1278         for_each_reloc(rsec, reloc) {
1279                 if (reloc->sym->type != STT_SECTION) {
1280                         WARN("unexpected relocation symbol type in %s", rsec->name);
1281                         return -1;
1282                 }
1283
1284                 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
1285                 if (!insn) {
1286                         WARN("bad .discard.ignore_alts entry");
1287                         return -1;
1288                 }
1289
1290                 insn->ignore_alts = true;
1291         }
1292
1293         return 0;
1294 }
1295
1296 /*
1297  * Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol
1298  * will be added to the .retpoline_sites section.
1299  */
1300 __weak bool arch_is_retpoline(struct symbol *sym)
1301 {
1302         return false;
1303 }
1304
1305 /*
1306  * Symbols that replace INSN_RETURN, every (tail) call to such a symbol
1307  * will be added to the .return_sites section.
1308  */
1309 __weak bool arch_is_rethunk(struct symbol *sym)
1310 {
1311         return false;
1312 }
1313
1314 /*
1315  * Symbols that are embedded inside other instructions, because sometimes crazy
1316  * code exists. These are mostly ignored for validation purposes.
1317  */
1318 __weak bool arch_is_embedded_insn(struct symbol *sym)
1319 {
1320         return false;
1321 }
1322
1323 static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
1324 {
1325         struct reloc *reloc;
1326
1327         if (insn->no_reloc)
1328                 return NULL;
1329
1330         if (!file)
1331                 return NULL;
1332
1333         reloc = find_reloc_by_dest_range(file->elf, insn->sec,
1334                                          insn->offset, insn->len);
1335         if (!reloc) {
1336                 insn->no_reloc = 1;
1337                 return NULL;
1338         }
1339
1340         return reloc;
1341 }
1342
1343 static void remove_insn_ops(struct instruction *insn)
1344 {
1345         struct stack_op *op, *next;
1346
1347         for (op = insn->stack_ops; op; op = next) {
1348                 next = op->next;
1349                 free(op);
1350         }
1351         insn->stack_ops = NULL;
1352 }
1353
1354 static void annotate_call_site(struct objtool_file *file,
1355                                struct instruction *insn, bool sibling)
1356 {
1357         struct reloc *reloc = insn_reloc(file, insn);
1358         struct symbol *sym = insn_call_dest(insn);
1359
1360         if (!sym)
1361                 sym = reloc->sym;
1362
1363         /*
1364          * Alternative replacement code is just template code which is
1365          * sometimes copied to the original instruction. For now, don't
1366          * annotate it. (In the future we might consider annotating the
1367          * original instruction if/when it ever makes sense to do so.)
1368          */
1369         if (!strcmp(insn->sec->name, ".altinstr_replacement"))
1370                 return;
1371
1372         if (sym->static_call_tramp) {
1373                 list_add_tail(&insn->call_node, &file->static_call_list);
1374                 return;
1375         }
1376
1377         if (sym->retpoline_thunk) {
1378                 list_add_tail(&insn->call_node, &file->retpoline_call_list);
1379                 return;
1380         }
1381
1382         /*
1383          * Many compilers cannot disable KCOV or sanitizer calls with a function
1384          * attribute so they need a little help, NOP out any such calls from
1385          * noinstr text.
1386          */
1387         if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) {
1388                 if (reloc)
1389                         set_reloc_type(file->elf, reloc, R_NONE);
1390
1391                 elf_write_insn(file->elf, insn->sec,
1392                                insn->offset, insn->len,
1393                                sibling ? arch_ret_insn(insn->len)
1394                                        : arch_nop_insn(insn->len));
1395
1396                 insn->type = sibling ? INSN_RETURN : INSN_NOP;
1397
1398                 if (sibling) {
1399                         /*
1400                          * We've replaced the tail-call JMP insn by two new
1401                          * insn: RET; INT3, except we only have a single struct
1402                          * insn here. Mark it retpoline_safe to avoid the SLS
1403                          * warning, instead of adding another insn.
1404                          */
1405                         insn->retpoline_safe = true;
1406                 }
1407
1408                 return;
1409         }
1410
1411         if (opts.mcount && sym->fentry) {
1412                 if (sibling)
1413                         WARN_INSN(insn, "tail call to __fentry__ !?!?");
1414                 if (opts.mnop) {
1415                         if (reloc)
1416                                 set_reloc_type(file->elf, reloc, R_NONE);
1417
1418                         elf_write_insn(file->elf, insn->sec,
1419                                        insn->offset, insn->len,
1420                                        arch_nop_insn(insn->len));
1421
1422                         insn->type = INSN_NOP;
1423                 }
1424
1425                 list_add_tail(&insn->call_node, &file->mcount_loc_list);
1426                 return;
1427         }
1428
1429         if (insn->type == INSN_CALL && !insn->sec->init)
1430                 list_add_tail(&insn->call_node, &file->call_list);
1431
1432         if (!sibling && dead_end_function(file, sym))
1433                 insn->dead_end = true;
1434 }
1435
1436 static void add_call_dest(struct objtool_file *file, struct instruction *insn,
1437                           struct symbol *dest, bool sibling)
1438 {
1439         insn->_call_dest = dest;
1440         if (!dest)
1441                 return;
1442
1443         /*
1444          * Whatever stack impact regular CALLs have, should be undone
1445          * by the RETURN of the called function.
1446          *
1447          * Annotated intra-function calls retain the stack_ops but
1448          * are converted to JUMP, see read_intra_function_calls().
1449          */
1450         remove_insn_ops(insn);
1451
1452         annotate_call_site(file, insn, sibling);
1453 }
1454
1455 static void add_retpoline_call(struct objtool_file *file, struct instruction *insn)
1456 {
1457         /*
1458          * Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1459          * so convert them accordingly.
1460          */
1461         switch (insn->type) {
1462         case INSN_CALL:
1463                 insn->type = INSN_CALL_DYNAMIC;
1464                 break;
1465         case INSN_JUMP_UNCONDITIONAL:
1466                 insn->type = INSN_JUMP_DYNAMIC;
1467                 break;
1468         case INSN_JUMP_CONDITIONAL:
1469                 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1470                 break;
1471         default:
1472                 return;
1473         }
1474
1475         insn->retpoline_safe = true;
1476
1477         /*
1478          * Whatever stack impact regular CALLs have, should be undone
1479          * by the RETURN of the called function.
1480          *
1481          * Annotated intra-function calls retain the stack_ops but
1482          * are converted to JUMP, see read_intra_function_calls().
1483          */
1484         remove_insn_ops(insn);
1485
1486         annotate_call_site(file, insn, false);
1487 }
1488
1489 static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add)
1490 {
1491         /*
1492          * Return thunk tail calls are really just returns in disguise,
1493          * so convert them accordingly.
1494          */
1495         insn->type = INSN_RETURN;
1496         insn->retpoline_safe = true;
1497
1498         if (add)
1499                 list_add_tail(&insn->call_node, &file->return_thunk_list);
1500 }
1501
1502 static bool is_first_func_insn(struct objtool_file *file,
1503                                struct instruction *insn, struct symbol *sym)
1504 {
1505         if (insn->offset == sym->offset)
1506                 return true;
1507
1508         /* Allow direct CALL/JMP past ENDBR */
1509         if (opts.ibt) {
1510                 struct instruction *prev = prev_insn_same_sym(file, insn);
1511
1512                 if (prev && prev->type == INSN_ENDBR &&
1513                     insn->offset == sym->offset + prev->len)
1514                         return true;
1515         }
1516
1517         return false;
1518 }
1519
1520 /*
1521  * A sibling call is a tail-call to another symbol -- to differentiate from a
1522  * recursive tail-call which is to the same symbol.
1523  */
1524 static bool jump_is_sibling_call(struct objtool_file *file,
1525                                  struct instruction *from, struct instruction *to)
1526 {
1527         struct symbol *fs = from->sym;
1528         struct symbol *ts = to->sym;
1529
1530         /* Not a sibling call if from/to a symbol hole */
1531         if (!fs || !ts)
1532                 return false;
1533
1534         /* Not a sibling call if not targeting the start of a symbol. */
1535         if (!is_first_func_insn(file, to, ts))
1536                 return false;
1537
1538         /* Disallow sibling calls into STT_NOTYPE */
1539         if (ts->type == STT_NOTYPE)
1540                 return false;
1541
1542         /* Must not be self to be a sibling */
1543         return fs->pfunc != ts->pfunc;
1544 }
1545
1546 /*
1547  * Find the destination instructions for all jumps.
1548  */
1549 static int add_jump_destinations(struct objtool_file *file)
1550 {
1551         struct instruction *insn, *jump_dest;
1552         struct reloc *reloc;
1553         struct section *dest_sec;
1554         unsigned long dest_off;
1555
1556         for_each_insn(file, insn) {
1557                 if (insn->jump_dest) {
1558                         /*
1559                          * handle_group_alt() may have previously set
1560                          * 'jump_dest' for some alternatives.
1561                          */
1562                         continue;
1563                 }
1564                 if (!is_static_jump(insn))
1565                         continue;
1566
1567                 reloc = insn_reloc(file, insn);
1568                 if (!reloc) {
1569                         dest_sec = insn->sec;
1570                         dest_off = arch_jump_destination(insn);
1571                 } else if (reloc->sym->type == STT_SECTION) {
1572                         dest_sec = reloc->sym->sec;
1573                         dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1574                 } else if (reloc->sym->retpoline_thunk) {
1575                         add_retpoline_call(file, insn);
1576                         continue;
1577                 } else if (reloc->sym->return_thunk) {
1578                         add_return_call(file, insn, true);
1579                         continue;
1580                 } else if (insn_func(insn)) {
1581                         /*
1582                          * External sibling call or internal sibling call with
1583                          * STT_FUNC reloc.
1584                          */
1585                         add_call_dest(file, insn, reloc->sym, true);
1586                         continue;
1587                 } else if (reloc->sym->sec->idx) {
1588                         dest_sec = reloc->sym->sec;
1589                         dest_off = reloc->sym->sym.st_value +
1590                                    arch_dest_reloc_offset(reloc_addend(reloc));
1591                 } else {
1592                         /* non-func asm code jumping to another file */
1593                         continue;
1594                 }
1595
1596                 jump_dest = find_insn(file, dest_sec, dest_off);
1597                 if (!jump_dest) {
1598                         struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
1599
1600                         /*
1601                          * This is a special case for retbleed_untrain_ret().
1602                          * It jumps to __x86_return_thunk(), but objtool
1603                          * can't find the thunk's starting RET
1604                          * instruction, because the RET is also in the
1605                          * middle of another instruction.  Objtool only
1606                          * knows about the outer instruction.
1607                          */
1608                         if (sym && sym->embedded_insn) {
1609                                 add_return_call(file, insn, false);
1610                                 continue;
1611                         }
1612
1613                         WARN_INSN(insn, "can't find jump dest instruction at %s+0x%lx",
1614                                   dest_sec->name, dest_off);
1615                         return -1;
1616                 }
1617
1618                 /*
1619                  * An intra-TU jump in retpoline.o might not have a relocation
1620                  * for its jump dest, in which case the above
1621                  * add_{retpoline,return}_call() didn't happen.
1622                  */
1623                 if (jump_dest->sym && jump_dest->offset == jump_dest->sym->offset) {
1624                         if (jump_dest->sym->retpoline_thunk) {
1625                                 add_retpoline_call(file, insn);
1626                                 continue;
1627                         }
1628                         if (jump_dest->sym->return_thunk) {
1629                                 add_return_call(file, insn, true);
1630                                 continue;
1631                         }
1632                 }
1633
1634                 /*
1635                  * Cross-function jump.
1636                  */
1637                 if (insn_func(insn) && insn_func(jump_dest) &&
1638                     insn_func(insn) != insn_func(jump_dest)) {
1639
1640                         /*
1641                          * For GCC 8+, create parent/child links for any cold
1642                          * subfunctions.  This is _mostly_ redundant with a
1643                          * similar initialization in read_symbols().
1644                          *
1645                          * If a function has aliases, we want the *first* such
1646                          * function in the symbol table to be the subfunction's
1647                          * parent.  In that case we overwrite the
1648                          * initialization done in read_symbols().
1649                          *
1650                          * However this code can't completely replace the
1651                          * read_symbols() code because this doesn't detect the
1652                          * case where the parent function's only reference to a
1653                          * subfunction is through a jump table.
1654                          */
1655                         if (!strstr(insn_func(insn)->name, ".cold") &&
1656                             strstr(insn_func(jump_dest)->name, ".cold")) {
1657                                 insn_func(insn)->cfunc = insn_func(jump_dest);
1658                                 insn_func(jump_dest)->pfunc = insn_func(insn);
1659                         }
1660                 }
1661
1662                 if (jump_is_sibling_call(file, insn, jump_dest)) {
1663                         /*
1664                          * Internal sibling call without reloc or with
1665                          * STT_SECTION reloc.
1666                          */
1667                         add_call_dest(file, insn, insn_func(jump_dest), true);
1668                         continue;
1669                 }
1670
1671                 insn->jump_dest = jump_dest;
1672         }
1673
1674         return 0;
1675 }
1676
1677 static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
1678 {
1679         struct symbol *call_dest;
1680
1681         call_dest = find_func_by_offset(sec, offset);
1682         if (!call_dest)
1683                 call_dest = find_symbol_by_offset(sec, offset);
1684
1685         return call_dest;
1686 }
1687
1688 /*
1689  * Find the destination instructions for all calls.
1690  */
1691 static int add_call_destinations(struct objtool_file *file)
1692 {
1693         struct instruction *insn;
1694         unsigned long dest_off;
1695         struct symbol *dest;
1696         struct reloc *reloc;
1697
1698         for_each_insn(file, insn) {
1699                 if (insn->type != INSN_CALL)
1700                         continue;
1701
1702                 reloc = insn_reloc(file, insn);
1703                 if (!reloc) {
1704                         dest_off = arch_jump_destination(insn);
1705                         dest = find_call_destination(insn->sec, dest_off);
1706
1707                         add_call_dest(file, insn, dest, false);
1708
1709                         if (insn->ignore)
1710                                 continue;
1711
1712                         if (!insn_call_dest(insn)) {
1713                                 WARN_INSN(insn, "unannotated intra-function call");
1714                                 return -1;
1715                         }
1716
1717                         if (insn_func(insn) && insn_call_dest(insn)->type != STT_FUNC) {
1718                                 WARN_INSN(insn, "unsupported call to non-function");
1719                                 return -1;
1720                         }
1721
1722                 } else if (reloc->sym->type == STT_SECTION) {
1723                         dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1724                         dest = find_call_destination(reloc->sym->sec, dest_off);
1725                         if (!dest) {
1726                                 WARN_INSN(insn, "can't find call dest symbol at %s+0x%lx",
1727                                           reloc->sym->sec->name, dest_off);
1728                                 return -1;
1729                         }
1730
1731                         add_call_dest(file, insn, dest, false);
1732
1733                 } else if (reloc->sym->retpoline_thunk) {
1734                         add_retpoline_call(file, insn);
1735
1736                 } else
1737                         add_call_dest(file, insn, reloc->sym, false);
1738         }
1739
1740         return 0;
1741 }
1742
1743 /*
1744  * The .alternatives section requires some extra special care over and above
1745  * other special sections because alternatives are patched in place.
1746  */
1747 static int handle_group_alt(struct objtool_file *file,
1748                             struct special_alt *special_alt,
1749                             struct instruction *orig_insn,
1750                             struct instruction **new_insn)
1751 {
1752         struct instruction *last_new_insn = NULL, *insn, *nop = NULL;
1753         struct alt_group *orig_alt_group, *new_alt_group;
1754         unsigned long dest_off;
1755
1756         orig_alt_group = orig_insn->alt_group;
1757         if (!orig_alt_group) {
1758                 struct instruction *last_orig_insn = NULL;
1759
1760                 orig_alt_group = malloc(sizeof(*orig_alt_group));
1761                 if (!orig_alt_group) {
1762                         WARN("malloc failed");
1763                         return -1;
1764                 }
1765                 orig_alt_group->cfi = calloc(special_alt->orig_len,
1766                                              sizeof(struct cfi_state *));
1767                 if (!orig_alt_group->cfi) {
1768                         WARN("calloc failed");
1769                         return -1;
1770                 }
1771
1772                 insn = orig_insn;
1773                 sec_for_each_insn_from(file, insn) {
1774                         if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1775                                 break;
1776
1777                         insn->alt_group = orig_alt_group;
1778                         last_orig_insn = insn;
1779                 }
1780                 orig_alt_group->orig_group = NULL;
1781                 orig_alt_group->first_insn = orig_insn;
1782                 orig_alt_group->last_insn = last_orig_insn;
1783                 orig_alt_group->nop = NULL;
1784         } else {
1785                 if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len -
1786                     orig_alt_group->first_insn->offset != special_alt->orig_len) {
1787                         WARN_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d",
1788                                   orig_alt_group->last_insn->offset +
1789                                   orig_alt_group->last_insn->len -
1790                                   orig_alt_group->first_insn->offset,
1791                                   special_alt->orig_len);
1792                         return -1;
1793                 }
1794         }
1795
1796         new_alt_group = malloc(sizeof(*new_alt_group));
1797         if (!new_alt_group) {
1798                 WARN("malloc failed");
1799                 return -1;
1800         }
1801
1802         if (special_alt->new_len < special_alt->orig_len) {
1803                 /*
1804                  * Insert a fake nop at the end to make the replacement
1805                  * alt_group the same size as the original.  This is needed to
1806                  * allow propagate_alt_cfi() to do its magic.  When the last
1807                  * instruction affects the stack, the instruction after it (the
1808                  * nop) will propagate the new state to the shared CFI array.
1809                  */
1810                 nop = malloc(sizeof(*nop));
1811                 if (!nop) {
1812                         WARN("malloc failed");
1813                         return -1;
1814                 }
1815                 memset(nop, 0, sizeof(*nop));
1816
1817                 nop->sec = special_alt->new_sec;
1818                 nop->offset = special_alt->new_off + special_alt->new_len;
1819                 nop->len = special_alt->orig_len - special_alt->new_len;
1820                 nop->type = INSN_NOP;
1821                 nop->sym = orig_insn->sym;
1822                 nop->alt_group = new_alt_group;
1823                 nop->ignore = orig_insn->ignore_alts;
1824         }
1825
1826         if (!special_alt->new_len) {
1827                 *new_insn = nop;
1828                 goto end;
1829         }
1830
1831         insn = *new_insn;
1832         sec_for_each_insn_from(file, insn) {
1833                 struct reloc *alt_reloc;
1834
1835                 if (insn->offset >= special_alt->new_off + special_alt->new_len)
1836                         break;
1837
1838                 last_new_insn = insn;
1839
1840                 insn->ignore = orig_insn->ignore_alts;
1841                 insn->sym = orig_insn->sym;
1842                 insn->alt_group = new_alt_group;
1843
1844                 /*
1845                  * Since alternative replacement code is copy/pasted by the
1846                  * kernel after applying relocations, generally such code can't
1847                  * have relative-address relocation references to outside the
1848                  * .altinstr_replacement section, unless the arch's
1849                  * alternatives code can adjust the relative offsets
1850                  * accordingly.
1851                  */
1852                 alt_reloc = insn_reloc(file, insn);
1853                 if (alt_reloc && arch_pc_relative_reloc(alt_reloc) &&
1854                     !arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1855
1856                         WARN_INSN(insn, "unsupported relocation in alternatives section");
1857                         return -1;
1858                 }
1859
1860                 if (!is_static_jump(insn))
1861                         continue;
1862
1863                 if (!insn->immediate)
1864                         continue;
1865
1866                 dest_off = arch_jump_destination(insn);
1867                 if (dest_off == special_alt->new_off + special_alt->new_len) {
1868                         insn->jump_dest = next_insn_same_sec(file, orig_alt_group->last_insn);
1869                         if (!insn->jump_dest) {
1870                                 WARN_INSN(insn, "can't find alternative jump destination");
1871                                 return -1;
1872                         }
1873                 }
1874         }
1875
1876         if (!last_new_insn) {
1877                 WARN_FUNC("can't find last new alternative instruction",
1878                           special_alt->new_sec, special_alt->new_off);
1879                 return -1;
1880         }
1881
1882 end:
1883         new_alt_group->orig_group = orig_alt_group;
1884         new_alt_group->first_insn = *new_insn;
1885         new_alt_group->last_insn = last_new_insn;
1886         new_alt_group->nop = nop;
1887         new_alt_group->cfi = orig_alt_group->cfi;
1888         return 0;
1889 }
1890
1891 /*
1892  * A jump table entry can either convert a nop to a jump or a jump to a nop.
1893  * If the original instruction is a jump, make the alt entry an effective nop
1894  * by just skipping the original instruction.
1895  */
1896 static int handle_jump_alt(struct objtool_file *file,
1897                            struct special_alt *special_alt,
1898                            struct instruction *orig_insn,
1899                            struct instruction **new_insn)
1900 {
1901         if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1902             orig_insn->type != INSN_NOP) {
1903
1904                 WARN_INSN(orig_insn, "unsupported instruction at jump label");
1905                 return -1;
1906         }
1907
1908         if (opts.hack_jump_label && special_alt->key_addend & 2) {
1909                 struct reloc *reloc = insn_reloc(file, orig_insn);
1910
1911                 if (reloc)
1912                         set_reloc_type(file->elf, reloc, R_NONE);
1913                 elf_write_insn(file->elf, orig_insn->sec,
1914                                orig_insn->offset, orig_insn->len,
1915                                arch_nop_insn(orig_insn->len));
1916                 orig_insn->type = INSN_NOP;
1917         }
1918
1919         if (orig_insn->type == INSN_NOP) {
1920                 if (orig_insn->len == 2)
1921                         file->jl_nop_short++;
1922                 else
1923                         file->jl_nop_long++;
1924
1925                 return 0;
1926         }
1927
1928         if (orig_insn->len == 2)
1929                 file->jl_short++;
1930         else
1931                 file->jl_long++;
1932
1933         *new_insn = next_insn_same_sec(file, orig_insn);
1934         return 0;
1935 }
1936
1937 /*
1938  * Read all the special sections which have alternate instructions which can be
1939  * patched in or redirected to at runtime.  Each instruction having alternate
1940  * instruction(s) has them added to its insn->alts list, which will be
1941  * traversed in validate_branch().
1942  */
1943 static int add_special_section_alts(struct objtool_file *file)
1944 {
1945         struct list_head special_alts;
1946         struct instruction *orig_insn, *new_insn;
1947         struct special_alt *special_alt, *tmp;
1948         struct alternative *alt;
1949         int ret;
1950
1951         ret = special_get_alts(file->elf, &special_alts);
1952         if (ret)
1953                 return ret;
1954
1955         list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
1956
1957                 orig_insn = find_insn(file, special_alt->orig_sec,
1958                                       special_alt->orig_off);
1959                 if (!orig_insn) {
1960                         WARN_FUNC("special: can't find orig instruction",
1961                                   special_alt->orig_sec, special_alt->orig_off);
1962                         ret = -1;
1963                         goto out;
1964                 }
1965
1966                 new_insn = NULL;
1967                 if (!special_alt->group || special_alt->new_len) {
1968                         new_insn = find_insn(file, special_alt->new_sec,
1969                                              special_alt->new_off);
1970                         if (!new_insn) {
1971                                 WARN_FUNC("special: can't find new instruction",
1972                                           special_alt->new_sec,
1973                                           special_alt->new_off);
1974                                 ret = -1;
1975                                 goto out;
1976                         }
1977                 }
1978
1979                 if (special_alt->group) {
1980                         if (!special_alt->orig_len) {
1981                                 WARN_INSN(orig_insn, "empty alternative entry");
1982                                 continue;
1983                         }
1984
1985                         ret = handle_group_alt(file, special_alt, orig_insn,
1986                                                &new_insn);
1987                         if (ret)
1988                                 goto out;
1989                 } else if (special_alt->jump_or_nop) {
1990                         ret = handle_jump_alt(file, special_alt, orig_insn,
1991                                               &new_insn);
1992                         if (ret)
1993                                 goto out;
1994                 }
1995
1996                 alt = malloc(sizeof(*alt));
1997                 if (!alt) {
1998                         WARN("malloc failed");
1999                         ret = -1;
2000                         goto out;
2001                 }
2002
2003                 alt->insn = new_insn;
2004                 alt->skip_orig = special_alt->skip_orig;
2005                 orig_insn->ignore_alts |= special_alt->skip_alt;
2006                 alt->next = orig_insn->alts;
2007                 orig_insn->alts = alt;
2008
2009                 list_del(&special_alt->list);
2010                 free(special_alt);
2011         }
2012
2013         if (opts.stats) {
2014                 printf("jl\\\tNOP\tJMP\n");
2015                 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
2016                 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
2017         }
2018
2019 out:
2020         return ret;
2021 }
2022
2023 static int add_jump_table(struct objtool_file *file, struct instruction *insn,
2024                           struct reloc *next_table)
2025 {
2026         struct symbol *pfunc = insn_func(insn)->pfunc;
2027         struct reloc *table = insn_jump_table(insn);
2028         struct instruction *dest_insn;
2029         unsigned int prev_offset = 0;
2030         struct reloc *reloc = table;
2031         struct alternative *alt;
2032
2033         /*
2034          * Each @reloc is a switch table relocation which points to the target
2035          * instruction.
2036          */
2037         for_each_reloc_from(table->sec, reloc) {
2038
2039                 /* Check for the end of the table: */
2040                 if (reloc != table && reloc == next_table)
2041                         break;
2042
2043                 /* Make sure the table entries are consecutive: */
2044                 if (prev_offset && reloc_offset(reloc) != prev_offset + 8)
2045                         break;
2046
2047                 /* Detect function pointers from contiguous objects: */
2048                 if (reloc->sym->sec == pfunc->sec &&
2049                     reloc_addend(reloc) == pfunc->offset)
2050                         break;
2051
2052                 dest_insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2053                 if (!dest_insn)
2054                         break;
2055
2056                 /* Make sure the destination is in the same function: */
2057                 if (!insn_func(dest_insn) || insn_func(dest_insn)->pfunc != pfunc)
2058                         break;
2059
2060                 alt = malloc(sizeof(*alt));
2061                 if (!alt) {
2062                         WARN("malloc failed");
2063                         return -1;
2064                 }
2065
2066                 alt->insn = dest_insn;
2067                 alt->next = insn->alts;
2068                 insn->alts = alt;
2069                 prev_offset = reloc_offset(reloc);
2070         }
2071
2072         if (!prev_offset) {
2073                 WARN_INSN(insn, "can't find switch jump table");
2074                 return -1;
2075         }
2076
2077         return 0;
2078 }
2079
2080 /*
2081  * find_jump_table() - Given a dynamic jump, find the switch jump table
2082  * associated with it.
2083  */
2084 static struct reloc *find_jump_table(struct objtool_file *file,
2085                                       struct symbol *func,
2086                                       struct instruction *insn)
2087 {
2088         struct reloc *table_reloc;
2089         struct instruction *dest_insn, *orig_insn = insn;
2090
2091         /*
2092          * Backward search using the @first_jump_src links, these help avoid
2093          * much of the 'in between' code. Which avoids us getting confused by
2094          * it.
2095          */
2096         for (;
2097              insn && insn_func(insn) && insn_func(insn)->pfunc == func;
2098              insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
2099
2100                 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
2101                         break;
2102
2103                 /* allow small jumps within the range */
2104                 if (insn->type == INSN_JUMP_UNCONDITIONAL &&
2105                     insn->jump_dest &&
2106                     (insn->jump_dest->offset <= insn->offset ||
2107                      insn->jump_dest->offset > orig_insn->offset))
2108                     break;
2109
2110                 table_reloc = arch_find_switch_table(file, insn);
2111                 if (!table_reloc)
2112                         continue;
2113                 dest_insn = find_insn(file, table_reloc->sym->sec, reloc_addend(table_reloc));
2114                 if (!dest_insn || !insn_func(dest_insn) || insn_func(dest_insn)->pfunc != func)
2115                         continue;
2116
2117                 return table_reloc;
2118         }
2119
2120         return NULL;
2121 }
2122
2123 /*
2124  * First pass: Mark the head of each jump table so that in the next pass,
2125  * we know when a given jump table ends and the next one starts.
2126  */
2127 static void mark_func_jump_tables(struct objtool_file *file,
2128                                     struct symbol *func)
2129 {
2130         struct instruction *insn, *last = NULL;
2131         struct reloc *reloc;
2132
2133         func_for_each_insn(file, func, insn) {
2134                 if (!last)
2135                         last = insn;
2136
2137                 /*
2138                  * Store back-pointers for unconditional forward jumps such
2139                  * that find_jump_table() can back-track using those and
2140                  * avoid some potentially confusing code.
2141                  */
2142                 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
2143                     insn->offset > last->offset &&
2144                     insn->jump_dest->offset > insn->offset &&
2145                     !insn->jump_dest->first_jump_src) {
2146
2147                         insn->jump_dest->first_jump_src = insn;
2148                         last = insn->jump_dest;
2149                 }
2150
2151                 if (insn->type != INSN_JUMP_DYNAMIC)
2152                         continue;
2153
2154                 reloc = find_jump_table(file, func, insn);
2155                 if (reloc)
2156                         insn->_jump_table = reloc;
2157         }
2158 }
2159
2160 static int add_func_jump_tables(struct objtool_file *file,
2161                                   struct symbol *func)
2162 {
2163         struct instruction *insn, *insn_t1 = NULL, *insn_t2;
2164         int ret = 0;
2165
2166         func_for_each_insn(file, func, insn) {
2167                 if (!insn_jump_table(insn))
2168                         continue;
2169
2170                 if (!insn_t1) {
2171                         insn_t1 = insn;
2172                         continue;
2173                 }
2174
2175                 insn_t2 = insn;
2176
2177                 ret = add_jump_table(file, insn_t1, insn_jump_table(insn_t2));
2178                 if (ret)
2179                         return ret;
2180
2181                 insn_t1 = insn_t2;
2182         }
2183
2184         if (insn_t1)
2185                 ret = add_jump_table(file, insn_t1, NULL);
2186
2187         return ret;
2188 }
2189
2190 /*
2191  * For some switch statements, gcc generates a jump table in the .rodata
2192  * section which contains a list of addresses within the function to jump to.
2193  * This finds these jump tables and adds them to the insn->alts lists.
2194  */
2195 static int add_jump_table_alts(struct objtool_file *file)
2196 {
2197         struct symbol *func;
2198         int ret;
2199
2200         if (!file->rodata)
2201                 return 0;
2202
2203         for_each_sym(file, func) {
2204                 if (func->type != STT_FUNC)
2205                         continue;
2206
2207                 mark_func_jump_tables(file, func);
2208                 ret = add_func_jump_tables(file, func);
2209                 if (ret)
2210                         return ret;
2211         }
2212
2213         return 0;
2214 }
2215
2216 static void set_func_state(struct cfi_state *state)
2217 {
2218         state->cfa = initial_func_cfi.cfa;
2219         memcpy(&state->regs, &initial_func_cfi.regs,
2220                CFI_NUM_REGS * sizeof(struct cfi_reg));
2221         state->stack_size = initial_func_cfi.cfa.offset;
2222         state->type = UNWIND_HINT_TYPE_CALL;
2223 }
2224
2225 static int read_unwind_hints(struct objtool_file *file)
2226 {
2227         struct cfi_state cfi = init_cfi;
2228         struct section *sec;
2229         struct unwind_hint *hint;
2230         struct instruction *insn;
2231         struct reloc *reloc;
2232         unsigned long offset;
2233         int i;
2234
2235         sec = find_section_by_name(file->elf, ".discard.unwind_hints");
2236         if (!sec)
2237                 return 0;
2238
2239         if (!sec->rsec) {
2240                 WARN("missing .rela.discard.unwind_hints section");
2241                 return -1;
2242         }
2243
2244         if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
2245                 WARN("struct unwind_hint size mismatch");
2246                 return -1;
2247         }
2248
2249         file->hints = true;
2250
2251         for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
2252                 hint = (struct unwind_hint *)sec->data->d_buf + i;
2253
2254                 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
2255                 if (!reloc) {
2256                         WARN("can't find reloc for unwind_hints[%d]", i);
2257                         return -1;
2258                 }
2259
2260                 if (reloc->sym->type == STT_SECTION) {
2261                         offset = reloc_addend(reloc);
2262                 } else if (reloc->sym->local_label) {
2263                         offset = reloc->sym->offset;
2264                 } else {
2265                         WARN("unexpected relocation symbol type in %s", sec->rsec->name);
2266                         return -1;
2267                 }
2268
2269                 insn = find_insn(file, reloc->sym->sec, offset);
2270                 if (!insn) {
2271                         WARN("can't find insn for unwind_hints[%d]", i);
2272                         return -1;
2273                 }
2274
2275                 insn->hint = true;
2276
2277                 if (hint->type == UNWIND_HINT_TYPE_UNDEFINED) {
2278                         insn->cfi = &force_undefined_cfi;
2279                         continue;
2280                 }
2281
2282                 if (hint->type == UNWIND_HINT_TYPE_SAVE) {
2283                         insn->hint = false;
2284                         insn->save = true;
2285                         continue;
2286                 }
2287
2288                 if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
2289                         insn->restore = true;
2290                         continue;
2291                 }
2292
2293                 if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
2294                         struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
2295
2296                         if (sym && sym->bind == STB_GLOBAL) {
2297                                 if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) {
2298                                         WARN_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR");
2299                                 }
2300                         }
2301                 }
2302
2303                 if (hint->type == UNWIND_HINT_TYPE_FUNC) {
2304                         insn->cfi = &func_cfi;
2305                         continue;
2306                 }
2307
2308                 if (insn->cfi)
2309                         cfi = *(insn->cfi);
2310
2311                 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
2312                         WARN_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg);
2313                         return -1;
2314                 }
2315
2316                 cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset);
2317                 cfi.type = hint->type;
2318                 cfi.signal = hint->signal;
2319
2320                 insn->cfi = cfi_hash_find_or_add(&cfi);
2321         }
2322
2323         return 0;
2324 }
2325
2326 static int read_noendbr_hints(struct objtool_file *file)
2327 {
2328         struct instruction *insn;
2329         struct section *rsec;
2330         struct reloc *reloc;
2331
2332         rsec = find_section_by_name(file->elf, ".rela.discard.noendbr");
2333         if (!rsec)
2334                 return 0;
2335
2336         for_each_reloc(rsec, reloc) {
2337                 insn = find_insn(file, reloc->sym->sec,
2338                                  reloc->sym->offset + reloc_addend(reloc));
2339                 if (!insn) {
2340                         WARN("bad .discard.noendbr entry");
2341                         return -1;
2342                 }
2343
2344                 insn->noendbr = 1;
2345         }
2346
2347         return 0;
2348 }
2349
2350 static int read_retpoline_hints(struct objtool_file *file)
2351 {
2352         struct section *rsec;
2353         struct instruction *insn;
2354         struct reloc *reloc;
2355
2356         rsec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
2357         if (!rsec)
2358                 return 0;
2359
2360         for_each_reloc(rsec, reloc) {
2361                 if (reloc->sym->type != STT_SECTION) {
2362                         WARN("unexpected relocation symbol type in %s", rsec->name);
2363                         return -1;
2364                 }
2365
2366                 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2367                 if (!insn) {
2368                         WARN("bad .discard.retpoline_safe entry");
2369                         return -1;
2370                 }
2371
2372                 if (insn->type != INSN_JUMP_DYNAMIC &&
2373                     insn->type != INSN_CALL_DYNAMIC &&
2374                     insn->type != INSN_RETURN &&
2375                     insn->type != INSN_NOP) {
2376                         WARN_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop");
2377                         return -1;
2378                 }
2379
2380                 insn->retpoline_safe = true;
2381         }
2382
2383         return 0;
2384 }
2385
2386 static int read_instr_hints(struct objtool_file *file)
2387 {
2388         struct section *rsec;
2389         struct instruction *insn;
2390         struct reloc *reloc;
2391
2392         rsec = find_section_by_name(file->elf, ".rela.discard.instr_end");
2393         if (!rsec)
2394                 return 0;
2395
2396         for_each_reloc(rsec, reloc) {
2397                 if (reloc->sym->type != STT_SECTION) {
2398                         WARN("unexpected relocation symbol type in %s", rsec->name);
2399                         return -1;
2400                 }
2401
2402                 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2403                 if (!insn) {
2404                         WARN("bad .discard.instr_end entry");
2405                         return -1;
2406                 }
2407
2408                 insn->instr--;
2409         }
2410
2411         rsec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
2412         if (!rsec)
2413                 return 0;
2414
2415         for_each_reloc(rsec, reloc) {
2416                 if (reloc->sym->type != STT_SECTION) {
2417                         WARN("unexpected relocation symbol type in %s", rsec->name);
2418                         return -1;
2419                 }
2420
2421                 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2422                 if (!insn) {
2423                         WARN("bad .discard.instr_begin entry");
2424                         return -1;
2425                 }
2426
2427                 insn->instr++;
2428         }
2429
2430         return 0;
2431 }
2432
2433 static int read_validate_unret_hints(struct objtool_file *file)
2434 {
2435         struct section *rsec;
2436         struct instruction *insn;
2437         struct reloc *reloc;
2438
2439         rsec = find_section_by_name(file->elf, ".rela.discard.validate_unret");
2440         if (!rsec)
2441                 return 0;
2442
2443         for_each_reloc(rsec, reloc) {
2444                 if (reloc->sym->type != STT_SECTION) {
2445                         WARN("unexpected relocation symbol type in %s", rsec->name);
2446                         return -1;
2447                 }
2448
2449                 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2450                 if (!insn) {
2451                         WARN("bad .discard.instr_end entry");
2452                         return -1;
2453                 }
2454                 insn->unret = 1;
2455         }
2456
2457         return 0;
2458 }
2459
2460
2461 static int read_intra_function_calls(struct objtool_file *file)
2462 {
2463         struct instruction *insn;
2464         struct section *rsec;
2465         struct reloc *reloc;
2466
2467         rsec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
2468         if (!rsec)
2469                 return 0;
2470
2471         for_each_reloc(rsec, reloc) {
2472                 unsigned long dest_off;
2473
2474                 if (reloc->sym->type != STT_SECTION) {
2475                         WARN("unexpected relocation symbol type in %s",
2476                              rsec->name);
2477                         return -1;
2478                 }
2479
2480                 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2481                 if (!insn) {
2482                         WARN("bad .discard.intra_function_call entry");
2483                         return -1;
2484                 }
2485
2486                 if (insn->type != INSN_CALL) {
2487                         WARN_INSN(insn, "intra_function_call not a direct call");
2488                         return -1;
2489                 }
2490
2491                 /*
2492                  * Treat intra-function CALLs as JMPs, but with a stack_op.
2493                  * See add_call_destinations(), which strips stack_ops from
2494                  * normal CALLs.
2495                  */
2496                 insn->type = INSN_JUMP_UNCONDITIONAL;
2497
2498                 dest_off = arch_jump_destination(insn);
2499                 insn->jump_dest = find_insn(file, insn->sec, dest_off);
2500                 if (!insn->jump_dest) {
2501                         WARN_INSN(insn, "can't find call dest at %s+0x%lx",
2502                                   insn->sec->name, dest_off);
2503                         return -1;
2504                 }
2505         }
2506
2507         return 0;
2508 }
2509
2510 /*
2511  * Return true if name matches an instrumentation function, where calls to that
2512  * function from noinstr code can safely be removed, but compilers won't do so.
2513  */
2514 static bool is_profiling_func(const char *name)
2515 {
2516         /*
2517          * Many compilers cannot disable KCOV with a function attribute.
2518          */
2519         if (!strncmp(name, "__sanitizer_cov_", 16))
2520                 return true;
2521
2522         /*
2523          * Some compilers currently do not remove __tsan_func_entry/exit nor
2524          * __tsan_atomic_signal_fence (used for barrier instrumentation) with
2525          * the __no_sanitize_thread attribute, remove them. Once the kernel's
2526          * minimum Clang version is 14.0, this can be removed.
2527          */
2528         if (!strncmp(name, "__tsan_func_", 12) ||
2529             !strcmp(name, "__tsan_atomic_signal_fence"))
2530                 return true;
2531
2532         return false;
2533 }
2534
2535 static int classify_symbols(struct objtool_file *file)
2536 {
2537         struct symbol *func;
2538
2539         for_each_sym(file, func) {
2540                 if (func->type == STT_NOTYPE && strstarts(func->name, ".L"))
2541                         func->local_label = true;
2542
2543                 if (func->bind != STB_GLOBAL)
2544                         continue;
2545
2546                 if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
2547                              strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
2548                         func->static_call_tramp = true;
2549
2550                 if (arch_is_retpoline(func))
2551                         func->retpoline_thunk = true;
2552
2553                 if (arch_is_rethunk(func))
2554                         func->return_thunk = true;
2555
2556                 if (arch_is_embedded_insn(func))
2557                         func->embedded_insn = true;
2558
2559                 if (arch_ftrace_match(func->name))
2560                         func->fentry = true;
2561
2562                 if (is_profiling_func(func->name))
2563                         func->profiling_func = true;
2564         }
2565
2566         return 0;
2567 }
2568
2569 static void mark_rodata(struct objtool_file *file)
2570 {
2571         struct section *sec;
2572         bool found = false;
2573
2574         /*
2575          * Search for the following rodata sections, each of which can
2576          * potentially contain jump tables:
2577          *
2578          * - .rodata: can contain GCC switch tables
2579          * - .rodata.<func>: same, if -fdata-sections is being used
2580          * - .rodata..c_jump_table: contains C annotated jump tables
2581          *
2582          * .rodata.str1.* sections are ignored; they don't contain jump tables.
2583          */
2584         for_each_sec(file, sec) {
2585                 if (!strncmp(sec->name, ".rodata", 7) &&
2586                     !strstr(sec->name, ".str1.")) {
2587                         sec->rodata = true;
2588                         found = true;
2589                 }
2590         }
2591
2592         file->rodata = found;
2593 }
2594
2595 static int decode_sections(struct objtool_file *file)
2596 {
2597         int ret;
2598
2599         mark_rodata(file);
2600
2601         ret = init_pv_ops(file);
2602         if (ret)
2603                 return ret;
2604
2605         /*
2606          * Must be before add_{jump_call}_destination.
2607          */
2608         ret = classify_symbols(file);
2609         if (ret)
2610                 return ret;
2611
2612         ret = decode_instructions(file);
2613         if (ret)
2614                 return ret;
2615
2616         add_ignores(file);
2617         add_uaccess_safe(file);
2618
2619         ret = add_ignore_alternatives(file);
2620         if (ret)
2621                 return ret;
2622
2623         /*
2624          * Must be before read_unwind_hints() since that needs insn->noendbr.
2625          */
2626         ret = read_noendbr_hints(file);
2627         if (ret)
2628                 return ret;
2629
2630         /*
2631          * Must be before add_jump_destinations(), which depends on 'func'
2632          * being set for alternatives, to enable proper sibling call detection.
2633          */
2634         if (opts.stackval || opts.orc || opts.uaccess || opts.noinstr) {
2635                 ret = add_special_section_alts(file);
2636                 if (ret)
2637                         return ret;
2638         }
2639
2640         ret = add_jump_destinations(file);
2641         if (ret)
2642                 return ret;
2643
2644         /*
2645          * Must be before add_call_destination(); it changes INSN_CALL to
2646          * INSN_JUMP.
2647          */
2648         ret = read_intra_function_calls(file);
2649         if (ret)
2650                 return ret;
2651
2652         ret = add_call_destinations(file);
2653         if (ret)
2654                 return ret;
2655
2656         /*
2657          * Must be after add_call_destinations() such that it can override
2658          * dead_end_function() marks.
2659          */
2660         ret = add_dead_ends(file);
2661         if (ret)
2662                 return ret;
2663
2664         ret = add_jump_table_alts(file);
2665         if (ret)
2666                 return ret;
2667
2668         ret = read_unwind_hints(file);
2669         if (ret)
2670                 return ret;
2671
2672         ret = read_retpoline_hints(file);
2673         if (ret)
2674                 return ret;
2675
2676         ret = read_instr_hints(file);
2677         if (ret)
2678                 return ret;
2679
2680         ret = read_validate_unret_hints(file);
2681         if (ret)
2682                 return ret;
2683
2684         return 0;
2685 }
2686
2687 static bool is_special_call(struct instruction *insn)
2688 {
2689         if (insn->type == INSN_CALL) {
2690                 struct symbol *dest = insn_call_dest(insn);
2691
2692                 if (!dest)
2693                         return false;
2694
2695                 if (dest->fentry || dest->embedded_insn)
2696                         return true;
2697         }
2698
2699         return false;
2700 }
2701
2702 static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
2703 {
2704         struct cfi_state *cfi = &state->cfi;
2705         int i;
2706
2707         if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
2708                 return true;
2709
2710         if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
2711                 return true;
2712
2713         if (cfi->stack_size != initial_func_cfi.cfa.offset)
2714                 return true;
2715
2716         for (i = 0; i < CFI_NUM_REGS; i++) {
2717                 if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
2718                     cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
2719                         return true;
2720         }
2721
2722         return false;
2723 }
2724
2725 static bool check_reg_frame_pos(const struct cfi_reg *reg,
2726                                 int expected_offset)
2727 {
2728         return reg->base == CFI_CFA &&
2729                reg->offset == expected_offset;
2730 }
2731
2732 static bool has_valid_stack_frame(struct insn_state *state)
2733 {
2734         struct cfi_state *cfi = &state->cfi;
2735
2736         if (cfi->cfa.base == CFI_BP &&
2737             check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2738             check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
2739                 return true;
2740
2741         if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
2742                 return true;
2743
2744         return false;
2745 }
2746
2747 static int update_cfi_state_regs(struct instruction *insn,
2748                                   struct cfi_state *cfi,
2749                                   struct stack_op *op)
2750 {
2751         struct cfi_reg *cfa = &cfi->cfa;
2752
2753         if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
2754                 return 0;
2755
2756         /* push */
2757         if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
2758                 cfa->offset += 8;
2759
2760         /* pop */
2761         if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
2762                 cfa->offset -= 8;
2763
2764         /* add immediate to sp */
2765         if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2766             op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2767                 cfa->offset -= op->src.offset;
2768
2769         return 0;
2770 }
2771
2772 static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
2773 {
2774         if (arch_callee_saved_reg(reg) &&
2775             cfi->regs[reg].base == CFI_UNDEFINED) {
2776                 cfi->regs[reg].base = base;
2777                 cfi->regs[reg].offset = offset;
2778         }
2779 }
2780
2781 static void restore_reg(struct cfi_state *cfi, unsigned char reg)
2782 {
2783         cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2784         cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
2785 }
2786
2787 /*
2788  * A note about DRAP stack alignment:
2789  *
2790  * GCC has the concept of a DRAP register, which is used to help keep track of
2791  * the stack pointer when aligning the stack.  r10 or r13 is used as the DRAP
2792  * register.  The typical DRAP pattern is:
2793  *
2794  *   4c 8d 54 24 08             lea    0x8(%rsp),%r10
2795  *   48 83 e4 c0                and    $0xffffffffffffffc0,%rsp
2796  *   41 ff 72 f8                pushq  -0x8(%r10)
2797  *   55                         push   %rbp
2798  *   48 89 e5                   mov    %rsp,%rbp
2799  *                              (more pushes)
2800  *   41 52                      push   %r10
2801  *                              ...
2802  *   41 5a                      pop    %r10
2803  *                              (more pops)
2804  *   5d                         pop    %rbp
2805  *   49 8d 62 f8                lea    -0x8(%r10),%rsp
2806  *   c3                         retq
2807  *
2808  * There are some variations in the epilogues, like:
2809  *
2810  *   5b                         pop    %rbx
2811  *   41 5a                      pop    %r10
2812  *   41 5c                      pop    %r12
2813  *   41 5d                      pop    %r13
2814  *   41 5e                      pop    %r14
2815  *   c9                         leaveq
2816  *   49 8d 62 f8                lea    -0x8(%r10),%rsp
2817  *   c3                         retq
2818  *
2819  * and:
2820  *
2821  *   4c 8b 55 e8                mov    -0x18(%rbp),%r10
2822  *   48 8b 5d e0                mov    -0x20(%rbp),%rbx
2823  *   4c 8b 65 f0                mov    -0x10(%rbp),%r12
2824  *   4c 8b 6d f8                mov    -0x8(%rbp),%r13
2825  *   c9                         leaveq
2826  *   49 8d 62 f8                lea    -0x8(%r10),%rsp
2827  *   c3                         retq
2828  *
2829  * Sometimes r13 is used as the DRAP register, in which case it's saved and
2830  * restored beforehand:
2831  *
2832  *   41 55                      push   %r13
2833  *   4c 8d 6c 24 10             lea    0x10(%rsp),%r13
2834  *   48 83 e4 f0                and    $0xfffffffffffffff0,%rsp
2835  *                              ...
2836  *   49 8d 65 f0                lea    -0x10(%r13),%rsp
2837  *   41 5d                      pop    %r13
2838  *   c3                         retq
2839  */
2840 static int update_cfi_state(struct instruction *insn,
2841                             struct instruction *next_insn,
2842                             struct cfi_state *cfi, struct stack_op *op)
2843 {
2844         struct cfi_reg *cfa = &cfi->cfa;
2845         struct cfi_reg *regs = cfi->regs;
2846
2847         /* ignore UNWIND_HINT_UNDEFINED regions */
2848         if (cfi->force_undefined)
2849                 return 0;
2850
2851         /* stack operations don't make sense with an undefined CFA */
2852         if (cfa->base == CFI_UNDEFINED) {
2853                 if (insn_func(insn)) {
2854                         WARN_INSN(insn, "undefined stack state");
2855                         return -1;
2856                 }
2857                 return 0;
2858         }
2859
2860         if (cfi->type == UNWIND_HINT_TYPE_REGS ||
2861             cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
2862                 return update_cfi_state_regs(insn, cfi, op);
2863
2864         switch (op->dest.type) {
2865
2866         case OP_DEST_REG:
2867                 switch (op->src.type) {
2868
2869                 case OP_SRC_REG:
2870                         if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2871                             cfa->base == CFI_SP &&
2872                             check_reg_frame_pos(&regs[CFI_BP], -cfa->offset)) {
2873
2874                                 /* mov %rsp, %rbp */
2875                                 cfa->base = op->dest.reg;
2876                                 cfi->bp_scratch = false;
2877                         }
2878
2879                         else if (op->src.reg == CFI_SP &&
2880                                  op->dest.reg == CFI_BP && cfi->drap) {
2881
2882                                 /* drap: mov %rsp, %rbp */
2883                                 regs[CFI_BP].base = CFI_BP;
2884                                 regs[CFI_BP].offset = -cfi->stack_size;
2885                                 cfi->bp_scratch = false;
2886                         }
2887
2888                         else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2889
2890                                 /*
2891                                  * mov %rsp, %reg
2892                                  *
2893                                  * This is needed for the rare case where GCC
2894                                  * does:
2895                                  *
2896                                  *   mov    %rsp, %rax
2897                                  *   ...
2898                                  *   mov    %rax, %rsp
2899                                  */
2900                                 cfi->vals[op->dest.reg].base = CFI_CFA;
2901                                 cfi->vals[op->dest.reg].offset = -cfi->stack_size;
2902                         }
2903
2904                         else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
2905                                  (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
2906
2907                                 /*
2908                                  * mov %rbp, %rsp
2909                                  *
2910                                  * Restore the original stack pointer (Clang).
2911                                  */
2912                                 cfi->stack_size = -cfi->regs[CFI_BP].offset;
2913                         }
2914
2915                         else if (op->dest.reg == cfa->base) {
2916
2917                                 /* mov %reg, %rsp */
2918                                 if (cfa->base == CFI_SP &&
2919                                     cfi->vals[op->src.reg].base == CFI_CFA) {
2920
2921                                         /*
2922                                          * This is needed for the rare case
2923                                          * where GCC does something dumb like:
2924                                          *
2925                                          *   lea    0x8(%rsp), %rcx
2926                                          *   ...
2927                                          *   mov    %rcx, %rsp
2928                                          */
2929                                         cfa->offset = -cfi->vals[op->src.reg].offset;
2930                                         cfi->stack_size = cfa->offset;
2931
2932                                 } else if (cfa->base == CFI_SP &&
2933                                            cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2934                                            cfi->vals[op->src.reg].offset == cfa->offset) {
2935
2936                                         /*
2937                                          * Stack swizzle:
2938                                          *
2939                                          * 1: mov %rsp, (%[tos])
2940                                          * 2: mov %[tos], %rsp
2941                                          *    ...
2942                                          * 3: pop %rsp
2943                                          *
2944                                          * Where:
2945                                          *
2946                                          * 1 - places a pointer to the previous
2947                                          *     stack at the Top-of-Stack of the
2948                                          *     new stack.
2949                                          *
2950                                          * 2 - switches to the new stack.
2951                                          *
2952                                          * 3 - pops the Top-of-Stack to restore
2953                                          *     the original stack.
2954                                          *
2955                                          * Note: we set base to SP_INDIRECT
2956                                          * here and preserve offset. Therefore
2957                                          * when the unwinder reaches ToS it
2958                                          * will dereference SP and then add the
2959                                          * offset to find the next frame, IOW:
2960                                          * (%rsp) + offset.
2961                                          */
2962                                         cfa->base = CFI_SP_INDIRECT;
2963
2964                                 } else {
2965                                         cfa->base = CFI_UNDEFINED;
2966                                         cfa->offset = 0;
2967                                 }
2968                         }
2969
2970                         else if (op->dest.reg == CFI_SP &&
2971                                  cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2972                                  cfi->vals[op->src.reg].offset == cfa->offset) {
2973
2974                                 /*
2975                                  * The same stack swizzle case 2) as above. But
2976                                  * because we can't change cfa->base, case 3)
2977                                  * will become a regular POP. Pretend we're a
2978                                  * PUSH so things don't go unbalanced.
2979                                  */
2980                                 cfi->stack_size += 8;
2981                         }
2982
2983
2984                         break;
2985
2986                 case OP_SRC_ADD:
2987                         if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
2988
2989                                 /* add imm, %rsp */
2990                                 cfi->stack_size -= op->src.offset;
2991                                 if (cfa->base == CFI_SP)
2992                                         cfa->offset -= op->src.offset;
2993                                 break;
2994                         }
2995
2996                         if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
2997
2998                                 /* lea disp(%rbp), %rsp */
2999                                 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
3000                                 break;
3001                         }
3002
3003                         if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
3004
3005                                 /* drap: lea disp(%rsp), %drap */
3006                                 cfi->drap_reg = op->dest.reg;
3007
3008                                 /*
3009                                  * lea disp(%rsp), %reg
3010                                  *
3011                                  * This is needed for the rare case where GCC
3012                                  * does something dumb like:
3013                                  *
3014                                  *   lea    0x8(%rsp), %rcx
3015                                  *   ...
3016                                  *   mov    %rcx, %rsp
3017                                  */
3018                                 cfi->vals[op->dest.reg].base = CFI_CFA;
3019                                 cfi->vals[op->dest.reg].offset = \
3020                                         -cfi->stack_size + op->src.offset;
3021
3022                                 break;
3023                         }
3024
3025                         if (cfi->drap && op->dest.reg == CFI_SP &&
3026                             op->src.reg == cfi->drap_reg) {
3027
3028                                  /* drap: lea disp(%drap), %rsp */
3029                                 cfa->base = CFI_SP;
3030                                 cfa->offset = cfi->stack_size = -op->src.offset;
3031                                 cfi->drap_reg = CFI_UNDEFINED;
3032                                 cfi->drap = false;
3033                                 break;
3034                         }
3035
3036                         if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
3037                                 WARN_INSN(insn, "unsupported stack register modification");
3038                                 return -1;
3039                         }
3040
3041                         break;
3042
3043                 case OP_SRC_AND:
3044                         if (op->dest.reg != CFI_SP ||
3045                             (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
3046                             (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
3047                                 WARN_INSN(insn, "unsupported stack pointer realignment");
3048                                 return -1;
3049                         }
3050
3051                         if (cfi->drap_reg != CFI_UNDEFINED) {
3052                                 /* drap: and imm, %rsp */
3053                                 cfa->base = cfi->drap_reg;
3054                                 cfa->offset = cfi->stack_size = 0;
3055                                 cfi->drap = true;
3056                         }
3057
3058                         /*
3059                          * Older versions of GCC (4.8ish) realign the stack
3060                          * without DRAP, with a frame pointer.
3061                          */
3062
3063                         break;
3064
3065                 case OP_SRC_POP:
3066                 case OP_SRC_POPF:
3067                         if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
3068
3069                                 /* pop %rsp; # restore from a stack swizzle */
3070                                 cfa->base = CFI_SP;
3071                                 break;
3072                         }
3073
3074                         if (!cfi->drap && op->dest.reg == cfa->base) {
3075
3076                                 /* pop %rbp */
3077                                 cfa->base = CFI_SP;
3078                         }
3079
3080                         if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
3081                             op->dest.reg == cfi->drap_reg &&
3082                             cfi->drap_offset == -cfi->stack_size) {
3083
3084                                 /* drap: pop %drap */
3085                                 cfa->base = cfi->drap_reg;
3086                                 cfa->offset = 0;
3087                                 cfi->drap_offset = -1;
3088
3089                         } else if (cfi->stack_size == -regs[op->dest.reg].offset) {
3090
3091                                 /* pop %reg */
3092                                 restore_reg(cfi, op->dest.reg);
3093                         }
3094
3095                         cfi->stack_size -= 8;
3096                         if (cfa->base == CFI_SP)
3097                                 cfa->offset -= 8;
3098
3099                         break;
3100
3101                 case OP_SRC_REG_INDIRECT:
3102                         if (!cfi->drap && op->dest.reg == cfa->base &&
3103                             op->dest.reg == CFI_BP) {
3104
3105                                 /* mov disp(%rsp), %rbp */
3106                                 cfa->base = CFI_SP;
3107                                 cfa->offset = cfi->stack_size;
3108                         }
3109
3110                         if (cfi->drap && op->src.reg == CFI_BP &&
3111                             op->src.offset == cfi->drap_offset) {
3112
3113                                 /* drap: mov disp(%rbp), %drap */
3114                                 cfa->base = cfi->drap_reg;
3115                                 cfa->offset = 0;
3116                                 cfi->drap_offset = -1;
3117                         }
3118
3119                         if (cfi->drap && op->src.reg == CFI_BP &&
3120                             op->src.offset == regs[op->dest.reg].offset) {
3121
3122                                 /* drap: mov disp(%rbp), %reg */
3123                                 restore_reg(cfi, op->dest.reg);
3124
3125                         } else if (op->src.reg == cfa->base &&
3126                             op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
3127
3128                                 /* mov disp(%rbp), %reg */
3129                                 /* mov disp(%rsp), %reg */
3130                                 restore_reg(cfi, op->dest.reg);
3131
3132                         } else if (op->src.reg == CFI_SP &&
3133                                    op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
3134
3135                                 /* mov disp(%rsp), %reg */
3136                                 restore_reg(cfi, op->dest.reg);
3137                         }
3138
3139                         break;
3140
3141                 default:
3142                         WARN_INSN(insn, "unknown stack-related instruction");
3143                         return -1;
3144                 }
3145
3146                 break;
3147
3148         case OP_DEST_PUSH:
3149         case OP_DEST_PUSHF:
3150                 cfi->stack_size += 8;
3151                 if (cfa->base == CFI_SP)
3152                         cfa->offset += 8;
3153
3154                 if (op->src.type != OP_SRC_REG)
3155                         break;
3156
3157                 if (cfi->drap) {
3158                         if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3159
3160                                 /* drap: push %drap */
3161                                 cfa->base = CFI_BP_INDIRECT;
3162                                 cfa->offset = -cfi->stack_size;
3163
3164                                 /* save drap so we know when to restore it */
3165                                 cfi->drap_offset = -cfi->stack_size;
3166
3167                         } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
3168
3169                                 /* drap: push %rbp */
3170                                 cfi->stack_size = 0;
3171
3172                         } else {
3173
3174                                 /* drap: push %reg */
3175                                 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
3176                         }
3177
3178                 } else {
3179
3180                         /* push %reg */
3181                         save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
3182                 }
3183
3184                 /* detect when asm code uses rbp as a scratch register */
3185                 if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP &&
3186                     cfa->base != CFI_BP)
3187                         cfi->bp_scratch = true;
3188                 break;
3189
3190         case OP_DEST_REG_INDIRECT:
3191
3192                 if (cfi->drap) {
3193                         if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3194
3195                                 /* drap: mov %drap, disp(%rbp) */
3196                                 cfa->base = CFI_BP_INDIRECT;
3197                                 cfa->offset = op->dest.offset;
3198
3199                                 /* save drap offset so we know when to restore it */
3200                                 cfi->drap_offset = op->dest.offset;
3201                         } else {
3202
3203                                 /* drap: mov reg, disp(%rbp) */
3204                                 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
3205                         }
3206
3207                 } else if (op->dest.reg == cfa->base) {
3208
3209                         /* mov reg, disp(%rbp) */
3210                         /* mov reg, disp(%rsp) */
3211                         save_reg(cfi, op->src.reg, CFI_CFA,
3212                                  op->dest.offset - cfi->cfa.offset);
3213
3214                 } else if (op->dest.reg == CFI_SP) {
3215
3216                         /* mov reg, disp(%rsp) */
3217                         save_reg(cfi, op->src.reg, CFI_CFA,
3218                                  op->dest.offset - cfi->stack_size);
3219
3220                 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
3221
3222                         /* mov %rsp, (%reg); # setup a stack swizzle. */
3223                         cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
3224                         cfi->vals[op->dest.reg].offset = cfa->offset;
3225                 }
3226
3227                 break;
3228
3229         case OP_DEST_MEM:
3230                 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
3231                         WARN_INSN(insn, "unknown stack-related memory operation");
3232                         return -1;
3233                 }
3234
3235                 /* pop mem */
3236                 cfi->stack_size -= 8;
3237                 if (cfa->base == CFI_SP)
3238                         cfa->offset -= 8;
3239
3240                 break;
3241
3242         default:
3243                 WARN_INSN(insn, "unknown stack-related instruction");
3244                 return -1;
3245         }
3246
3247         return 0;
3248 }
3249
3250 /*
3251  * The stack layouts of alternatives instructions can sometimes diverge when
3252  * they have stack modifications.  That's fine as long as the potential stack
3253  * layouts don't conflict at any given potential instruction boundary.
3254  *
3255  * Flatten the CFIs of the different alternative code streams (both original
3256  * and replacement) into a single shared CFI array which can be used to detect
3257  * conflicts and nicely feed a linear array of ORC entries to the unwinder.
3258  */
3259 static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
3260 {
3261         struct cfi_state **alt_cfi;
3262         int group_off;
3263
3264         if (!insn->alt_group)
3265                 return 0;
3266
3267         if (!insn->cfi) {
3268                 WARN("CFI missing");
3269                 return -1;
3270         }
3271
3272         alt_cfi = insn->alt_group->cfi;
3273         group_off = insn->offset - insn->alt_group->first_insn->offset;
3274
3275         if (!alt_cfi[group_off]) {
3276                 alt_cfi[group_off] = insn->cfi;
3277         } else {
3278                 if (cficmp(alt_cfi[group_off], insn->cfi)) {
3279                         struct alt_group *orig_group = insn->alt_group->orig_group ?: insn->alt_group;
3280                         struct instruction *orig = orig_group->first_insn;
3281                         char *where = offstr(insn->sec, insn->offset);
3282                         WARN_INSN(orig, "stack layout conflict in alternatives: %s", where);
3283                         free(where);
3284                         return -1;
3285                 }
3286         }
3287
3288         return 0;
3289 }
3290
3291 static int handle_insn_ops(struct instruction *insn,
3292                            struct instruction *next_insn,
3293                            struct insn_state *state)
3294 {
3295         struct stack_op *op;
3296
3297         for (op = insn->stack_ops; op; op = op->next) {
3298
3299                 if (update_cfi_state(insn, next_insn, &state->cfi, op))
3300                         return 1;
3301
3302                 if (!insn->alt_group)
3303                         continue;
3304
3305                 if (op->dest.type == OP_DEST_PUSHF) {
3306                         if (!state->uaccess_stack) {
3307                                 state->uaccess_stack = 1;
3308                         } else if (state->uaccess_stack >> 31) {
3309                                 WARN_INSN(insn, "PUSHF stack exhausted");
3310                                 return 1;
3311                         }
3312                         state->uaccess_stack <<= 1;
3313                         state->uaccess_stack  |= state->uaccess;
3314                 }
3315
3316                 if (op->src.type == OP_SRC_POPF) {
3317                         if (state->uaccess_stack) {
3318                                 state->uaccess = state->uaccess_stack & 1;
3319                                 state->uaccess_stack >>= 1;
3320                                 if (state->uaccess_stack == 1)
3321                                         state->uaccess_stack = 0;
3322                         }
3323                 }
3324         }
3325
3326         return 0;
3327 }
3328
3329 static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
3330 {
3331         struct cfi_state *cfi1 = insn->cfi;
3332         int i;
3333
3334         if (!cfi1) {
3335                 WARN("CFI missing");
3336                 return false;
3337         }
3338
3339         if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
3340
3341                 WARN_INSN(insn, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
3342                           cfi1->cfa.base, cfi1->cfa.offset,
3343                           cfi2->cfa.base, cfi2->cfa.offset);
3344
3345         } else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
3346                 for (i = 0; i < CFI_NUM_REGS; i++) {
3347                         if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
3348                                     sizeof(struct cfi_reg)))
3349                                 continue;
3350
3351                         WARN_INSN(insn, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
3352                                   i, cfi1->regs[i].base, cfi1->regs[i].offset,
3353                                   i, cfi2->regs[i].base, cfi2->regs[i].offset);
3354                         break;
3355                 }
3356
3357         } else if (cfi1->type != cfi2->type) {
3358
3359                 WARN_INSN(insn, "stack state mismatch: type1=%d type2=%d",
3360                           cfi1->type, cfi2->type);
3361
3362         } else if (cfi1->drap != cfi2->drap ||
3363                    (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
3364                    (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
3365
3366                 WARN_INSN(insn, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
3367                           cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
3368                           cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
3369
3370         } else
3371                 return true;
3372
3373         return false;
3374 }
3375
3376 static inline bool func_uaccess_safe(struct symbol *func)
3377 {
3378         if (func)
3379                 return func->uaccess_safe;
3380
3381         return false;
3382 }
3383
3384 static inline const char *call_dest_name(struct instruction *insn)
3385 {
3386         static char pvname[19];
3387         struct reloc *reloc;
3388         int idx;
3389
3390         if (insn_call_dest(insn))
3391                 return insn_call_dest(insn)->name;
3392
3393         reloc = insn_reloc(NULL, insn);
3394         if (reloc && !strcmp(reloc->sym->name, "pv_ops")) {
3395                 idx = (reloc_addend(reloc) / sizeof(void *));
3396                 snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx);
3397                 return pvname;
3398         }
3399
3400         return "{dynamic}";
3401 }
3402
3403 static bool pv_call_dest(struct objtool_file *file, struct instruction *insn)
3404 {
3405         struct symbol *target;
3406         struct reloc *reloc;
3407         int idx;
3408
3409         reloc = insn_reloc(file, insn);
3410         if (!reloc || strcmp(reloc->sym->name, "pv_ops"))
3411                 return false;
3412
3413         idx = (arch_dest_reloc_offset(reloc_addend(reloc)) / sizeof(void *));
3414
3415         if (file->pv_ops[idx].clean)
3416                 return true;
3417
3418         file->pv_ops[idx].clean = true;
3419
3420         list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) {
3421                 if (!target->sec->noinstr) {
3422                         WARN("pv_ops[%d]: %s", idx, target->name);
3423                         file->pv_ops[idx].clean = false;
3424                 }
3425         }
3426
3427         return file->pv_ops[idx].clean;
3428 }
3429
3430 static inline bool noinstr_call_dest(struct objtool_file *file,
3431                                      struct instruction *insn,
3432                                      struct symbol *func)
3433 {
3434         /*
3435          * We can't deal with indirect function calls at present;
3436          * assume they're instrumented.
3437          */
3438         if (!func) {
3439                 if (file->pv_ops)
3440                         return pv_call_dest(file, insn);
3441
3442                 return false;
3443         }
3444
3445         /*
3446          * If the symbol is from a noinstr section; we good.
3447          */
3448         if (func->sec->noinstr)
3449                 return true;
3450
3451         /*
3452          * If the symbol is a static_call trampoline, we can't tell.
3453          */
3454         if (func->static_call_tramp)
3455                 return true;
3456
3457         /*
3458          * The __ubsan_handle_*() calls are like WARN(), they only happen when
3459          * something 'BAD' happened. At the risk of taking the machine down,
3460          * let them proceed to get the message out.
3461          */
3462         if (!strncmp(func->name, "__ubsan_handle_", 15))
3463                 return true;
3464
3465         return false;
3466 }
3467
3468 static int validate_call(struct objtool_file *file,
3469                          struct instruction *insn,
3470                          struct insn_state *state)
3471 {
3472         if (state->noinstr && state->instr <= 0 &&
3473             !noinstr_call_dest(file, insn, insn_call_dest(insn))) {
3474                 WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn));
3475                 return 1;
3476         }
3477
3478         if (state->uaccess && !func_uaccess_safe(insn_call_dest(insn))) {
3479                 WARN_INSN(insn, "call to %s() with UACCESS enabled", call_dest_name(insn));
3480                 return 1;
3481         }
3482
3483         if (state->df) {
3484                 WARN_INSN(insn, "call to %s() with DF set", call_dest_name(insn));
3485                 return 1;
3486         }
3487
3488         return 0;
3489 }
3490
3491 static int validate_sibling_call(struct objtool_file *file,
3492                                  struct instruction *insn,
3493                                  struct insn_state *state)
3494 {
3495         if (insn_func(insn) && has_modified_stack_frame(insn, state)) {
3496                 WARN_INSN(insn, "sibling call from callable instruction with modified stack frame");
3497                 return 1;
3498         }
3499
3500         return validate_call(file, insn, state);
3501 }
3502
3503 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
3504 {
3505         if (state->noinstr && state->instr > 0) {
3506                 WARN_INSN(insn, "return with instrumentation enabled");
3507                 return 1;
3508         }
3509
3510         if (state->uaccess && !func_uaccess_safe(func)) {
3511                 WARN_INSN(insn, "return with UACCESS enabled");
3512                 return 1;
3513         }
3514
3515         if (!state->uaccess && func_uaccess_safe(func)) {
3516                 WARN_INSN(insn, "return with UACCESS disabled from a UACCESS-safe function");
3517                 return 1;
3518         }
3519
3520         if (state->df) {
3521                 WARN_INSN(insn, "return with DF set");
3522                 return 1;
3523         }
3524
3525         if (func && has_modified_stack_frame(insn, state)) {
3526                 WARN_INSN(insn, "return with modified stack frame");
3527                 return 1;
3528         }
3529
3530         if (state->cfi.bp_scratch) {
3531                 WARN_INSN(insn, "BP used as a scratch register");
3532                 return 1;
3533         }
3534
3535         return 0;
3536 }
3537
3538 static struct instruction *next_insn_to_validate(struct objtool_file *file,
3539                                                  struct instruction *insn)
3540 {
3541         struct alt_group *alt_group = insn->alt_group;
3542
3543         /*
3544          * Simulate the fact that alternatives are patched in-place.  When the
3545          * end of a replacement alt_group is reached, redirect objtool flow to
3546          * the end of the original alt_group.
3547          *
3548          * insn->alts->insn -> alt_group->first_insn
3549          *                     ...
3550          *                     alt_group->last_insn
3551          *                     [alt_group->nop]      -> next(orig_group->last_insn)
3552          */
3553         if (alt_group) {
3554                 if (alt_group->nop) {
3555                         /* ->nop implies ->orig_group */
3556                         if (insn == alt_group->last_insn)
3557                                 return alt_group->nop;
3558                         if (insn == alt_group->nop)
3559                                 goto next_orig;
3560                 }
3561                 if (insn == alt_group->last_insn && alt_group->orig_group)
3562                         goto next_orig;
3563         }
3564
3565         return next_insn_same_sec(file, insn);
3566
3567 next_orig:
3568         return next_insn_same_sec(file, alt_group->orig_group->last_insn);
3569 }
3570
3571 /*
3572  * Follow the branch starting at the given instruction, and recursively follow
3573  * any other branches (jumps).  Meanwhile, track the frame pointer state at
3574  * each instruction and validate all the rules described in
3575  * tools/objtool/Documentation/objtool.txt.
3576  */
3577 static int validate_branch(struct objtool_file *file, struct symbol *func,
3578                            struct instruction *insn, struct insn_state state)
3579 {
3580         struct alternative *alt;
3581         struct instruction *next_insn, *prev_insn = NULL;
3582         struct section *sec;
3583         u8 visited;
3584         int ret;
3585
3586         sec = insn->sec;
3587
3588         while (1) {
3589                 next_insn = next_insn_to_validate(file, insn);
3590
3591                 if (func && insn_func(insn) && func != insn_func(insn)->pfunc) {
3592                         /* Ignore KCFI type preambles, which always fall through */
3593                         if (!strncmp(func->name, "__cfi_", 6) ||
3594                             !strncmp(func->name, "__pfx_", 6))
3595                                 return 0;
3596
3597                         WARN("%s() falls through to next function %s()",
3598                              func->name, insn_func(insn)->name);
3599                         return 1;
3600                 }
3601
3602                 if (func && insn->ignore) {
3603                         WARN_INSN(insn, "BUG: why am I validating an ignored function?");
3604                         return 1;
3605                 }
3606
3607                 visited = VISITED_BRANCH << state.uaccess;
3608                 if (insn->visited & VISITED_BRANCH_MASK) {
3609                         if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
3610                                 return 1;
3611
3612                         if (insn->visited & visited)
3613                                 return 0;
3614                 } else {
3615                         nr_insns_visited++;
3616                 }
3617
3618                 if (state.noinstr)
3619                         state.instr += insn->instr;
3620
3621                 if (insn->hint) {
3622                         if (insn->restore) {
3623                                 struct instruction *save_insn, *i;
3624
3625                                 i = insn;
3626                                 save_insn = NULL;
3627
3628                                 sym_for_each_insn_continue_reverse(file, func, i) {
3629                                         if (i->save) {
3630                                                 save_insn = i;
3631                                                 break;
3632                                         }
3633                                 }
3634
3635                                 if (!save_insn) {
3636                                         WARN_INSN(insn, "no corresponding CFI save for CFI restore");
3637                                         return 1;
3638                                 }
3639
3640                                 if (!save_insn->visited) {
3641                                         /*
3642                                          * If the restore hint insn is at the
3643                                          * beginning of a basic block and was
3644                                          * branched to from elsewhere, and the
3645                                          * save insn hasn't been visited yet,
3646                                          * defer following this branch for now.
3647                                          * It will be seen later via the
3648                                          * straight-line path.
3649                                          */
3650                                         if (!prev_insn)
3651                                                 return 0;
3652
3653                                         WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo");
3654                                         return 1;
3655                                 }
3656
3657                                 insn->cfi = save_insn->cfi;
3658                                 nr_cfi_reused++;
3659                         }
3660
3661                         state.cfi = *insn->cfi;
3662                 } else {
3663                         /* XXX track if we actually changed state.cfi */
3664
3665                         if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) {
3666                                 insn->cfi = prev_insn->cfi;
3667                                 nr_cfi_reused++;
3668                         } else {
3669                                 insn->cfi = cfi_hash_find_or_add(&state.cfi);
3670                         }
3671                 }
3672
3673                 insn->visited |= visited;
3674
3675                 if (propagate_alt_cfi(file, insn))
3676                         return 1;
3677
3678                 if (!insn->ignore_alts && insn->alts) {
3679                         bool skip_orig = false;
3680
3681                         for (alt = insn->alts; alt; alt = alt->next) {
3682                                 if (alt->skip_orig)
3683                                         skip_orig = true;
3684
3685                                 ret = validate_branch(file, func, alt->insn, state);
3686                                 if (ret) {
3687                                         BT_INSN(insn, "(alt)");
3688                                         return ret;
3689                                 }
3690                         }
3691
3692                         if (skip_orig)
3693                                 return 0;
3694                 }
3695
3696                 if (handle_insn_ops(insn, next_insn, &state))
3697                         return 1;
3698
3699                 switch (insn->type) {
3700
3701                 case INSN_RETURN:
3702                         return validate_return(func, insn, &state);
3703
3704                 case INSN_CALL:
3705                 case INSN_CALL_DYNAMIC:
3706                         ret = validate_call(file, insn, &state);
3707                         if (ret)
3708                                 return ret;
3709
3710                         if (opts.stackval && func && !is_special_call(insn) &&
3711                             !has_valid_stack_frame(&state)) {
3712                                 WARN_INSN(insn, "call without frame pointer save/setup");
3713                                 return 1;
3714                         }
3715
3716                         if (insn->dead_end)
3717                                 return 0;
3718
3719                         break;
3720
3721                 case INSN_JUMP_CONDITIONAL:
3722                 case INSN_JUMP_UNCONDITIONAL:
3723                         if (is_sibling_call(insn)) {
3724                                 ret = validate_sibling_call(file, insn, &state);
3725                                 if (ret)
3726                                         return ret;
3727
3728                         } else if (insn->jump_dest) {
3729                                 ret = validate_branch(file, func,
3730                                                       insn->jump_dest, state);
3731                                 if (ret) {
3732                                         BT_INSN(insn, "(branch)");
3733                                         return ret;
3734                                 }
3735                         }
3736
3737                         if (insn->type == INSN_JUMP_UNCONDITIONAL)
3738                                 return 0;
3739
3740                         break;
3741
3742                 case INSN_JUMP_DYNAMIC:
3743                 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3744                         if (is_sibling_call(insn)) {
3745                                 ret = validate_sibling_call(file, insn, &state);
3746                                 if (ret)
3747                                         return ret;
3748                         }
3749
3750                         if (insn->type == INSN_JUMP_DYNAMIC)
3751                                 return 0;
3752
3753                         break;
3754
3755                 case INSN_CONTEXT_SWITCH:
3756                         if (func && (!next_insn || !next_insn->hint)) {
3757                                 WARN_INSN(insn, "unsupported instruction in callable function");
3758                                 return 1;
3759                         }
3760                         return 0;
3761
3762                 case INSN_STAC:
3763                         if (state.uaccess) {
3764                                 WARN_INSN(insn, "recursive UACCESS enable");
3765                                 return 1;
3766                         }
3767
3768                         state.uaccess = true;
3769                         break;
3770
3771                 case INSN_CLAC:
3772                         if (!state.uaccess && func) {
3773                                 WARN_INSN(insn, "redundant UACCESS disable");
3774                                 return 1;
3775                         }
3776
3777                         if (func_uaccess_safe(func) && !state.uaccess_stack) {
3778                                 WARN_INSN(insn, "UACCESS-safe disables UACCESS");
3779                                 return 1;
3780                         }
3781
3782                         state.uaccess = false;
3783                         break;
3784
3785                 case INSN_STD:
3786                         if (state.df) {
3787                                 WARN_INSN(insn, "recursive STD");
3788                                 return 1;
3789                         }
3790
3791                         state.df = true;
3792                         break;
3793
3794                 case INSN_CLD:
3795                         if (!state.df && func) {
3796                                 WARN_INSN(insn, "redundant CLD");
3797                                 return 1;
3798                         }
3799
3800                         state.df = false;
3801                         break;
3802
3803                 default:
3804                         break;
3805                 }
3806
3807                 if (insn->dead_end)
3808                         return 0;
3809
3810                 if (!next_insn) {
3811                         if (state.cfi.cfa.base == CFI_UNDEFINED)
3812                                 return 0;
3813                         WARN("%s: unexpected end of section", sec->name);
3814                         return 1;
3815                 }
3816
3817                 prev_insn = insn;
3818                 insn = next_insn;
3819         }
3820
3821         return 0;
3822 }
3823
3824 static int validate_unwind_hint(struct objtool_file *file,
3825                                   struct instruction *insn,
3826                                   struct insn_state *state)
3827 {
3828         if (insn->hint && !insn->visited && !insn->ignore) {
3829                 int ret = validate_branch(file, insn_func(insn), insn, *state);
3830                 if (ret)
3831                         BT_INSN(insn, "<=== (hint)");
3832                 return ret;
3833         }
3834
3835         return 0;
3836 }
3837
3838 static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
3839 {
3840         struct instruction *insn;
3841         struct insn_state state;
3842         int warnings = 0;
3843
3844         if (!file->hints)
3845                 return 0;
3846
3847         init_insn_state(file, &state, sec);
3848
3849         if (sec) {
3850                 sec_for_each_insn(file, sec, insn)
3851                         warnings += validate_unwind_hint(file, insn, &state);
3852         } else {
3853                 for_each_insn(file, insn)
3854                         warnings += validate_unwind_hint(file, insn, &state);
3855         }
3856
3857         return warnings;
3858 }
3859
3860 /*
3861  * Validate rethunk entry constraint: must untrain RET before the first RET.
3862  *
3863  * Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes
3864  * before an actual RET instruction.
3865  */
3866 static int validate_unret(struct objtool_file *file, struct instruction *insn)
3867 {
3868         struct instruction *next, *dest;
3869         int ret;
3870
3871         for (;;) {
3872                 next = next_insn_to_validate(file, insn);
3873
3874                 if (insn->visited & VISITED_UNRET)
3875                         return 0;
3876
3877                 insn->visited |= VISITED_UNRET;
3878
3879                 if (!insn->ignore_alts && insn->alts) {
3880                         struct alternative *alt;
3881                         bool skip_orig = false;
3882
3883                         for (alt = insn->alts; alt; alt = alt->next) {
3884                                 if (alt->skip_orig)
3885                                         skip_orig = true;
3886
3887                                 ret = validate_unret(file, alt->insn);
3888                                 if (ret) {
3889                                         BT_INSN(insn, "(alt)");
3890                                         return ret;
3891                                 }
3892                         }
3893
3894                         if (skip_orig)
3895                                 return 0;
3896                 }
3897
3898                 switch (insn->type) {
3899
3900                 case INSN_CALL_DYNAMIC:
3901                 case INSN_JUMP_DYNAMIC:
3902                 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3903                         WARN_INSN(insn, "early indirect call");
3904                         return 1;
3905
3906                 case INSN_JUMP_UNCONDITIONAL:
3907                 case INSN_JUMP_CONDITIONAL:
3908                         if (!is_sibling_call(insn)) {
3909                                 if (!insn->jump_dest) {
3910                                         WARN_INSN(insn, "unresolved jump target after linking?!?");
3911                                         return -1;
3912                                 }
3913                                 ret = validate_unret(file, insn->jump_dest);
3914                                 if (ret) {
3915                                         BT_INSN(insn, "(branch%s)",
3916                                                 insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : "");
3917                                         return ret;
3918                                 }
3919
3920                                 if (insn->type == INSN_JUMP_UNCONDITIONAL)
3921                                         return 0;
3922
3923                                 break;
3924                         }
3925
3926                         /* fallthrough */
3927                 case INSN_CALL:
3928                         dest = find_insn(file, insn_call_dest(insn)->sec,
3929                                          insn_call_dest(insn)->offset);
3930                         if (!dest) {
3931                                 WARN("Unresolved function after linking!?: %s",
3932                                      insn_call_dest(insn)->name);
3933                                 return -1;
3934                         }
3935
3936                         ret = validate_unret(file, dest);
3937                         if (ret) {
3938                                 BT_INSN(insn, "(call)");
3939                                 return ret;
3940                         }
3941                         /*
3942                          * If a call returns without error, it must have seen UNTRAIN_RET.
3943                          * Therefore any non-error return is a success.
3944                          */
3945                         return 0;
3946
3947                 case INSN_RETURN:
3948                         WARN_INSN(insn, "RET before UNTRAIN");
3949                         return 1;
3950
3951                 case INSN_NOP:
3952                         if (insn->retpoline_safe)
3953                                 return 0;
3954                         break;
3955
3956                 default:
3957                         break;
3958                 }
3959
3960                 if (!next) {
3961                         WARN_INSN(insn, "teh end!");
3962                         return -1;
3963                 }
3964                 insn = next;
3965         }
3966
3967         return 0;
3968 }
3969
3970 /*
3971  * Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter
3972  * VALIDATE_UNRET_END before RET.
3973  */
3974 static int validate_unrets(struct objtool_file *file)
3975 {
3976         struct instruction *insn;
3977         int ret, warnings = 0;
3978
3979         for_each_insn(file, insn) {
3980                 if (!insn->unret)
3981                         continue;
3982
3983                 ret = validate_unret(file, insn);
3984                 if (ret < 0) {
3985                         WARN_INSN(insn, "Failed UNRET validation");
3986                         return ret;
3987                 }
3988                 warnings += ret;
3989         }
3990
3991         return warnings;
3992 }
3993
3994 static int validate_retpoline(struct objtool_file *file)
3995 {
3996         struct instruction *insn;
3997         int warnings = 0;
3998
3999         for_each_insn(file, insn) {
4000                 if (insn->type != INSN_JUMP_DYNAMIC &&
4001                     insn->type != INSN_CALL_DYNAMIC &&
4002                     insn->type != INSN_RETURN)
4003                         continue;
4004
4005                 if (insn->retpoline_safe)
4006                         continue;
4007
4008                 if (insn->sec->init)
4009                         continue;
4010
4011                 if (insn->type == INSN_RETURN) {
4012                         if (opts.rethunk) {
4013                                 WARN_INSN(insn, "'naked' return found in MITIGATION_RETHUNK build");
4014                         } else
4015                                 continue;
4016                 } else {
4017                         WARN_INSN(insn, "indirect %s found in MITIGATION_RETPOLINE build",
4018                                   insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
4019                 }
4020
4021                 warnings++;
4022         }
4023
4024         return warnings;
4025 }
4026
4027 static bool is_kasan_insn(struct instruction *insn)
4028 {
4029         return (insn->type == INSN_CALL &&
4030                 !strcmp(insn_call_dest(insn)->name, "__asan_handle_no_return"));
4031 }
4032
4033 static bool is_ubsan_insn(struct instruction *insn)
4034 {
4035         return (insn->type == INSN_CALL &&
4036                 !strcmp(insn_call_dest(insn)->name,
4037                         "__ubsan_handle_builtin_unreachable"));
4038 }
4039
4040 static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
4041 {
4042         int i;
4043         struct instruction *prev_insn;
4044
4045         if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP)
4046                 return true;
4047
4048         /*
4049          * Ignore alternative replacement instructions.  This can happen
4050          * when a whitelisted function uses one of the ALTERNATIVE macros.
4051          */
4052         if (!strcmp(insn->sec->name, ".altinstr_replacement") ||
4053             !strcmp(insn->sec->name, ".altinstr_aux"))
4054                 return true;
4055
4056         /*
4057          * Whole archive runs might encounter dead code from weak symbols.
4058          * This is where the linker will have dropped the weak symbol in
4059          * favour of a regular symbol, but leaves the code in place.
4060          *
4061          * In this case we'll find a piece of code (whole function) that is not
4062          * covered by a !section symbol. Ignore them.
4063          */
4064         if (opts.link && !insn_func(insn)) {
4065                 int size = find_symbol_hole_containing(insn->sec, insn->offset);
4066                 unsigned long end = insn->offset + size;
4067
4068                 if (!size) /* not a hole */
4069                         return false;
4070
4071                 if (size < 0) /* hole until the end */
4072                         return true;
4073
4074                 sec_for_each_insn_continue(file, insn) {
4075                         /*
4076                          * If we reach a visited instruction at or before the
4077                          * end of the hole, ignore the unreachable.
4078                          */
4079                         if (insn->visited)
4080                                 return true;
4081
4082                         if (insn->offset >= end)
4083                                 break;
4084
4085                         /*
4086                          * If this hole jumps to a .cold function, mark it ignore too.
4087                          */
4088                         if (insn->jump_dest && insn_func(insn->jump_dest) &&
4089                             strstr(insn_func(insn->jump_dest)->name, ".cold")) {
4090                                 struct instruction *dest = insn->jump_dest;
4091                                 func_for_each_insn(file, insn_func(dest), dest)
4092                                         dest->ignore = true;
4093                         }
4094                 }
4095
4096                 return false;
4097         }
4098
4099         if (!insn_func(insn))
4100                 return false;
4101
4102         if (insn_func(insn)->static_call_tramp)
4103                 return true;
4104
4105         /*
4106          * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
4107          * __builtin_unreachable().  The BUG() macro has an unreachable() after
4108          * the UD2, which causes GCC's undefined trap logic to emit another UD2
4109          * (or occasionally a JMP to UD2).
4110          *
4111          * It may also insert a UD2 after calling a __noreturn function.
4112          */
4113         prev_insn = prev_insn_same_sec(file, insn);
4114         if (prev_insn->dead_end &&
4115             (insn->type == INSN_BUG ||
4116              (insn->type == INSN_JUMP_UNCONDITIONAL &&
4117               insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
4118                 return true;
4119
4120         /*
4121          * Check if this (or a subsequent) instruction is related to
4122          * CONFIG_UBSAN or CONFIG_KASAN.
4123          *
4124          * End the search at 5 instructions to avoid going into the weeds.
4125          */
4126         for (i = 0; i < 5; i++) {
4127
4128                 if (is_kasan_insn(insn) || is_ubsan_insn(insn))
4129                         return true;
4130
4131                 if (insn->type == INSN_JUMP_UNCONDITIONAL) {
4132                         if (insn->jump_dest &&
4133                             insn_func(insn->jump_dest) == insn_func(insn)) {
4134                                 insn = insn->jump_dest;
4135                                 continue;
4136                         }
4137
4138                         break;
4139                 }
4140
4141                 if (insn->offset + insn->len >= insn_func(insn)->offset + insn_func(insn)->len)
4142                         break;
4143
4144                 insn = next_insn_same_sec(file, insn);
4145         }
4146
4147         return false;
4148 }
4149
4150 static int add_prefix_symbol(struct objtool_file *file, struct symbol *func)
4151 {
4152         struct instruction *insn, *prev;
4153         struct cfi_state *cfi;
4154
4155         insn = find_insn(file, func->sec, func->offset);
4156         if (!insn)
4157                 return -1;
4158
4159         for (prev = prev_insn_same_sec(file, insn);
4160              prev;
4161              prev = prev_insn_same_sec(file, prev)) {
4162                 u64 offset;
4163
4164                 if (prev->type != INSN_NOP)
4165                         return -1;
4166
4167                 offset = func->offset - prev->offset;
4168
4169                 if (offset > opts.prefix)
4170                         return -1;
4171
4172                 if (offset < opts.prefix)
4173                         continue;
4174
4175                 elf_create_prefix_symbol(file->elf, func, opts.prefix);
4176                 break;
4177         }
4178
4179         if (!prev)
4180                 return -1;
4181
4182         if (!insn->cfi) {
4183                 /*
4184                  * This can happen if stack validation isn't enabled or the
4185                  * function is annotated with STACK_FRAME_NON_STANDARD.
4186                  */
4187                 return 0;
4188         }
4189
4190         /* Propagate insn->cfi to the prefix code */
4191         cfi = cfi_hash_find_or_add(insn->cfi);
4192         for (; prev != insn; prev = next_insn_same_sec(file, prev))
4193                 prev->cfi = cfi;
4194
4195         return 0;
4196 }
4197
4198 static int add_prefix_symbols(struct objtool_file *file)
4199 {
4200         struct section *sec;
4201         struct symbol *func;
4202
4203         for_each_sec(file, sec) {
4204                 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4205                         continue;
4206
4207                 sec_for_each_sym(sec, func) {
4208                         if (func->type != STT_FUNC)
4209                                 continue;
4210
4211                         add_prefix_symbol(file, func);
4212                 }
4213         }
4214
4215         return 0;
4216 }
4217
4218 static int validate_symbol(struct objtool_file *file, struct section *sec,
4219                            struct symbol *sym, struct insn_state *state)
4220 {
4221         struct instruction *insn;
4222         int ret;
4223
4224         if (!sym->len) {
4225                 WARN("%s() is missing an ELF size annotation", sym->name);
4226                 return 1;
4227         }
4228
4229         if (sym->pfunc != sym || sym->alias != sym)
4230                 return 0;
4231
4232         insn = find_insn(file, sec, sym->offset);
4233         if (!insn || insn->ignore || insn->visited)
4234                 return 0;
4235
4236         state->uaccess = sym->uaccess_safe;
4237
4238         ret = validate_branch(file, insn_func(insn), insn, *state);
4239         if (ret)
4240                 BT_INSN(insn, "<=== (sym)");
4241         return ret;
4242 }
4243
4244 static int validate_section(struct objtool_file *file, struct section *sec)
4245 {
4246         struct insn_state state;
4247         struct symbol *func;
4248         int warnings = 0;
4249
4250         sec_for_each_sym(sec, func) {
4251                 if (func->type != STT_FUNC)
4252                         continue;
4253
4254                 init_insn_state(file, &state, sec);
4255                 set_func_state(&state.cfi);
4256
4257                 warnings += validate_symbol(file, sec, func, &state);
4258         }
4259
4260         return warnings;
4261 }
4262
4263 static int validate_noinstr_sections(struct objtool_file *file)
4264 {
4265         struct section *sec;
4266         int warnings = 0;
4267
4268         sec = find_section_by_name(file->elf, ".noinstr.text");
4269         if (sec) {
4270                 warnings += validate_section(file, sec);
4271                 warnings += validate_unwind_hints(file, sec);
4272         }
4273
4274         sec = find_section_by_name(file->elf, ".entry.text");
4275         if (sec) {
4276                 warnings += validate_section(file, sec);
4277                 warnings += validate_unwind_hints(file, sec);
4278         }
4279
4280         sec = find_section_by_name(file->elf, ".cpuidle.text");
4281         if (sec) {
4282                 warnings += validate_section(file, sec);
4283                 warnings += validate_unwind_hints(file, sec);
4284         }
4285
4286         return warnings;
4287 }
4288
4289 static int validate_functions(struct objtool_file *file)
4290 {
4291         struct section *sec;
4292         int warnings = 0;
4293
4294         for_each_sec(file, sec) {
4295                 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4296                         continue;
4297
4298                 warnings += validate_section(file, sec);
4299         }
4300
4301         return warnings;
4302 }
4303
4304 static void mark_endbr_used(struct instruction *insn)
4305 {
4306         if (!list_empty(&insn->call_node))
4307                 list_del_init(&insn->call_node);
4308 }
4309
4310 static bool noendbr_range(struct objtool_file *file, struct instruction *insn)
4311 {
4312         struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1);
4313         struct instruction *first;
4314
4315         if (!sym)
4316                 return false;
4317
4318         first = find_insn(file, sym->sec, sym->offset);
4319         if (!first)
4320                 return false;
4321
4322         if (first->type != INSN_ENDBR && !first->noendbr)
4323                 return false;
4324
4325         return insn->offset == sym->offset + sym->len;
4326 }
4327
4328 static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn)
4329 {
4330         struct instruction *dest;
4331         struct reloc *reloc;
4332         unsigned long off;
4333         int warnings = 0;
4334
4335         /*
4336          * Looking for function pointer load relocations.  Ignore
4337          * direct/indirect branches:
4338          */
4339         switch (insn->type) {
4340         case INSN_CALL:
4341         case INSN_CALL_DYNAMIC:
4342         case INSN_JUMP_CONDITIONAL:
4343         case INSN_JUMP_UNCONDITIONAL:
4344         case INSN_JUMP_DYNAMIC:
4345         case INSN_JUMP_DYNAMIC_CONDITIONAL:
4346         case INSN_RETURN:
4347         case INSN_NOP:
4348                 return 0;
4349         default:
4350                 break;
4351         }
4352
4353         for (reloc = insn_reloc(file, insn);
4354              reloc;
4355              reloc = find_reloc_by_dest_range(file->elf, insn->sec,
4356                                               reloc_offset(reloc) + 1,
4357                                               (insn->offset + insn->len) - (reloc_offset(reloc) + 1))) {
4358
4359                 /*
4360                  * static_call_update() references the trampoline, which
4361                  * doesn't have (or need) ENDBR.  Skip warning in that case.
4362                  */
4363                 if (reloc->sym->static_call_tramp)
4364                         continue;
4365
4366                 off = reloc->sym->offset;
4367                 if (reloc_type(reloc) == R_X86_64_PC32 ||
4368                     reloc_type(reloc) == R_X86_64_PLT32)
4369                         off += arch_dest_reloc_offset(reloc_addend(reloc));
4370                 else
4371                         off += reloc_addend(reloc);
4372
4373                 dest = find_insn(file, reloc->sym->sec, off);
4374                 if (!dest)
4375                         continue;
4376
4377                 if (dest->type == INSN_ENDBR) {
4378                         mark_endbr_used(dest);
4379                         continue;
4380                 }
4381
4382                 if (insn_func(dest) && insn_func(insn) &&
4383                     insn_func(dest)->pfunc == insn_func(insn)->pfunc) {
4384                         /*
4385                          * Anything from->to self is either _THIS_IP_ or
4386                          * IRET-to-self.
4387                          *
4388                          * There is no sane way to annotate _THIS_IP_ since the
4389                          * compiler treats the relocation as a constant and is
4390                          * happy to fold in offsets, skewing any annotation we
4391                          * do, leading to vast amounts of false-positives.
4392                          *
4393                          * There's also compiler generated _THIS_IP_ through
4394                          * KCOV and such which we have no hope of annotating.
4395                          *
4396                          * As such, blanket accept self-references without
4397                          * issue.
4398                          */
4399                         continue;
4400                 }
4401
4402                 /*
4403                  * Accept anything ANNOTATE_NOENDBR.
4404                  */
4405                 if (dest->noendbr)
4406                         continue;
4407
4408                 /*
4409                  * Accept if this is the instruction after a symbol
4410                  * that is (no)endbr -- typical code-range usage.
4411                  */
4412                 if (noendbr_range(file, dest))
4413                         continue;
4414
4415                 WARN_INSN(insn, "relocation to !ENDBR: %s", offstr(dest->sec, dest->offset));
4416
4417                 warnings++;
4418         }
4419
4420         return warnings;
4421 }
4422
4423 static int validate_ibt_data_reloc(struct objtool_file *file,
4424                                    struct reloc *reloc)
4425 {
4426         struct instruction *dest;
4427
4428         dest = find_insn(file, reloc->sym->sec,
4429                          reloc->sym->offset + reloc_addend(reloc));
4430         if (!dest)
4431                 return 0;
4432
4433         if (dest->type == INSN_ENDBR) {
4434                 mark_endbr_used(dest);
4435                 return 0;
4436         }
4437
4438         if (dest->noendbr)
4439                 return 0;
4440
4441         WARN_FUNC("data relocation to !ENDBR: %s",
4442                   reloc->sec->base, reloc_offset(reloc),
4443                   offstr(dest->sec, dest->offset));
4444
4445         return 1;
4446 }
4447
4448 /*
4449  * Validate IBT rules and remove used ENDBR instructions from the seal list.
4450  * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with
4451  * NOPs) later, in create_ibt_endbr_seal_sections().
4452  */
4453 static int validate_ibt(struct objtool_file *file)
4454 {
4455         struct section *sec;
4456         struct reloc *reloc;
4457         struct instruction *insn;
4458         int warnings = 0;
4459
4460         for_each_insn(file, insn)
4461                 warnings += validate_ibt_insn(file, insn);
4462
4463         for_each_sec(file, sec) {
4464
4465                 /* Already done by validate_ibt_insn() */
4466                 if (sec->sh.sh_flags & SHF_EXECINSTR)
4467                         continue;
4468
4469                 if (!sec->rsec)
4470                         continue;
4471
4472                 /*
4473                  * These sections can reference text addresses, but not with
4474                  * the intent to indirect branch to them.
4475                  */
4476                 if ((!strncmp(sec->name, ".discard", 8) &&
4477                      strcmp(sec->name, ".discard.ibt_endbr_noseal"))    ||
4478                     !strncmp(sec->name, ".debug", 6)                    ||
4479                     !strcmp(sec->name, ".altinstructions")              ||
4480                     !strcmp(sec->name, ".ibt_endbr_seal")               ||
4481                     !strcmp(sec->name, ".orc_unwind_ip")                ||
4482                     !strcmp(sec->name, ".parainstructions")             ||
4483                     !strcmp(sec->name, ".retpoline_sites")              ||
4484                     !strcmp(sec->name, ".smp_locks")                    ||
4485                     !strcmp(sec->name, ".static_call_sites")            ||
4486                     !strcmp(sec->name, "_error_injection_whitelist")    ||
4487                     !strcmp(sec->name, "_kprobe_blacklist")             ||
4488                     !strcmp(sec->name, "__bug_table")                   ||
4489                     !strcmp(sec->name, "__ex_table")                    ||
4490                     !strcmp(sec->name, "__jump_table")                  ||
4491                     !strcmp(sec->name, "__mcount_loc")                  ||
4492                     !strcmp(sec->name, ".kcfi_traps")                   ||
4493                     strstr(sec->name, "__patchable_function_entries"))
4494                         continue;
4495
4496                 for_each_reloc(sec->rsec, reloc)
4497                         warnings += validate_ibt_data_reloc(file, reloc);
4498         }
4499
4500         return warnings;
4501 }
4502
4503 static int validate_sls(struct objtool_file *file)
4504 {
4505         struct instruction *insn, *next_insn;
4506         int warnings = 0;
4507
4508         for_each_insn(file, insn) {
4509                 next_insn = next_insn_same_sec(file, insn);
4510
4511                 if (insn->retpoline_safe)
4512                         continue;
4513
4514                 switch (insn->type) {
4515                 case INSN_RETURN:
4516                         if (!next_insn || next_insn->type != INSN_TRAP) {
4517                                 WARN_INSN(insn, "missing int3 after ret");
4518                                 warnings++;
4519                         }
4520
4521                         break;
4522                 case INSN_JUMP_DYNAMIC:
4523                         if (!next_insn || next_insn->type != INSN_TRAP) {
4524                                 WARN_INSN(insn, "missing int3 after indirect jump");
4525                                 warnings++;
4526                         }
4527                         break;
4528                 default:
4529                         break;
4530                 }
4531         }
4532
4533         return warnings;
4534 }
4535
4536 static bool ignore_noreturn_call(struct instruction *insn)
4537 {
4538         struct symbol *call_dest = insn_call_dest(insn);
4539
4540         /*
4541          * FIXME: hack, we need a real noreturn solution
4542          *
4543          * Problem is, exc_double_fault() may or may not return, depending on
4544          * whether CONFIG_X86_ESPFIX64 is set.  But objtool has no visibility
4545          * to the kernel config.
4546          *
4547          * Other potential ways to fix it:
4548          *
4549          *   - have compiler communicate __noreturn functions somehow
4550          *   - remove CONFIG_X86_ESPFIX64
4551          *   - read the .config file
4552          *   - add a cmdline option
4553          *   - create a generic objtool annotation format (vs a bunch of custom
4554          *     formats) and annotate it
4555          */
4556         if (!strcmp(call_dest->name, "exc_double_fault")) {
4557                 /* prevent further unreachable warnings for the caller */
4558                 insn->sym->warned = 1;
4559                 return true;
4560         }
4561
4562         return false;
4563 }
4564
4565 static int validate_reachable_instructions(struct objtool_file *file)
4566 {
4567         struct instruction *insn, *prev_insn;
4568         struct symbol *call_dest;
4569         int warnings = 0;
4570
4571         if (file->ignore_unreachables)
4572                 return 0;
4573
4574         for_each_insn(file, insn) {
4575                 if (insn->visited || ignore_unreachable_insn(file, insn))
4576                         continue;
4577
4578                 prev_insn = prev_insn_same_sec(file, insn);
4579                 if (prev_insn && prev_insn->dead_end) {
4580                         call_dest = insn_call_dest(prev_insn);
4581                         if (call_dest && !ignore_noreturn_call(prev_insn)) {
4582                                 WARN_INSN(insn, "%s() is missing a __noreturn annotation",
4583                                           call_dest->name);
4584                                 warnings++;
4585                                 continue;
4586                         }
4587                 }
4588
4589                 WARN_INSN(insn, "unreachable instruction");
4590                 warnings++;
4591         }
4592
4593         return warnings;
4594 }
4595
4596 /* 'funcs' is a space-separated list of function names */
4597 static int disas_funcs(const char *funcs)
4598 {
4599         const char *objdump_str, *cross_compile;
4600         int size, ret;
4601         char *cmd;
4602
4603         cross_compile = getenv("CROSS_COMPILE");
4604
4605         objdump_str = "%sobjdump -wdr %s | gawk -M -v _funcs='%s' '"
4606                         "BEGIN { split(_funcs, funcs); }"
4607                         "/^$/ { func_match = 0; }"
4608                         "/<.*>:/ { "
4609                                 "f = gensub(/.*<(.*)>:/, \"\\\\1\", 1);"
4610                                 "for (i in funcs) {"
4611                                         "if (funcs[i] == f) {"
4612                                                 "func_match = 1;"
4613                                                 "base = strtonum(\"0x\" $1);"
4614                                                 "break;"
4615                                         "}"
4616                                 "}"
4617                         "}"
4618                         "{"
4619                                 "if (func_match) {"
4620                                         "addr = strtonum(\"0x\" $1);"
4621                                         "printf(\"%%04x \", addr - base);"
4622                                         "print;"
4623                                 "}"
4624                         "}' 1>&2";
4625
4626         /* fake snprintf() to calculate the size */
4627         size = snprintf(NULL, 0, objdump_str, cross_compile, objname, funcs) + 1;
4628         if (size <= 0) {
4629                 WARN("objdump string size calculation failed");
4630                 return -1;
4631         }
4632
4633         cmd = malloc(size);
4634
4635         /* real snprintf() */
4636         snprintf(cmd, size, objdump_str, cross_compile, objname, funcs);
4637         ret = system(cmd);
4638         if (ret) {
4639                 WARN("disassembly failed: %d", ret);
4640                 return -1;
4641         }
4642
4643         return 0;
4644 }
4645
4646 static int disas_warned_funcs(struct objtool_file *file)
4647 {
4648         struct symbol *sym;
4649         char *funcs = NULL, *tmp;
4650
4651         for_each_sym(file, sym) {
4652                 if (sym->warned) {
4653                         if (!funcs) {
4654                                 funcs = malloc(strlen(sym->name) + 1);
4655                                 strcpy(funcs, sym->name);
4656                         } else {
4657                                 tmp = malloc(strlen(funcs) + strlen(sym->name) + 2);
4658                                 sprintf(tmp, "%s %s", funcs, sym->name);
4659                                 free(funcs);
4660                                 funcs = tmp;
4661                         }
4662                 }
4663         }
4664
4665         if (funcs)
4666                 disas_funcs(funcs);
4667
4668         return 0;
4669 }
4670
4671 struct insn_chunk {
4672         void *addr;
4673         struct insn_chunk *next;
4674 };
4675
4676 /*
4677  * Reduce peak RSS usage by freeing insns memory before writing the ELF file,
4678  * which can trigger more allocations for .debug_* sections whose data hasn't
4679  * been read yet.
4680  */
4681 static void free_insns(struct objtool_file *file)
4682 {
4683         struct instruction *insn;
4684         struct insn_chunk *chunks = NULL, *chunk;
4685
4686         for_each_insn(file, insn) {
4687                 if (!insn->idx) {
4688                         chunk = malloc(sizeof(*chunk));
4689                         chunk->addr = insn;
4690                         chunk->next = chunks;
4691                         chunks = chunk;
4692                 }
4693         }
4694
4695         for (chunk = chunks; chunk; chunk = chunk->next)
4696                 free(chunk->addr);
4697 }
4698
4699 int check(struct objtool_file *file)
4700 {
4701         int ret, warnings = 0;
4702
4703         arch_initial_func_cfi_state(&initial_func_cfi);
4704         init_cfi_state(&init_cfi);
4705         init_cfi_state(&func_cfi);
4706         set_func_state(&func_cfi);
4707         init_cfi_state(&force_undefined_cfi);
4708         force_undefined_cfi.force_undefined = true;
4709
4710         if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3)))
4711                 goto out;
4712
4713         cfi_hash_add(&init_cfi);
4714         cfi_hash_add(&func_cfi);
4715
4716         ret = decode_sections(file);
4717         if (ret < 0)
4718                 goto out;
4719
4720         warnings += ret;
4721
4722         if (!nr_insns)
4723                 goto out;
4724
4725         if (opts.retpoline) {
4726                 ret = validate_retpoline(file);
4727                 if (ret < 0)
4728                         return ret;
4729                 warnings += ret;
4730         }
4731
4732         if (opts.stackval || opts.orc || opts.uaccess) {
4733                 ret = validate_functions(file);
4734                 if (ret < 0)
4735                         goto out;
4736                 warnings += ret;
4737
4738                 ret = validate_unwind_hints(file, NULL);
4739                 if (ret < 0)
4740                         goto out;
4741                 warnings += ret;
4742
4743                 if (!warnings) {
4744                         ret = validate_reachable_instructions(file);
4745                         if (ret < 0)
4746                                 goto out;
4747                         warnings += ret;
4748                 }
4749
4750         } else if (opts.noinstr) {
4751                 ret = validate_noinstr_sections(file);
4752                 if (ret < 0)
4753                         goto out;
4754                 warnings += ret;
4755         }
4756
4757         if (opts.unret) {
4758                 /*
4759                  * Must be after validate_branch() and friends, it plays
4760                  * further games with insn->visited.
4761                  */
4762                 ret = validate_unrets(file);
4763                 if (ret < 0)
4764                         return ret;
4765                 warnings += ret;
4766         }
4767
4768         if (opts.ibt) {
4769                 ret = validate_ibt(file);
4770                 if (ret < 0)
4771                         goto out;
4772                 warnings += ret;
4773         }
4774
4775         if (opts.sls) {
4776                 ret = validate_sls(file);
4777                 if (ret < 0)
4778                         goto out;
4779                 warnings += ret;
4780         }
4781
4782         if (opts.static_call) {
4783                 ret = create_static_call_sections(file);
4784                 if (ret < 0)
4785                         goto out;
4786                 warnings += ret;
4787         }
4788
4789         if (opts.retpoline) {
4790                 ret = create_retpoline_sites_sections(file);
4791                 if (ret < 0)
4792                         goto out;
4793                 warnings += ret;
4794         }
4795
4796         if (opts.cfi) {
4797                 ret = create_cfi_sections(file);
4798                 if (ret < 0)
4799                         goto out;
4800                 warnings += ret;
4801         }
4802
4803         if (opts.rethunk) {
4804                 ret = create_return_sites_sections(file);
4805                 if (ret < 0)
4806                         goto out;
4807                 warnings += ret;
4808
4809                 if (opts.hack_skylake) {
4810                         ret = create_direct_call_sections(file);
4811                         if (ret < 0)
4812                                 goto out;
4813                         warnings += ret;
4814                 }
4815         }
4816
4817         if (opts.mcount) {
4818                 ret = create_mcount_loc_sections(file);
4819                 if (ret < 0)
4820                         goto out;
4821                 warnings += ret;
4822         }
4823
4824         if (opts.prefix) {
4825                 ret = add_prefix_symbols(file);
4826                 if (ret < 0)
4827                         return ret;
4828                 warnings += ret;
4829         }
4830
4831         if (opts.ibt) {
4832                 ret = create_ibt_endbr_seal_sections(file);
4833                 if (ret < 0)
4834                         goto out;
4835                 warnings += ret;
4836         }
4837
4838         if (opts.orc && nr_insns) {
4839                 ret = orc_create(file);
4840                 if (ret < 0)
4841                         goto out;
4842                 warnings += ret;
4843         }
4844
4845         free_insns(file);
4846
4847         if (opts.verbose)
4848                 disas_warned_funcs(file);
4849
4850         if (opts.stats) {
4851                 printf("nr_insns_visited: %ld\n", nr_insns_visited);
4852                 printf("nr_cfi: %ld\n", nr_cfi);
4853                 printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
4854                 printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
4855         }
4856
4857 out:
4858         /*
4859          *  For now, don't fail the kernel build on fatal warnings.  These
4860          *  errors are still fairly common due to the growing matrix of
4861          *  supported toolchains and their recent pace of change.
4862          */
4863         return 0;
4864 }