tools/lib/bpf/linker.c

   1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
   2 /*
   3  * BPF static linker
   4  *
   5  * Copyright (c) 2021 Facebook
   6  */
   7 #ifndef _GNU_SOURCE
   8 #define _GNU_SOURCE
   9 #endif
  10
  11 #include <stdbool.h>
  12 #include <stddef.h>
  13 #include <stdio.h>
  14 #include <stdlib.h>
  15 #include <string.h>
  16 #include <unistd.h>
  17 #include <errno.h>
  18 #include <linux/err.h>
  19 #include <linux/btf.h>
  20 #include <elf.h>
  21 #include <libelf.h>
  22 #include <fcntl.h>
  23 #include <sys/mman.h>
  24 #include "libbpf.h"
  25 #include "btf.h"
  26 #include "libbpf_internal.h"
  27 #include "strset.h"
  28 #include "str_error.h"
  29
  30 #define BTF_EXTERN_SEC ".extern"
  31
  32 struct src_sec {
  33         const char *sec_name;
  34         /* positional (not necessarily ELF) index in an array of sections */
  35         int id;
  36         /* positional (not necessarily ELF) index of a matching section in a final object file */
  37         int dst_id;
  38         /* section data offset in a matching output section */
  39         int dst_off;
  40         /* whether section is omitted from the final ELF file */
  41         bool skipped;
  42         /* whether section is an ephemeral section, not mapped to an ELF section */
  43         bool ephemeral;
  44
  45         /* ELF info */
  46         size_t sec_idx;
  47         Elf_Scn *scn;
  48         Elf64_Shdr *shdr;
  49         Elf_Data *data;
  50
  51         /* corresponding BTF DATASEC type ID */
  52         int sec_type_id;
  53 };
  54
  55 struct src_obj {
  56         const char *filename;
  57         int fd;
  58         Elf *elf;
  59         /* Section header strings section index */
  60         size_t shstrs_sec_idx;
  61         /* SYMTAB section index */
  62         size_t symtab_sec_idx;
  63
  64         struct btf *btf;
  65         struct btf_ext *btf_ext;
  66
  67         /* List of sections (including ephemeral). Slot zero is unused. */
  68         struct src_sec *secs;
  69         int sec_cnt;
  70
  71         /* mapping of symbol indices from src to dst ELF */
  72         int *sym_map;
  73         /* mapping from the src BTF type IDs to dst ones */
  74         int *btf_type_map;
  75 };
  76
  77 /* single .BTF.ext data section */
  78 struct btf_ext_sec_data {
  79         size_t rec_cnt;
  80         __u32 rec_sz;
  81         void *recs;
  82 };
  83
  84 struct glob_sym {
  85         /* ELF symbol index */
  86         int sym_idx;
  87         /* associated section id for .ksyms, .kconfig, etc, but not .extern */
  88         int sec_id;
  89         /* extern name offset in STRTAB */
  90         int name_off;
  91         /* optional associated BTF type ID */
  92         int btf_id;
  93         /* BTF type ID to which VAR/FUNC type is pointing to; used for
  94          * rewriting types when extern VAR/FUNC is resolved to a concrete
  95          * definition
  96          */
  97         int underlying_btf_id;
  98         /* sec_var index in the corresponding dst_sec, if exists */
  99         int var_idx;
 100
 101         /* extern or resolved/global symbol */
 102         bool is_extern;
 103         /* weak or strong symbol, never goes back from strong to weak */
 104         bool is_weak;
 105 };
 106
 107 struct dst_sec {
 108         char *sec_name;
 109         /* positional (not necessarily ELF) index in an array of sections */
 110         int id;
 111
 112         bool ephemeral;
 113
 114         /* ELF info */
 115         size_t sec_idx;
 116         Elf_Scn *scn;
 117         Elf64_Shdr *shdr;
 118         Elf_Data *data;
 119
 120         /* final output section size */
 121         int sec_sz;
 122         /* final output contents of the section */
 123         void *raw_data;
 124
 125         /* corresponding STT_SECTION symbol index in SYMTAB */
 126         int sec_sym_idx;
 127
 128         /* section's DATASEC variable info, emitted on BTF finalization */
 129         bool has_btf;
 130         int sec_var_cnt;
 131         struct btf_var_secinfo *sec_vars;
 132
 133         /* section's .BTF.ext data */
 134         struct btf_ext_sec_data func_info;
 135         struct btf_ext_sec_data line_info;
 136         struct btf_ext_sec_data core_relo_info;
 137 };
 138
 139 struct bpf_linker {
 140         char *filename;
 141         int fd;
 142         Elf *elf;
 143         Elf64_Ehdr *elf_hdr;
 144         bool swapped_endian;
 145
 146         /* Output sections metadata */
 147         struct dst_sec *secs;
 148         int sec_cnt;
 149
 150         struct strset *strtab_strs; /* STRTAB unique strings */
 151         size_t strtab_sec_idx; /* STRTAB section index */
 152         size_t symtab_sec_idx; /* SYMTAB section index */
 153
 154         struct btf *btf;
 155         struct btf_ext *btf_ext;
 156
 157         /* global (including extern) ELF symbols */
 158         int glob_sym_cnt;
 159         struct glob_sym *glob_syms;
 160
 161         bool fd_is_owned;
 162 };
 163
 164 #define pr_warn_elf(fmt, ...)                                                                   \
 165         libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1))
 166
 167 static int init_output_elf(struct bpf_linker *linker);
 168
 169 static int bpf_linker_add_file(struct bpf_linker *linker, int fd,
 170                                const char *filename);
 171
 172 static int linker_load_obj_file(struct bpf_linker *linker,
 173                                 struct src_obj *obj);
 174 static int linker_sanity_check_elf(struct src_obj *obj);
 175 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec);
 176 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec);
 177 static int linker_sanity_check_btf(struct src_obj *obj);
 178 static int linker_sanity_check_btf_ext(struct src_obj *obj);
 179 static int linker_fixup_btf(struct src_obj *obj);
 180 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj);
 181 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj);
 182 static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
 183                                  Elf64_Sym *sym, const char *sym_name, int src_sym_idx);
 184 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj);
 185 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj);
 186 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj);
 187
 188 static int finalize_btf(struct bpf_linker *linker);
 189 static int finalize_btf_ext(struct bpf_linker *linker);
 190
 191 void bpf_linker__free(struct bpf_linker *linker)
 192 {
 193         int i;
 194
 195         if (!linker)
 196                 return;
 197
 198         free(linker->filename);
 199
 200         if (linker->elf)
 201                 elf_end(linker->elf);
 202
 203         if (linker->fd >= 0 && linker->fd_is_owned)
 204                 close(linker->fd);
 205
 206         strset__free(linker->strtab_strs);
 207
 208         btf__free(linker->btf);
 209         btf_ext__free(linker->btf_ext);
 210
 211         for (i = 1; i < linker->sec_cnt; i++) {
 212                 struct dst_sec *sec = &linker->secs[i];
 213
 214                 free(sec->sec_name);
 215                 free(sec->raw_data);
 216                 free(sec->sec_vars);
 217
 218                 free(sec->func_info.recs);
 219                 free(sec->line_info.recs);
 220                 free(sec->core_relo_info.recs);
 221         }
 222         free(linker->secs);
 223
 224         free(linker->glob_syms);
 225         free(linker);
 226 }
 227
 228 struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts)
 229 {
 230         struct bpf_linker *linker;
 231         int err;
 232
 233         if (!OPTS_VALID(opts, bpf_linker_opts))
 234                 return errno = EINVAL, NULL;
 235
 236         if (elf_version(EV_CURRENT) == EV_NONE) {
 237                 pr_warn_elf("libelf initialization failed");
 238                 return errno = EINVAL, NULL;
 239         }
 240
 241         linker = calloc(1, sizeof(*linker));
 242         if (!linker)
 243                 return errno = ENOMEM, NULL;
 244
 245         linker->filename = strdup(filename);
 246         if (!linker->filename) {
 247                 err = -ENOMEM;
 248                 goto err_out;
 249         }
 250
 251         linker->fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);
 252         if (linker->fd < 0) {
 253                 err = -errno;
 254                 pr_warn("failed to create '%s': %d\n", filename, err);
 255                 goto err_out;
 256         }
 257         linker->fd_is_owned = true;
 258
 259         err = init_output_elf(linker);
 260         if (err)
 261                 goto err_out;
 262
 263         return linker;
 264
 265 err_out:
 266         bpf_linker__free(linker);
 267         return errno = -err, NULL;
 268 }
 269
 270 struct bpf_linker *bpf_linker__new_fd(int fd, struct bpf_linker_opts *opts)
 271 {
 272         struct bpf_linker *linker;
 273         char filename[32];
 274         int err;
 275
 276         if (fd < 0)
 277                 return errno = EINVAL, NULL;
 278
 279         if (!OPTS_VALID(opts, bpf_linker_opts))
 280                 return errno = EINVAL, NULL;
 281
 282         if (elf_version(EV_CURRENT) == EV_NONE) {
 283                 pr_warn_elf("libelf initialization failed");
 284                 return errno = EINVAL, NULL;
 285         }
 286
 287         linker = calloc(1, sizeof(*linker));
 288         if (!linker)
 289                 return errno = ENOMEM, NULL;
 290
 291         snprintf(filename, sizeof(filename), "fd:%d", fd);
 292         linker->filename = strdup(filename);
 293         if (!linker->filename) {
 294                 err = -ENOMEM;
 295                 goto err_out;
 296         }
 297
 298         linker->fd = fd;
 299         linker->fd_is_owned = false;
 300
 301         err = init_output_elf(linker);
 302         if (err)
 303                 goto err_out;
 304
 305         return linker;
 306
 307 err_out:
 308         bpf_linker__free(linker);
 309         return errno = -err, NULL;
 310 }
 311
 312 static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name)
 313 {
 314         struct dst_sec *secs = linker->secs, *sec;
 315         size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2;
 316
 317         secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
 318         if (!secs)
 319                 return NULL;
 320
 321         /* zero out newly allocated memory */
 322         memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs));
 323
 324         linker->secs = secs;
 325         linker->sec_cnt = new_cnt;
 326
 327         sec = &linker->secs[new_cnt - 1];
 328         sec->id = new_cnt - 1;
 329         sec->sec_name = strdup(sec_name);
 330         if (!sec->sec_name)
 331                 return NULL;
 332
 333         return sec;
 334 }
 335
 336 static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx)
 337 {
 338         struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
 339         Elf64_Sym *syms, *sym;
 340         size_t sym_cnt = symtab->sec_sz / sizeof(*sym);
 341
 342         syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym));
 343         if (!syms)
 344                 return NULL;
 345
 346         sym = &syms[sym_cnt];
 347         memset(sym, 0, sizeof(*sym));
 348
 349         symtab->raw_data = syms;
 350         symtab->sec_sz += sizeof(*sym);
 351         symtab->shdr->sh_size += sizeof(*sym);
 352         symtab->data->d_size += sizeof(*sym);
 353
 354         if (sym_idx)
 355                 *sym_idx = sym_cnt;
 356
 357         return sym;
 358 }
 359
 360 static int init_output_elf(struct bpf_linker *linker)
 361 {
 362         int err, str_off;
 363         Elf64_Sym *init_sym;
 364         struct dst_sec *sec;
 365
 366         linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL);
 367         if (!linker->elf) {
 368                 pr_warn_elf("failed to create ELF object");
 369                 return -EINVAL;
 370         }
 371
 372         /* ELF header */
 373         linker->elf_hdr = elf64_newehdr(linker->elf);
 374         if (!linker->elf_hdr) {
 375                 pr_warn_elf("failed to create ELF header");
 376                 return -EINVAL;
 377         }
 378
 379         linker->elf_hdr->e_machine = EM_BPF;
 380         linker->elf_hdr->e_type = ET_REL;
 381         /* Set unknown ELF endianness, assign later from input files */
 382         linker->elf_hdr->e_ident[EI_DATA] = ELFDATANONE;
 383
 384         /* STRTAB */
 385         /* initialize strset with an empty string to conform to ELF */
 386         linker->strtab_strs = strset__new(INT_MAX, "", sizeof(""));
 387         if (libbpf_get_error(linker->strtab_strs))
 388                 return libbpf_get_error(linker->strtab_strs);
 389
 390         sec = add_dst_sec(linker, ".strtab");
 391         if (!sec)
 392                 return -ENOMEM;
 393
 394         sec->scn = elf_newscn(linker->elf);
 395         if (!sec->scn) {
 396                 pr_warn_elf("failed to create STRTAB section");
 397                 return -EINVAL;
 398         }
 399
 400         sec->shdr = elf64_getshdr(sec->scn);
 401         if (!sec->shdr)
 402                 return -EINVAL;
 403
 404         sec->data = elf_newdata(sec->scn);
 405         if (!sec->data) {
 406                 pr_warn_elf("failed to create STRTAB data");
 407                 return -EINVAL;
 408         }
 409
 410         str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
 411         if (str_off < 0)
 412                 return str_off;
 413
 414         sec->sec_idx = elf_ndxscn(sec->scn);
 415         linker->elf_hdr->e_shstrndx = sec->sec_idx;
 416         linker->strtab_sec_idx = sec->sec_idx;
 417
 418         sec->shdr->sh_name = str_off;
 419         sec->shdr->sh_type = SHT_STRTAB;
 420         sec->shdr->sh_flags = SHF_STRINGS;
 421         sec->shdr->sh_offset = 0;
 422         sec->shdr->sh_link = 0;
 423         sec->shdr->sh_info = 0;
 424         sec->shdr->sh_addralign = 1;
 425         sec->shdr->sh_size = sec->sec_sz = 0;
 426         sec->shdr->sh_entsize = 0;
 427
 428         /* SYMTAB */
 429         sec = add_dst_sec(linker, ".symtab");
 430         if (!sec)
 431                 return -ENOMEM;
 432
 433         sec->scn = elf_newscn(linker->elf);
 434         if (!sec->scn) {
 435                 pr_warn_elf("failed to create SYMTAB section");
 436                 return -EINVAL;
 437         }
 438
 439         sec->shdr = elf64_getshdr(sec->scn);
 440         if (!sec->shdr)
 441                 return -EINVAL;
 442
 443         sec->data = elf_newdata(sec->scn);
 444         if (!sec->data) {
 445                 pr_warn_elf("failed to create SYMTAB data");
 446                 return -EINVAL;
 447         }
 448         /* Ensure libelf translates byte-order of symbol records */
 449         sec->data->d_type = ELF_T_SYM;
 450
 451         str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
 452         if (str_off < 0)
 453                 return str_off;
 454
 455         sec->sec_idx = elf_ndxscn(sec->scn);
 456         linker->symtab_sec_idx = sec->sec_idx;
 457
 458         sec->shdr->sh_name = str_off;
 459         sec->shdr->sh_type = SHT_SYMTAB;
 460         sec->shdr->sh_flags = 0;
 461         sec->shdr->sh_offset = 0;
 462         sec->shdr->sh_link = linker->strtab_sec_idx;
 463         /* sh_info should be one greater than the index of the last local
 464          * symbol (i.e., binding is STB_LOCAL). But why and who cares?
 465          */
 466         sec->shdr->sh_info = 0;
 467         sec->shdr->sh_addralign = 8;
 468         sec->shdr->sh_entsize = sizeof(Elf64_Sym);
 469
 470         /* .BTF */
 471         linker->btf = btf__new_empty();
 472         err = libbpf_get_error(linker->btf);
 473         if (err)
 474                 return err;
 475
 476         /* add the special all-zero symbol */
 477         init_sym = add_new_sym(linker, NULL);
 478         if (!init_sym)
 479                 return -EINVAL;
 480
 481         init_sym->st_name = 0;
 482         init_sym->st_info = 0;
 483         init_sym->st_other = 0;
 484         init_sym->st_shndx = SHN_UNDEF;
 485         init_sym->st_value = 0;
 486         init_sym->st_size = 0;
 487
 488         return 0;
 489 }
 490
 491 static int bpf_linker_add_file(struct bpf_linker *linker, int fd,
 492                                const char *filename)
 493 {
 494         struct src_obj obj = {};
 495         int err = 0;
 496
 497         obj.filename = filename;
 498         obj.fd = fd;
 499
 500         err = err ?: linker_load_obj_file(linker, &obj);
 501         err = err ?: linker_append_sec_data(linker, &obj);
 502         err = err ?: linker_append_elf_syms(linker, &obj);
 503         err = err ?: linker_append_elf_relos(linker, &obj);
 504         err = err ?: linker_append_btf(linker, &obj);
 505         err = err ?: linker_append_btf_ext(linker, &obj);
 506
 507         /* free up src_obj resources */
 508         free(obj.btf_type_map);
 509         btf__free(obj.btf);
 510         btf_ext__free(obj.btf_ext);
 511         free(obj.secs);
 512         free(obj.sym_map);
 513         if (obj.elf)
 514                 elf_end(obj.elf);
 515
 516         return err;
 517 }
 518
 519 int bpf_linker__add_file(struct bpf_linker *linker, const char *filename,
 520                          const struct bpf_linker_file_opts *opts)
 521 {
 522         int fd, err;
 523
 524         if (!OPTS_VALID(opts, bpf_linker_file_opts))
 525                 return libbpf_err(-EINVAL);
 526
 527         if (!linker->elf)
 528                 return libbpf_err(-EINVAL);
 529
 530         fd = open(filename, O_RDONLY | O_CLOEXEC);
 531         if (fd < 0) {
 532                 err = -errno;
 533                 pr_warn("failed to open file '%s': %s\n", filename, errstr(err));
 534                 return libbpf_err(err);
 535         }
 536
 537         err = bpf_linker_add_file(linker, fd, filename);
 538         close(fd);
 539         return libbpf_err(err);
 540 }
 541
 542 int bpf_linker__add_fd(struct bpf_linker *linker, int fd,
 543                        const struct bpf_linker_file_opts *opts)
 544 {
 545         char filename[32];
 546         int err;
 547
 548         if (!OPTS_VALID(opts, bpf_linker_file_opts))
 549                 return libbpf_err(-EINVAL);
 550
 551         if (!linker->elf)
 552                 return libbpf_err(-EINVAL);
 553
 554         if (fd < 0)
 555                 return libbpf_err(-EINVAL);
 556
 557         snprintf(filename, sizeof(filename), "fd:%d", fd);
 558         err = bpf_linker_add_file(linker, fd, filename);
 559         return libbpf_err(err);
 560 }
 561
 562 int bpf_linker__add_buf(struct bpf_linker *linker, void *buf, size_t buf_sz,
 563                         const struct bpf_linker_file_opts *opts)
 564 {
 565         char filename[32];
 566         int fd, written, ret;
 567
 568         if (!OPTS_VALID(opts, bpf_linker_file_opts))
 569                 return libbpf_err(-EINVAL);
 570
 571         if (!linker->elf)
 572                 return libbpf_err(-EINVAL);
 573
 574         snprintf(filename, sizeof(filename), "mem:%p+%zu", buf, buf_sz);
 575
 576         fd = memfd_create(filename, 0);
 577         if (fd < 0) {
 578                 ret = -errno;
 579                 pr_warn("failed to create memfd '%s': %s\n", filename, errstr(ret));
 580                 return libbpf_err(ret);
 581         }
 582
 583         written = 0;
 584         while (written < buf_sz) {
 585                 ret = write(fd, buf, buf_sz);
 586                 if (ret < 0) {
 587                         ret = -errno;
 588                         pr_warn("failed to write '%s': %s\n", filename, errstr(ret));
 589                         goto err_out;
 590                 }
 591                 written += ret;
 592         }
 593
 594         ret = bpf_linker_add_file(linker, fd, filename);
 595 err_out:
 596         close(fd);
 597         return libbpf_err(ret);
 598 }
 599
 600 static bool is_dwarf_sec_name(const char *name)
 601 {
 602         /* approximation, but the actual list is too long */
 603         return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
 604 }
 605
 606 static bool is_ignored_sec(struct src_sec *sec)
 607 {
 608         Elf64_Shdr *shdr = sec->shdr;
 609         const char *name = sec->sec_name;
 610
 611         /* no special handling of .strtab */
 612         if (shdr->sh_type == SHT_STRTAB)
 613                 return true;
 614
 615         /* ignore .llvm_addrsig section as well */
 616         if (shdr->sh_type == SHT_LLVM_ADDRSIG)
 617                 return true;
 618
 619         /* no subprograms will lead to an empty .text section, ignore it */
 620         if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 &&
 621             strcmp(sec->sec_name, ".text") == 0)
 622                 return true;
 623
 624         /* DWARF sections */
 625         if (is_dwarf_sec_name(sec->sec_name))
 626                 return true;
 627
 628         if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
 629                 name += sizeof(".rel") - 1;
 630                 /* DWARF section relocations */
 631                 if (is_dwarf_sec_name(name))
 632                         return true;
 633
 634                 /* .BTF and .BTF.ext don't need relocations */
 635                 if (strcmp(name, BTF_ELF_SEC) == 0 ||
 636                     strcmp(name, BTF_EXT_ELF_SEC) == 0)
 637                         return true;
 638         }
 639
 640         return false;
 641 }
 642
 643 static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name)
 644 {
 645         struct src_sec *secs = obj->secs, *sec;
 646         size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2;
 647
 648         secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
 649         if (!secs)
 650                 return NULL;
 651
 652         /* zero out newly allocated memory */
 653         memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs));
 654
 655         obj->secs = secs;
 656         obj->sec_cnt = new_cnt;
 657
 658         sec = &obj->secs[new_cnt - 1];
 659         sec->id = new_cnt - 1;
 660         sec->sec_name = sec_name;
 661
 662         return sec;
 663 }
 664
 665 static int linker_load_obj_file(struct bpf_linker *linker,
 666                                 struct src_obj *obj)
 667 {
 668         int err = 0;
 669         Elf_Scn *scn;
 670         Elf_Data *data;
 671         Elf64_Ehdr *ehdr;
 672         Elf64_Shdr *shdr;
 673         struct src_sec *sec;
 674         unsigned char obj_byteorder;
 675         unsigned char link_byteorder = linker->elf_hdr->e_ident[EI_DATA];
 676 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 677         const unsigned char host_byteorder = ELFDATA2LSB;
 678 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
 679         const unsigned char host_byteorder = ELFDATA2MSB;
 680 #else
 681 #error "Unknown __BYTE_ORDER__"
 682 #endif
 683
 684         pr_debug("linker: adding object file '%s'...\n", obj->filename);
 685
 686         obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL);
 687         if (!obj->elf) {
 688                 pr_warn_elf("failed to parse ELF file '%s'", obj->filename);
 689                 return -EINVAL;
 690         }
 691
 692         /* Sanity check ELF file high-level properties */
 693         ehdr = elf64_getehdr(obj->elf);
 694         if (!ehdr) {
 695                 pr_warn_elf("failed to get ELF header for %s", obj->filename);
 696                 return -EINVAL;
 697         }
 698
 699         /* Linker output endianness set by first input object */
 700         obj_byteorder = ehdr->e_ident[EI_DATA];
 701         if (obj_byteorder != ELFDATA2LSB && obj_byteorder != ELFDATA2MSB) {
 702                 err = -EOPNOTSUPP;
 703                 pr_warn("unknown byte order of ELF file %s\n", obj->filename);
 704                 return err;
 705         }
 706         if (link_byteorder == ELFDATANONE) {
 707                 linker->elf_hdr->e_ident[EI_DATA] = obj_byteorder;
 708                 linker->swapped_endian = obj_byteorder != host_byteorder;
 709                 pr_debug("linker: set %s-endian output byte order\n",
 710                          obj_byteorder == ELFDATA2MSB ? "big" : "little");
 711         } else if (link_byteorder != obj_byteorder) {
 712                 err = -EOPNOTSUPP;
 713                 pr_warn("byte order mismatch with ELF file %s\n", obj->filename);
 714                 return err;
 715         }
 716
 717         if (ehdr->e_type != ET_REL
 718             || ehdr->e_machine != EM_BPF
 719             || ehdr->e_ident[EI_CLASS] != ELFCLASS64) {
 720                 err = -EOPNOTSUPP;
 721                 pr_warn_elf("unsupported kind of ELF file %s", obj->filename);
 722                 return err;
 723         }
 724
 725         if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) {
 726                 pr_warn_elf("failed to get SHSTRTAB section index for %s", obj->filename);
 727                 return -EINVAL;
 728         }
 729
 730         scn = NULL;
 731         while ((scn = elf_nextscn(obj->elf, scn)) != NULL) {
 732                 size_t sec_idx = elf_ndxscn(scn);
 733                 const char *sec_name;
 734
 735                 shdr = elf64_getshdr(scn);
 736                 if (!shdr) {
 737                         pr_warn_elf("failed to get section #%zu header for %s",
 738                                     sec_idx, obj->filename);
 739                         return -EINVAL;
 740                 }
 741
 742                 sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name);
 743                 if (!sec_name) {
 744                         pr_warn_elf("failed to get section #%zu name for %s",
 745                                     sec_idx, obj->filename);
 746                         return -EINVAL;
 747                 }
 748
 749                 data = elf_getdata(scn, 0);
 750                 if (!data) {
 751                         pr_warn_elf("failed to get section #%zu (%s) data from %s",
 752                                     sec_idx, sec_name, obj->filename);
 753                         return -EINVAL;
 754                 }
 755
 756                 sec = add_src_sec(obj, sec_name);
 757                 if (!sec)
 758                         return -ENOMEM;
 759
 760                 sec->scn = scn;
 761                 sec->shdr = shdr;
 762                 sec->data = data;
 763                 sec->sec_idx = elf_ndxscn(scn);
 764
 765                 if (is_ignored_sec(sec)) {
 766                         sec->skipped = true;
 767                         continue;
 768                 }
 769
 770                 switch (shdr->sh_type) {
 771                 case SHT_SYMTAB:
 772                         if (obj->symtab_sec_idx) {
 773                                 err = -EOPNOTSUPP;
 774                                 pr_warn("multiple SYMTAB sections found, not supported\n");
 775                                 return err;
 776                         }
 777                         obj->symtab_sec_idx = sec_idx;
 778                         break;
 779                 case SHT_STRTAB:
 780                         /* we'll construct our own string table */
 781                         break;
 782                 case SHT_PROGBITS:
 783                         if (strcmp(sec_name, BTF_ELF_SEC) == 0) {
 784                                 obj->btf = btf__new(data->d_buf, shdr->sh_size);
 785                                 err = libbpf_get_error(obj->btf);
 786                                 if (err) {
 787                                         pr_warn("failed to parse .BTF from %s: %s\n",
 788                                                 obj->filename, errstr(err));
 789                                         return err;
 790                                 }
 791                                 sec->skipped = true;
 792                                 continue;
 793                         }
 794                         if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) {
 795                                 obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size);
 796                                 err = libbpf_get_error(obj->btf_ext);
 797                                 if (err) {
 798                                         pr_warn("failed to parse .BTF.ext from '%s': %s\n",
 799                                                 obj->filename, errstr(err));
 800                                         return err;
 801                                 }
 802                                 sec->skipped = true;
 803                                 continue;
 804                         }
 805
 806                         /* data & code */
 807                         break;
 808                 case SHT_NOBITS:
 809                         /* BSS */
 810                         break;
 811                 case SHT_REL:
 812                         /* relocations */
 813                         break;
 814                 default:
 815                         pr_warn("unrecognized section #%zu (%s) in %s\n",
 816                                 sec_idx, sec_name, obj->filename);
 817                         err = -EINVAL;
 818                         return err;
 819                 }
 820         }
 821
 822         err = err ?: linker_sanity_check_elf(obj);
 823         err = err ?: linker_sanity_check_btf(obj);
 824         err = err ?: linker_sanity_check_btf_ext(obj);
 825         err = err ?: linker_fixup_btf(obj);
 826
 827         return err;
 828 }
 829
 830 static int linker_sanity_check_elf(struct src_obj *obj)
 831 {
 832         struct src_sec *sec;
 833         int i, err;
 834
 835         if (!obj->symtab_sec_idx) {
 836                 pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename);
 837                 return -EINVAL;
 838         }
 839         if (!obj->shstrs_sec_idx) {
 840                 pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename);
 841                 return -EINVAL;
 842         }
 843
 844         for (i = 1; i < obj->sec_cnt; i++) {
 845                 sec = &obj->secs[i];
 846
 847                 if (sec->sec_name[0] == '\0') {
 848                         pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename);
 849                         return -EINVAL;
 850                 }
 851
 852                 if (is_dwarf_sec_name(sec->sec_name))
 853                         continue;
 854
 855                 if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign)) {
 856                         pr_warn("ELF section #%zu alignment %llu is non pow-of-2 alignment in %s\n",
 857                                 sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign,
 858                                 obj->filename);
 859                         return -EINVAL;
 860                 }
 861                 if (sec->shdr->sh_addralign != sec->data->d_align) {
 862                         pr_warn("ELF section #%zu has inconsistent alignment addr=%llu != d=%llu in %s\n",
 863                                 sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign,
 864                                 (long long unsigned)sec->data->d_align, obj->filename);
 865                         return -EINVAL;
 866                 }
 867
 868                 if (sec->shdr->sh_size != sec->data->d_size) {
 869                         pr_warn("ELF section #%zu has inconsistent section size sh=%llu != d=%llu in %s\n",
 870                                 sec->sec_idx, (long long unsigned)sec->shdr->sh_size,
 871                                 (long long unsigned)sec->data->d_size, obj->filename);
 872                         return -EINVAL;
 873                 }
 874
 875                 switch (sec->shdr->sh_type) {
 876                 case SHT_SYMTAB:
 877                         err = linker_sanity_check_elf_symtab(obj, sec);
 878                         if (err)
 879                                 return err;
 880                         break;
 881                 case SHT_STRTAB:
 882                         break;
 883                 case SHT_PROGBITS:
 884                         if (sec->shdr->sh_flags & SHF_EXECINSTR) {
 885                                 if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0) {
 886                                         pr_warn("ELF section #%zu has unexpected size alignment %llu in %s\n",
 887                                                 sec->sec_idx, (long long unsigned)sec->shdr->sh_size,
 888                                                 obj->filename);
 889                                         return -EINVAL;
 890                                 }
 891                         }
 892                         break;
 893                 case SHT_NOBITS:
 894                         break;
 895                 case SHT_REL:
 896                         err = linker_sanity_check_elf_relos(obj, sec);
 897                         if (err)
 898                                 return err;
 899                         break;
 900                 case SHT_LLVM_ADDRSIG:
 901                         break;
 902                 default:
 903                         pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n",
 904                                 sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename);
 905                         return -EINVAL;
 906                 }
 907         }
 908
 909         return 0;
 910 }
 911
 912 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec)
 913 {
 914         struct src_sec *link_sec;
 915         Elf64_Sym *sym;
 916         int i, n;
 917
 918         if (sec->shdr->sh_entsize != sizeof(Elf64_Sym))
 919                 return -EINVAL;
 920         if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
 921                 return -EINVAL;
 922
 923         if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) {
 924                 pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n",
 925                         sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
 926                 return -EINVAL;
 927         }
 928         link_sec = &obj->secs[sec->shdr->sh_link];
 929         if (link_sec->shdr->sh_type != SHT_STRTAB) {
 930                 pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n",
 931                         sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
 932                 return -EINVAL;
 933         }
 934
 935         n = sec->shdr->sh_size / sec->shdr->sh_entsize;
 936         sym = sec->data->d_buf;
 937         for (i = 0; i < n; i++, sym++) {
 938                 int sym_type = ELF64_ST_TYPE(sym->st_info);
 939                 int sym_bind = ELF64_ST_BIND(sym->st_info);
 940                 int sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
 941
 942                 if (i == 0) {
 943                         if (sym->st_name != 0 || sym->st_info != 0
 944                             || sym->st_other != 0 || sym->st_shndx != 0
 945                             || sym->st_value != 0 || sym->st_size != 0) {
 946                                 pr_warn("ELF sym #0 is invalid in %s\n", obj->filename);
 947                                 return -EINVAL;
 948                         }
 949                         continue;
 950                 }
 951                 if (sym_bind != STB_LOCAL && sym_bind != STB_GLOBAL && sym_bind != STB_WEAK) {
 952                         pr_warn("ELF sym #%d in section #%zu has unsupported symbol binding %d\n",
 953                                 i, sec->sec_idx, sym_bind);
 954                         return -EINVAL;
 955                 }
 956                 if (sym_vis != STV_DEFAULT && sym_vis != STV_HIDDEN) {
 957                         pr_warn("ELF sym #%d in section #%zu has unsupported symbol visibility %d\n",
 958                                 i, sec->sec_idx, sym_vis);
 959                         return -EINVAL;
 960                 }
 961                 if (sym->st_shndx == 0) {
 962                         if (sym_type != STT_NOTYPE || sym_bind == STB_LOCAL
 963                             || sym->st_value != 0 || sym->st_size != 0) {
 964                                 pr_warn("ELF sym #%d is invalid extern symbol in %s\n",
 965                                         i, obj->filename);
 966
 967                                 return -EINVAL;
 968                         }
 969                         continue;
 970                 }
 971                 if (sym->st_shndx < SHN_LORESERVE && sym->st_shndx >= obj->sec_cnt) {
 972                         pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n",
 973                                 i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename);
 974                         return -EINVAL;
 975                 }
 976                 if (sym_type == STT_SECTION) {
 977                         if (sym->st_value != 0)
 978                                 return -EINVAL;
 979                         continue;
 980                 }
 981         }
 982
 983         return 0;
 984 }
 985
 986 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec)
 987 {
 988         struct src_sec *link_sec, *sym_sec;
 989         Elf64_Rel *relo;
 990         int i, n;
 991
 992         if (sec->shdr->sh_entsize != sizeof(Elf64_Rel))
 993                 return -EINVAL;
 994         if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
 995                 return -EINVAL;
 996
 997         /* SHT_REL's sh_link should point to SYMTAB */
 998         if (sec->shdr->sh_link != obj->symtab_sec_idx) {
 999                 pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n",
1000                         sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
1001                 return -EINVAL;
1002         }
1003
1004         /* SHT_REL's sh_info points to relocated section */
1005         if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) {
1006                 pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n",
1007                         sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
1008                 return -EINVAL;
1009         }
1010         link_sec = &obj->secs[sec->shdr->sh_info];
1011
1012         /* .rel<secname> -> <secname> pattern is followed */
1013         if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0
1014             || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) {
1015                 pr_warn("ELF relo section #%zu name has invalid name in %s\n",
1016                         sec->sec_idx, obj->filename);
1017                 return -EINVAL;
1018         }
1019
1020         /* don't further validate relocations for ignored sections */
1021         if (link_sec->skipped)
1022                 return 0;
1023
1024         /* relocatable section is data or instructions */
1025         if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) {
1026                 pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n",
1027                         sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
1028                 return -EINVAL;
1029         }
1030
1031         /* check sanity of each relocation */
1032         n = sec->shdr->sh_size / sec->shdr->sh_entsize;
1033         relo = sec->data->d_buf;
1034         sym_sec = &obj->secs[obj->symtab_sec_idx];
1035         for (i = 0; i < n; i++, relo++) {
1036                 size_t sym_idx = ELF64_R_SYM(relo->r_info);
1037                 size_t sym_type = ELF64_R_TYPE(relo->r_info);
1038
1039                 if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32 &&
1040                     sym_type != R_BPF_64_ABS64 && sym_type != R_BPF_64_ABS32) {
1041                         pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n",
1042                                 i, sec->sec_idx, sym_type, obj->filename);
1043                         return -EINVAL;
1044                 }
1045
1046                 if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) {
1047                         pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n",
1048                                 i, sec->sec_idx, sym_idx, obj->filename);
1049                         return -EINVAL;
1050                 }
1051
1052                 if (link_sec->shdr->sh_flags & SHF_EXECINSTR) {
1053                         if (relo->r_offset % sizeof(struct bpf_insn) != 0) {
1054                                 pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n",
1055                                         i, sec->sec_idx, sym_idx, obj->filename);
1056                                 return -EINVAL;
1057                         }
1058                 }
1059         }
1060
1061         return 0;
1062 }
1063
1064 static int check_btf_type_id(__u32 *type_id, void *ctx)
1065 {
1066         struct btf *btf = ctx;
1067
1068         if (*type_id >= btf__type_cnt(btf))
1069                 return -EINVAL;
1070
1071         return 0;
1072 }
1073
1074 static int check_btf_str_off(__u32 *str_off, void *ctx)
1075 {
1076         struct btf *btf = ctx;
1077         const char *s;
1078
1079         s = btf__str_by_offset(btf, *str_off);
1080
1081         if (!s)
1082                 return -EINVAL;
1083
1084         return 0;
1085 }
1086
1087 static int linker_sanity_check_btf(struct src_obj *obj)
1088 {
1089         struct btf_type *t;
1090         int i, n, err;
1091
1092         if (!obj->btf)
1093                 return 0;
1094
1095         n = btf__type_cnt(obj->btf);
1096         for (i = 1; i < n; i++) {
1097                 struct btf_field_iter it;
1098                 __u32 *type_id, *str_off;
1099
1100                 t = btf_type_by_id(obj->btf, i);
1101
1102                 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS);
1103                 if (err)
1104                         return err;
1105                 while ((type_id = btf_field_iter_next(&it))) {
1106                         if (*type_id >= n)
1107                                 return -EINVAL;
1108                 }
1109
1110                 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS);
1111                 if (err)
1112                         return err;
1113                 while ((str_off = btf_field_iter_next(&it))) {
1114                         if (!btf__str_by_offset(obj->btf, *str_off))
1115                                 return -EINVAL;
1116                 }
1117         }
1118
1119         return 0;
1120 }
1121
1122 static int linker_sanity_check_btf_ext(struct src_obj *obj)
1123 {
1124         int err = 0;
1125
1126         if (!obj->btf_ext)
1127                 return 0;
1128
1129         /* can't use .BTF.ext without .BTF */
1130         if (!obj->btf)
1131                 return -EINVAL;
1132
1133         err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf);
1134         err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf);
1135         if (err)
1136                 return err;
1137
1138         return 0;
1139 }
1140
1141 static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec)
1142 {
1143         Elf_Scn *scn;
1144         Elf_Data *data;
1145         Elf64_Shdr *shdr;
1146         int name_off;
1147
1148         dst_sec->sec_sz = 0;
1149         dst_sec->sec_idx = 0;
1150         dst_sec->ephemeral = src_sec->ephemeral;
1151
1152         /* ephemeral sections are just thin section shells lacking most parts */
1153         if (src_sec->ephemeral)
1154                 return 0;
1155
1156         scn = elf_newscn(linker->elf);
1157         if (!scn)
1158                 return -ENOMEM;
1159         data = elf_newdata(scn);
1160         if (!data)
1161                 return -ENOMEM;
1162         shdr = elf64_getshdr(scn);
1163         if (!shdr)
1164                 return -ENOMEM;
1165
1166         dst_sec->scn = scn;
1167         dst_sec->shdr = shdr;
1168         dst_sec->data = data;
1169         dst_sec->sec_idx = elf_ndxscn(scn);
1170
1171         name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name);
1172         if (name_off < 0)
1173                 return name_off;
1174
1175         shdr->sh_name = name_off;
1176         shdr->sh_type = src_sec->shdr->sh_type;
1177         shdr->sh_flags = src_sec->shdr->sh_flags;
1178         shdr->sh_size = 0;
1179         /* sh_link and sh_info have different meaning for different types of
1180          * sections, so we leave it up to the caller code to fill them in, if
1181          * necessary
1182          */
1183         shdr->sh_link = 0;
1184         shdr->sh_info = 0;
1185         shdr->sh_addralign = src_sec->shdr->sh_addralign;
1186         shdr->sh_entsize = src_sec->shdr->sh_entsize;
1187
1188         data->d_type = src_sec->data->d_type;
1189         data->d_size = 0;
1190         data->d_buf = NULL;
1191         data->d_align = src_sec->data->d_align;
1192         data->d_off = 0;
1193
1194         return 0;
1195 }
1196
1197 static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name)
1198 {
1199         struct dst_sec *sec;
1200         int i;
1201
1202         for (i = 1; i < linker->sec_cnt; i++) {
1203                 sec = &linker->secs[i];
1204
1205                 if (strcmp(sec->sec_name, sec_name) == 0)
1206                         return sec;
1207         }
1208
1209         return NULL;
1210 }
1211
1212 static bool secs_match(struct dst_sec *dst, struct src_sec *src)
1213 {
1214         if (dst->ephemeral || src->ephemeral)
1215                 return true;
1216
1217         if (dst->shdr->sh_type != src->shdr->sh_type) {
1218                 pr_warn("sec %s types mismatch\n", dst->sec_name);
1219                 return false;
1220         }
1221         if (dst->shdr->sh_flags != src->shdr->sh_flags) {
1222                 pr_warn("sec %s flags mismatch\n", dst->sec_name);
1223                 return false;
1224         }
1225         if (dst->shdr->sh_entsize != src->shdr->sh_entsize) {
1226                 pr_warn("sec %s entsize mismatch\n", dst->sec_name);
1227                 return false;
1228         }
1229
1230         return true;
1231 }
1232
1233 static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec)
1234 {
1235         if (dst_sec->sec_sz != src_sec->shdr->sh_size)
1236                 return false;
1237         if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0)
1238                 return false;
1239         return true;
1240 }
1241
1242 static bool is_exec_sec(struct dst_sec *sec)
1243 {
1244         if (!sec || sec->ephemeral)
1245                 return false;
1246         return (sec->shdr->sh_type == SHT_PROGBITS) &&
1247                (sec->shdr->sh_flags & SHF_EXECINSTR);
1248 }
1249
1250 static void exec_sec_bswap(void *raw_data, int size)
1251 {
1252         const int insn_cnt = size / sizeof(struct bpf_insn);
1253         struct bpf_insn *insn = raw_data;
1254         int i;
1255
1256         for (i = 0; i < insn_cnt; i++, insn++)
1257                 bpf_insn_bswap(insn);
1258 }
1259
1260 static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src_sec *src)
1261 {
1262         void *tmp;
1263         size_t dst_align, src_align;
1264         size_t dst_align_sz, dst_final_sz;
1265         int err;
1266
1267         /* Ephemeral source section doesn't contribute anything to ELF
1268          * section data.
1269          */
1270         if (src->ephemeral)
1271                 return 0;
1272
1273         /* Some sections (like .maps) can contain both externs (and thus be
1274          * ephemeral) and non-externs (map definitions). So it's possible that
1275          * it has to be "upgraded" from ephemeral to non-ephemeral when the
1276          * first non-ephemeral entity appears. In such case, we add ELF
1277          * section, data, etc.
1278          */
1279         if (dst->ephemeral) {
1280                 err = init_sec(linker, dst, src);
1281                 if (err)
1282                         return err;
1283         }
1284
1285         dst_align = dst->shdr->sh_addralign;
1286         src_align = src->shdr->sh_addralign;
1287         if (dst_align == 0)
1288                 dst_align = 1;
1289         if (dst_align < src_align)
1290                 dst_align = src_align;
1291
1292         dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align;
1293
1294         /* no need to re-align final size */
1295         dst_final_sz = dst_align_sz + src->shdr->sh_size;
1296
1297         if (src->shdr->sh_type != SHT_NOBITS) {
1298                 tmp = realloc(dst->raw_data, dst_final_sz);
1299                 /* If dst_align_sz == 0, realloc() behaves in a special way:
1300                  * 1. When dst->raw_data is NULL it returns:
1301                  *    "either NULL or a pointer suitable to be passed to free()" [1].
1302                  * 2. When dst->raw_data is not-NULL it frees dst->raw_data and returns NULL,
1303                  *    thus invalidating any "pointer suitable to be passed to free()" obtained
1304                  *    at step (1).
1305                  *
1306                  * The dst_align_sz > 0 check avoids error exit after (2), otherwise
1307                  * dst->raw_data would be freed again in bpf_linker__free().
1308                  *
1309                  * [1] man 3 realloc
1310                  */
1311                 if (!tmp && dst_align_sz > 0)
1312                         return -ENOMEM;
1313                 dst->raw_data = tmp;
1314
1315                 /* pad dst section, if it's alignment forced size increase */
1316                 memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz);
1317                 /* now copy src data at a properly aligned offset */
1318                 memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size);
1319
1320                 /* convert added bpf insns to native byte-order */
1321                 if (linker->swapped_endian && is_exec_sec(dst))
1322                         exec_sec_bswap(dst->raw_data + dst_align_sz, src->shdr->sh_size);
1323         }
1324
1325         dst->sec_sz = dst_final_sz;
1326         dst->shdr->sh_size = dst_final_sz;
1327         dst->data->d_size = dst_final_sz;
1328
1329         dst->shdr->sh_addralign = dst_align;
1330         dst->data->d_align = dst_align;
1331
1332         src->dst_off = dst_align_sz;
1333
1334         return 0;
1335 }
1336
1337 static bool is_data_sec(struct src_sec *sec)
1338 {
1339         if (!sec || sec->skipped)
1340                 return false;
1341         /* ephemeral sections are data sections, e.g., .kconfig, .ksyms */
1342         if (sec->ephemeral)
1343                 return true;
1344         return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS;
1345 }
1346
1347 static bool is_relo_sec(struct src_sec *sec)
1348 {
1349         if (!sec || sec->skipped || sec->ephemeral)
1350                 return false;
1351         return sec->shdr->sh_type == SHT_REL;
1352 }
1353
1354 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj)
1355 {
1356         int i, err;
1357
1358         for (i = 1; i < obj->sec_cnt; i++) {
1359                 struct src_sec *src_sec;
1360                 struct dst_sec *dst_sec;
1361
1362                 src_sec = &obj->secs[i];
1363                 if (!is_data_sec(src_sec))
1364                         continue;
1365
1366                 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
1367                 if (!dst_sec) {
1368                         dst_sec = add_dst_sec(linker, src_sec->sec_name);
1369                         if (!dst_sec)
1370                                 return -ENOMEM;
1371                         err = init_sec(linker, dst_sec, src_sec);
1372                         if (err) {
1373                                 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
1374                                 return err;
1375                         }
1376                 } else {
1377                         if (!secs_match(dst_sec, src_sec)) {
1378                                 pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name);
1379                                 return -1;
1380                         }
1381
1382                         /* "license" and "version" sections are deduped */
1383                         if (strcmp(src_sec->sec_name, "license") == 0
1384                             || strcmp(src_sec->sec_name, "version") == 0) {
1385                                 if (!sec_content_is_same(dst_sec, src_sec)) {
1386                                         pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name);
1387                                         return -EINVAL;
1388                                 }
1389                                 src_sec->skipped = true;
1390                                 src_sec->dst_id = dst_sec->id;
1391                                 continue;
1392                         }
1393                 }
1394
1395                 /* record mapped section index */
1396                 src_sec->dst_id = dst_sec->id;
1397
1398                 err = extend_sec(linker, dst_sec, src_sec);
1399                 if (err)
1400                         return err;
1401         }
1402
1403         return 0;
1404 }
1405
1406 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj)
1407 {
1408         struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
1409         Elf64_Sym *sym = symtab->data->d_buf;
1410         int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize, err;
1411         int str_sec_idx = symtab->shdr->sh_link;
1412         const char *sym_name;
1413
1414         obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map));
1415         if (!obj->sym_map)
1416                 return -ENOMEM;
1417
1418         for (i = 0; i < n; i++, sym++) {
1419                 /* We already validated all-zero symbol #0 and we already
1420                  * appended it preventively to the final SYMTAB, so skip it.
1421                  */
1422                 if (i == 0)
1423                         continue;
1424
1425                 sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
1426                 if (!sym_name) {
1427                         pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename);
1428                         return -EINVAL;
1429                 }
1430
1431                 err = linker_append_elf_sym(linker, obj, sym, sym_name, i);
1432                 if (err)
1433                         return err;
1434         }
1435
1436         return 0;
1437 }
1438
1439 static Elf64_Sym *get_sym_by_idx(struct bpf_linker *linker, size_t sym_idx)
1440 {
1441         struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
1442         Elf64_Sym *syms = symtab->raw_data;
1443
1444         return &syms[sym_idx];
1445 }
1446
1447 static struct glob_sym *find_glob_sym(struct bpf_linker *linker, const char *sym_name)
1448 {
1449         struct glob_sym *glob_sym;
1450         const char *name;
1451         int i;
1452
1453         for (i = 0; i < linker->glob_sym_cnt; i++) {
1454                 glob_sym = &linker->glob_syms[i];
1455                 name = strset__data(linker->strtab_strs) + glob_sym->name_off;
1456
1457                 if (strcmp(name, sym_name) == 0)
1458                         return glob_sym;
1459         }
1460
1461         return NULL;
1462 }
1463
1464 static struct glob_sym *add_glob_sym(struct bpf_linker *linker)
1465 {
1466         struct glob_sym *syms, *sym;
1467
1468         syms = libbpf_reallocarray(linker->glob_syms, linker->glob_sym_cnt + 1,
1469                                    sizeof(*linker->glob_syms));
1470         if (!syms)
1471                 return NULL;
1472
1473         sym = &syms[linker->glob_sym_cnt];
1474         memset(sym, 0, sizeof(*sym));
1475         sym->var_idx = -1;
1476
1477         linker->glob_syms = syms;
1478         linker->glob_sym_cnt++;
1479
1480         return sym;
1481 }
1482
1483 static bool glob_sym_btf_matches(const char *sym_name, bool exact,
1484                                  const struct btf *btf1, __u32 id1,
1485                                  const struct btf *btf2, __u32 id2)
1486 {
1487         const struct btf_type *t1, *t2;
1488         bool is_static1, is_static2;
1489         const char *n1, *n2;
1490         int i, n;
1491
1492 recur:
1493         n1 = n2 = NULL;
1494         t1 = skip_mods_and_typedefs(btf1, id1, &id1);
1495         t2 = skip_mods_and_typedefs(btf2, id2, &id2);
1496
1497         /* check if only one side is FWD, otherwise handle with common logic */
1498         if (!exact && btf_is_fwd(t1) != btf_is_fwd(t2)) {
1499                 n1 = btf__str_by_offset(btf1, t1->name_off);
1500                 n2 = btf__str_by_offset(btf2, t2->name_off);
1501                 if (strcmp(n1, n2) != 0) {
1502                         pr_warn("global '%s': incompatible forward declaration names '%s' and '%s'\n",
1503                                 sym_name, n1, n2);
1504                         return false;
1505                 }
1506                 /* validate if FWD kind matches concrete kind */
1507                 if (btf_is_fwd(t1)) {
1508                         if (btf_kflag(t1) && btf_is_union(t2))
1509                                 return true;
1510                         if (!btf_kflag(t1) && btf_is_struct(t2))
1511                                 return true;
1512                         pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1513                                 sym_name, btf_kflag(t1) ? "union" : "struct", btf_kind_str(t2));
1514                 } else {
1515                         if (btf_kflag(t2) && btf_is_union(t1))
1516                                 return true;
1517                         if (!btf_kflag(t2) && btf_is_struct(t1))
1518                                 return true;
1519                         pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1520                                 sym_name, btf_kflag(t2) ? "union" : "struct", btf_kind_str(t1));
1521                 }
1522                 return false;
1523         }
1524
1525         if (btf_kind(t1) != btf_kind(t2)) {
1526                 pr_warn("global '%s': incompatible BTF kinds %s and %s\n",
1527                         sym_name, btf_kind_str(t1), btf_kind_str(t2));
1528                 return false;
1529         }
1530
1531         switch (btf_kind(t1)) {
1532         case BTF_KIND_STRUCT:
1533         case BTF_KIND_UNION:
1534         case BTF_KIND_ENUM:
1535         case BTF_KIND_ENUM64:
1536         case BTF_KIND_FWD:
1537         case BTF_KIND_FUNC:
1538         case BTF_KIND_VAR:
1539                 n1 = btf__str_by_offset(btf1, t1->name_off);
1540                 n2 = btf__str_by_offset(btf2, t2->name_off);
1541                 if (strcmp(n1, n2) != 0) {
1542                         pr_warn("global '%s': incompatible %s names '%s' and '%s'\n",
1543                                 sym_name, btf_kind_str(t1), n1, n2);
1544                         return false;
1545                 }
1546                 break;
1547         default:
1548                 break;
1549         }
1550
1551         switch (btf_kind(t1)) {
1552         case BTF_KIND_UNKN: /* void */
1553         case BTF_KIND_FWD:
1554                 return true;
1555         case BTF_KIND_INT:
1556         case BTF_KIND_FLOAT:
1557         case BTF_KIND_ENUM:
1558         case BTF_KIND_ENUM64:
1559                 /* ignore encoding for int and enum values for enum */
1560                 if (t1->size != t2->size) {
1561                         pr_warn("global '%s': incompatible %s '%s' size %u and %u\n",
1562                                 sym_name, btf_kind_str(t1), n1, t1->size, t2->size);
1563                         return false;
1564                 }
1565                 return true;
1566         case BTF_KIND_PTR:
1567                 /* just validate overall shape of the referenced type, so no
1568                  * contents comparison for struct/union, and allowed fwd vs
1569                  * struct/union
1570                  */
1571                 exact = false;
1572                 id1 = t1->type;
1573                 id2 = t2->type;
1574                 goto recur;
1575         case BTF_KIND_ARRAY:
1576                 /* ignore index type and array size */
1577                 id1 = btf_array(t1)->type;
1578                 id2 = btf_array(t2)->type;
1579                 goto recur;
1580         case BTF_KIND_FUNC:
1581                 /* extern and global linkages are compatible */
1582                 is_static1 = btf_func_linkage(t1) == BTF_FUNC_STATIC;
1583                 is_static2 = btf_func_linkage(t2) == BTF_FUNC_STATIC;
1584                 if (is_static1 != is_static2) {
1585                         pr_warn("global '%s': incompatible func '%s' linkage\n", sym_name, n1);
1586                         return false;
1587                 }
1588
1589                 id1 = t1->type;
1590                 id2 = t2->type;
1591                 goto recur;
1592         case BTF_KIND_VAR:
1593                 /* extern and global linkages are compatible */
1594                 is_static1 = btf_var(t1)->linkage == BTF_VAR_STATIC;
1595                 is_static2 = btf_var(t2)->linkage == BTF_VAR_STATIC;
1596                 if (is_static1 != is_static2) {
1597                         pr_warn("global '%s': incompatible var '%s' linkage\n", sym_name, n1);
1598                         return false;
1599                 }
1600
1601                 id1 = t1->type;
1602                 id2 = t2->type;
1603                 goto recur;
1604         case BTF_KIND_STRUCT:
1605         case BTF_KIND_UNION: {
1606                 const struct btf_member *m1, *m2;
1607
1608                 if (!exact)
1609                         return true;
1610
1611                 if (btf_vlen(t1) != btf_vlen(t2)) {
1612                         pr_warn("global '%s': incompatible number of %s fields %u and %u\n",
1613                                 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1614                         return false;
1615                 }
1616
1617                 n = btf_vlen(t1);
1618                 m1 = btf_members(t1);
1619                 m2 = btf_members(t2);
1620                 for (i = 0; i < n; i++, m1++, m2++) {
1621                         n1 = btf__str_by_offset(btf1, m1->name_off);
1622                         n2 = btf__str_by_offset(btf2, m2->name_off);
1623                         if (strcmp(n1, n2) != 0) {
1624                                 pr_warn("global '%s': incompatible field #%d names '%s' and '%s'\n",
1625                                         sym_name, i, n1, n2);
1626                                 return false;
1627                         }
1628                         if (m1->offset != m2->offset) {
1629                                 pr_warn("global '%s': incompatible field #%d ('%s') offsets\n",
1630                                         sym_name, i, n1);
1631                                 return false;
1632                         }
1633                         if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1634                                 return false;
1635                 }
1636
1637                 return true;
1638         }
1639         case BTF_KIND_FUNC_PROTO: {
1640                 const struct btf_param *m1, *m2;
1641
1642                 if (btf_vlen(t1) != btf_vlen(t2)) {
1643                         pr_warn("global '%s': incompatible number of %s params %u and %u\n",
1644                                 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1645                         return false;
1646                 }
1647
1648                 n = btf_vlen(t1);
1649                 m1 = btf_params(t1);
1650                 m2 = btf_params(t2);
1651                 for (i = 0; i < n; i++, m1++, m2++) {
1652                         /* ignore func arg names */
1653                         if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1654                                 return false;
1655                 }
1656
1657                 /* now check return type as well */
1658                 id1 = t1->type;
1659                 id2 = t2->type;
1660                 goto recur;
1661         }
1662
1663         /* skip_mods_and_typedefs() make this impossible */
1664         case BTF_KIND_TYPEDEF:
1665         case BTF_KIND_VOLATILE:
1666         case BTF_KIND_CONST:
1667         case BTF_KIND_RESTRICT:
1668         /* DATASECs are never compared with each other */
1669         case BTF_KIND_DATASEC:
1670         default:
1671                 pr_warn("global '%s': unsupported BTF kind %s\n",
1672                         sym_name, btf_kind_str(t1));
1673                 return false;
1674         }
1675 }
1676
1677 static bool map_defs_match(const char *sym_name,
1678                            const struct btf *main_btf,
1679                            const struct btf_map_def *main_def,
1680                            const struct btf_map_def *main_inner_def,
1681                            const struct btf *extra_btf,
1682                            const struct btf_map_def *extra_def,
1683                            const struct btf_map_def *extra_inner_def)
1684 {
1685         const char *reason;
1686
1687         if (main_def->map_type != extra_def->map_type) {
1688                 reason = "type";
1689                 goto mismatch;
1690         }
1691
1692         /* check key type/size match */
1693         if (main_def->key_size != extra_def->key_size) {
1694                 reason = "key_size";
1695                 goto mismatch;
1696         }
1697         if (!!main_def->key_type_id != !!extra_def->key_type_id) {
1698                 reason = "key type";
1699                 goto mismatch;
1700         }
1701         if ((main_def->parts & MAP_DEF_KEY_TYPE)
1702              && !glob_sym_btf_matches(sym_name, true /*exact*/,
1703                                       main_btf, main_def->key_type_id,
1704                                       extra_btf, extra_def->key_type_id)) {
1705                 reason = "key type";
1706                 goto mismatch;
1707         }
1708
1709         /* validate value type/size match */
1710         if (main_def->value_size != extra_def->value_size) {
1711                 reason = "value_size";
1712                 goto mismatch;
1713         }
1714         if (!!main_def->value_type_id != !!extra_def->value_type_id) {
1715                 reason = "value type";
1716                 goto mismatch;
1717         }
1718         if ((main_def->parts & MAP_DEF_VALUE_TYPE)
1719              && !glob_sym_btf_matches(sym_name, true /*exact*/,
1720                                       main_btf, main_def->value_type_id,
1721                                       extra_btf, extra_def->value_type_id)) {
1722                 reason = "key type";
1723                 goto mismatch;
1724         }
1725
1726         if (main_def->max_entries != extra_def->max_entries) {
1727                 reason = "max_entries";
1728                 goto mismatch;
1729         }
1730         if (main_def->map_flags != extra_def->map_flags) {
1731                 reason = "map_flags";
1732                 goto mismatch;
1733         }
1734         if (main_def->numa_node != extra_def->numa_node) {
1735                 reason = "numa_node";
1736                 goto mismatch;
1737         }
1738         if (main_def->pinning != extra_def->pinning) {
1739                 reason = "pinning";
1740                 goto mismatch;
1741         }
1742
1743         if ((main_def->parts & MAP_DEF_INNER_MAP) != (extra_def->parts & MAP_DEF_INNER_MAP)) {
1744                 reason = "inner map";
1745                 goto mismatch;
1746         }
1747
1748         if (main_def->parts & MAP_DEF_INNER_MAP) {
1749                 char inner_map_name[128];
1750
1751                 snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", sym_name);
1752
1753                 return map_defs_match(inner_map_name,
1754                                       main_btf, main_inner_def, NULL,
1755                                       extra_btf, extra_inner_def, NULL);
1756         }
1757
1758         return true;
1759
1760 mismatch:
1761         pr_warn("global '%s': map %s mismatch\n", sym_name, reason);
1762         return false;
1763 }
1764
1765 static bool glob_map_defs_match(const char *sym_name,
1766                                 struct bpf_linker *linker, struct glob_sym *glob_sym,
1767                                 struct src_obj *obj, Elf64_Sym *sym, int btf_id)
1768 {
1769         struct btf_map_def dst_def = {}, dst_inner_def = {};
1770         struct btf_map_def src_def = {}, src_inner_def = {};
1771         const struct btf_type *t;
1772         int err;
1773
1774         t = btf__type_by_id(obj->btf, btf_id);
1775         if (!btf_is_var(t)) {
1776                 pr_warn("global '%s': invalid map definition type [%d]\n", sym_name, btf_id);
1777                 return false;
1778         }
1779         t = skip_mods_and_typedefs(obj->btf, t->type, NULL);
1780
1781         err = parse_btf_map_def(sym_name, obj->btf, t, true /*strict*/, &src_def, &src_inner_def);
1782         if (err) {
1783                 pr_warn("global '%s': invalid map definition\n", sym_name);
1784                 return false;
1785         }
1786
1787         /* re-parse existing map definition */
1788         t = btf__type_by_id(linker->btf, glob_sym->btf_id);
1789         t = skip_mods_and_typedefs(linker->btf, t->type, NULL);
1790         err = parse_btf_map_def(sym_name, linker->btf, t, true /*strict*/, &dst_def, &dst_inner_def);
1791         if (err) {
1792                 /* this should not happen, because we already validated it */
1793                 pr_warn("global '%s': invalid dst map definition\n", sym_name);
1794                 return false;
1795         }
1796
1797         /* Currently extern map definition has to be complete and match
1798          * concrete map definition exactly. This restriction might be lifted
1799          * in the future.
1800          */
1801         return map_defs_match(sym_name, linker->btf, &dst_def, &dst_inner_def,
1802                               obj->btf, &src_def, &src_inner_def);
1803 }
1804
1805 static bool glob_syms_match(const char *sym_name,
1806                             struct bpf_linker *linker, struct glob_sym *glob_sym,
1807                             struct src_obj *obj, Elf64_Sym *sym, size_t sym_idx, int btf_id)
1808 {
1809         const struct btf_type *src_t;
1810
1811         /* if we are dealing with externs, BTF types describing both global
1812          * and extern VARs/FUNCs should be completely present in all files
1813          */
1814         if (!glob_sym->btf_id || !btf_id) {
1815                 pr_warn("BTF info is missing for global symbol '%s'\n", sym_name);
1816                 return false;
1817         }
1818
1819         src_t = btf__type_by_id(obj->btf, btf_id);
1820         if (!btf_is_var(src_t) && !btf_is_func(src_t)) {
1821                 pr_warn("only extern variables and functions are supported, but got '%s' for '%s'\n",
1822                         btf_kind_str(src_t), sym_name);
1823                 return false;
1824         }
1825
1826         /* deal with .maps definitions specially */
1827         if (glob_sym->sec_id && strcmp(linker->secs[glob_sym->sec_id].sec_name, MAPS_ELF_SEC) == 0)
1828                 return glob_map_defs_match(sym_name, linker, glob_sym, obj, sym, btf_id);
1829
1830         if (!glob_sym_btf_matches(sym_name, true /*exact*/,
1831                                   linker->btf, glob_sym->btf_id, obj->btf, btf_id))
1832                 return false;
1833
1834         return true;
1835 }
1836
1837 static bool btf_is_non_static(const struct btf_type *t)
1838 {
1839         return (btf_is_var(t) && btf_var(t)->linkage != BTF_VAR_STATIC)
1840                || (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_STATIC);
1841 }
1842
1843 static int find_glob_sym_btf(struct src_obj *obj, Elf64_Sym *sym, const char *sym_name,
1844                              int *out_btf_sec_id, int *out_btf_id)
1845 {
1846         int i, j, n, m, btf_id = 0;
1847         const struct btf_type *t;
1848         const struct btf_var_secinfo *vi;
1849         const char *name;
1850
1851         if (!obj->btf) {
1852                 pr_warn("failed to find BTF info for object '%s'\n", obj->filename);
1853                 return -EINVAL;
1854         }
1855
1856         n = btf__type_cnt(obj->btf);
1857         for (i = 1; i < n; i++) {
1858                 t = btf__type_by_id(obj->btf, i);
1859
1860                 /* some global and extern FUNCs and VARs might not be associated with any
1861                  * DATASEC, so try to detect them in the same pass
1862                  */
1863                 if (btf_is_non_static(t)) {
1864                         name = btf__str_by_offset(obj->btf, t->name_off);
1865                         if (strcmp(name, sym_name) != 0)
1866                                 continue;
1867
1868                         /* remember and still try to find DATASEC */
1869                         btf_id = i;
1870                         continue;
1871                 }
1872
1873                 if (!btf_is_datasec(t))
1874                         continue;
1875
1876                 vi = btf_var_secinfos(t);
1877                 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
1878                         t = btf__type_by_id(obj->btf, vi->type);
1879                         name = btf__str_by_offset(obj->btf, t->name_off);
1880
1881                         if (strcmp(name, sym_name) != 0)
1882                                 continue;
1883                         if (btf_is_var(t) && btf_var(t)->linkage == BTF_VAR_STATIC)
1884                                 continue;
1885                         if (btf_is_func(t) && btf_func_linkage(t) == BTF_FUNC_STATIC)
1886                                 continue;
1887
1888                         if (btf_id && btf_id != vi->type) {
1889                                 pr_warn("global/extern '%s' BTF is ambiguous: both types #%d and #%u match\n",
1890                                         sym_name, btf_id, vi->type);
1891                                 return -EINVAL;
1892                         }
1893
1894                         *out_btf_sec_id = i;
1895                         *out_btf_id = vi->type;
1896
1897                         return 0;
1898                 }
1899         }
1900
1901         /* free-floating extern or global FUNC */
1902         if (btf_id) {
1903                 *out_btf_sec_id = 0;
1904                 *out_btf_id = btf_id;
1905                 return 0;
1906         }
1907
1908         pr_warn("failed to find BTF info for global/extern symbol '%s'\n", sym_name);
1909         return -ENOENT;
1910 }
1911
1912 static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name)
1913 {
1914         struct src_sec *sec;
1915         int i;
1916
1917         for (i = 1; i < obj->sec_cnt; i++) {
1918                 sec = &obj->secs[i];
1919
1920                 if (strcmp(sec->sec_name, sec_name) == 0)
1921                         return sec;
1922         }
1923
1924         return NULL;
1925 }
1926
1927 static int complete_extern_btf_info(struct btf *dst_btf, int dst_id,
1928                                     struct btf *src_btf, int src_id)
1929 {
1930         struct btf_type *dst_t = btf_type_by_id(dst_btf, dst_id);
1931         struct btf_type *src_t = btf_type_by_id(src_btf, src_id);
1932         struct btf_param *src_p, *dst_p;
1933         const char *s;
1934         int i, n, off;
1935
1936         /* We already made sure that source and destination types (FUNC or
1937          * VAR) match in terms of types and argument names.
1938          */
1939         if (btf_is_var(dst_t)) {
1940                 btf_var(dst_t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
1941                 return 0;
1942         }
1943
1944         dst_t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_GLOBAL, 0);
1945
1946         /* now onto FUNC_PROTO types */
1947         src_t = btf_type_by_id(src_btf, src_t->type);
1948         dst_t = btf_type_by_id(dst_btf, dst_t->type);
1949
1950         /* Fill in all the argument names, which for extern FUNCs are missing.
1951          * We'll end up with two copies of FUNCs/VARs for externs, but that
1952          * will be taken care of by BTF dedup at the very end.
1953          * It might be that BTF types for extern in one file has less/more BTF
1954          * information (e.g., FWD instead of full STRUCT/UNION information),
1955          * but that should be (in most cases, subject to BTF dedup rules)
1956          * handled and resolved by BTF dedup algorithm as well, so we won't
1957          * worry about it. Our only job is to make sure that argument names
1958          * are populated on both sides, otherwise BTF dedup will pedantically
1959          * consider them different.
1960          */
1961         src_p = btf_params(src_t);
1962         dst_p = btf_params(dst_t);
1963         for (i = 0, n = btf_vlen(dst_t); i < n; i++, src_p++, dst_p++) {
1964                 if (!src_p->name_off)
1965                         continue;
1966
1967                 /* src_btf has more complete info, so add name to dst_btf */
1968                 s = btf__str_by_offset(src_btf, src_p->name_off);
1969                 off = btf__add_str(dst_btf, s);
1970                 if (off < 0)
1971                         return off;
1972                 dst_p->name_off = off;
1973         }
1974         return 0;
1975 }
1976
1977 static void sym_update_bind(Elf64_Sym *sym, int sym_bind)
1978 {
1979         sym->st_info = ELF64_ST_INFO(sym_bind, ELF64_ST_TYPE(sym->st_info));
1980 }
1981
1982 static void sym_update_type(Elf64_Sym *sym, int sym_type)
1983 {
1984         sym->st_info = ELF64_ST_INFO(ELF64_ST_BIND(sym->st_info), sym_type);
1985 }
1986
1987 static void sym_update_visibility(Elf64_Sym *sym, int sym_vis)
1988 {
1989         /* libelf doesn't provide setters for ST_VISIBILITY,
1990          * but it is stored in the lower 2 bits of st_other
1991          */
1992         sym->st_other &= ~0x03;
1993         sym->st_other |= sym_vis;
1994 }
1995
1996 static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
1997                                  Elf64_Sym *sym, const char *sym_name, int src_sym_idx)
1998 {
1999         struct src_sec *src_sec = NULL;
2000         struct dst_sec *dst_sec = NULL;
2001         struct glob_sym *glob_sym = NULL;
2002         int name_off, sym_type, sym_bind, sym_vis, err;
2003         int btf_sec_id = 0, btf_id = 0;
2004         size_t dst_sym_idx;
2005         Elf64_Sym *dst_sym;
2006         bool sym_is_extern;
2007
2008         sym_type = ELF64_ST_TYPE(sym->st_info);
2009         sym_bind = ELF64_ST_BIND(sym->st_info);
2010         sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
2011         sym_is_extern = sym->st_shndx == SHN_UNDEF;
2012
2013         if (sym_is_extern) {
2014                 if (!obj->btf) {
2015                         pr_warn("externs without BTF info are not supported\n");
2016                         return -ENOTSUP;
2017                 }
2018         } else if (sym->st_shndx < SHN_LORESERVE) {
2019                 src_sec = &obj->secs[sym->st_shndx];
2020                 if (src_sec->skipped)
2021                         return 0;
2022                 dst_sec = &linker->secs[src_sec->dst_id];
2023
2024                 /* allow only one STT_SECTION symbol per section */
2025                 if (sym_type == STT_SECTION && dst_sec->sec_sym_idx) {
2026                         obj->sym_map[src_sym_idx] = dst_sec->sec_sym_idx;
2027                         return 0;
2028                 }
2029         }
2030
2031         if (sym_bind == STB_LOCAL)
2032                 goto add_sym;
2033
2034         /* find matching BTF info */
2035         err = find_glob_sym_btf(obj, sym, sym_name, &btf_sec_id, &btf_id);
2036         if (err)
2037                 return err;
2038
2039         if (sym_is_extern && btf_sec_id) {
2040                 const char *sec_name = NULL;
2041                 const struct btf_type *t;
2042
2043                 t = btf__type_by_id(obj->btf, btf_sec_id);
2044                 sec_name = btf__str_by_offset(obj->btf, t->name_off);
2045
2046                 /* Clang puts unannotated extern vars into
2047                  * '.extern' BTF DATASEC. Treat them the same
2048                  * as unannotated extern funcs (which are
2049                  * currently not put into any DATASECs).
2050                  * Those don't have associated src_sec/dst_sec.
2051                  */
2052                 if (strcmp(sec_name, BTF_EXTERN_SEC) != 0) {
2053                         src_sec = find_src_sec_by_name(obj, sec_name);
2054                         if (!src_sec) {
2055                                 pr_warn("failed to find matching ELF sec '%s'\n", sec_name);
2056                                 return -ENOENT;
2057                         }
2058                         dst_sec = &linker->secs[src_sec->dst_id];
2059                 }
2060         }
2061
2062         glob_sym = find_glob_sym(linker, sym_name);
2063         if (glob_sym) {
2064                 /* Preventively resolve to existing symbol. This is
2065                  * needed for further relocation symbol remapping in
2066                  * the next step of linking.
2067                  */
2068                 obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
2069
2070                 /* If both symbols are non-externs, at least one of
2071                  * them has to be STB_WEAK, otherwise they are in
2072                  * a conflict with each other.
2073                  */
2074                 if (!sym_is_extern && !glob_sym->is_extern
2075                     && !glob_sym->is_weak && sym_bind != STB_WEAK) {
2076                         pr_warn("conflicting non-weak symbol #%d (%s) definition in '%s'\n",
2077                                 src_sym_idx, sym_name, obj->filename);
2078                         return -EINVAL;
2079                 }
2080
2081                 if (!glob_syms_match(sym_name, linker, glob_sym, obj, sym, src_sym_idx, btf_id))
2082                         return -EINVAL;
2083
2084                 dst_sym = get_sym_by_idx(linker, glob_sym->sym_idx);
2085
2086                 /* If new symbol is strong, then force dst_sym to be strong as
2087                  * well; this way a mix of weak and non-weak extern
2088                  * definitions will end up being strong.
2089                  */
2090                 if (sym_bind == STB_GLOBAL) {
2091                         /* We still need to preserve type (NOTYPE or
2092                          * OBJECT/FUNC, depending on whether the symbol is
2093                          * extern or not)
2094                          */
2095                         sym_update_bind(dst_sym, STB_GLOBAL);
2096                         glob_sym->is_weak = false;
2097                 }
2098
2099                 /* Non-default visibility is "contaminating", with stricter
2100                  * visibility overwriting more permissive ones, even if more
2101                  * permissive visibility comes from just an extern definition.
2102                  * Currently only STV_DEFAULT and STV_HIDDEN are allowed and
2103                  * ensured by ELF symbol sanity checks above.
2104                  */
2105                 if (sym_vis > ELF64_ST_VISIBILITY(dst_sym->st_other))
2106                         sym_update_visibility(dst_sym, sym_vis);
2107
2108                 /* If the new symbol is extern, then regardless if
2109                  * existing symbol is extern or resolved global, just
2110                  * keep the existing one untouched.
2111                  */
2112                 if (sym_is_extern)
2113                         return 0;
2114
2115                 /* If existing symbol is a strong resolved symbol, bail out,
2116                  * because we lost resolution battle have nothing to
2117                  * contribute. We already checked above that there is no
2118                  * strong-strong conflict. We also already tightened binding
2119                  * and visibility, so nothing else to contribute at that point.
2120                  */
2121                 if (!glob_sym->is_extern && sym_bind == STB_WEAK)
2122                         return 0;
2123
2124                 /* At this point, new symbol is strong non-extern,
2125                  * so overwrite glob_sym with new symbol information.
2126                  * Preserve binding and visibility.
2127                  */
2128                 sym_update_type(dst_sym, sym_type);
2129                 dst_sym->st_shndx = dst_sec->sec_idx;
2130                 dst_sym->st_value = src_sec->dst_off + sym->st_value;
2131                 dst_sym->st_size = sym->st_size;
2132
2133                 /* see comment below about dst_sec->id vs dst_sec->sec_idx */
2134                 glob_sym->sec_id = dst_sec->id;
2135                 glob_sym->is_extern = false;
2136
2137                 if (complete_extern_btf_info(linker->btf, glob_sym->btf_id,
2138                                              obj->btf, btf_id))
2139                         return -EINVAL;
2140
2141                 /* request updating VAR's/FUNC's underlying BTF type when appending BTF type */
2142                 glob_sym->underlying_btf_id = 0;
2143
2144                 obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
2145                 return 0;
2146         }
2147
2148 add_sym:
2149         name_off = strset__add_str(linker->strtab_strs, sym_name);
2150         if (name_off < 0)
2151                 return name_off;
2152
2153         dst_sym = add_new_sym(linker, &dst_sym_idx);
2154         if (!dst_sym)
2155                 return -ENOMEM;
2156
2157         dst_sym->st_name = name_off;
2158         dst_sym->st_info = sym->st_info;
2159         dst_sym->st_other = sym->st_other;
2160         dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx;
2161         dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value;
2162         dst_sym->st_size = sym->st_size;
2163
2164         obj->sym_map[src_sym_idx] = dst_sym_idx;
2165
2166         if (sym_type == STT_SECTION && dst_sym) {
2167                 dst_sec->sec_sym_idx = dst_sym_idx;
2168                 dst_sym->st_value = 0;
2169         }
2170
2171         if (sym_bind != STB_LOCAL) {
2172                 glob_sym = add_glob_sym(linker);
2173                 if (!glob_sym)
2174                         return -ENOMEM;
2175
2176                 glob_sym->sym_idx = dst_sym_idx;
2177                 /* we use dst_sec->id (and not dst_sec->sec_idx), because
2178                  * ephemeral sections (.kconfig, .ksyms, etc) don't have
2179                  * sec_idx (as they don't have corresponding ELF section), but
2180                  * still have id. .extern doesn't have even ephemeral section
2181                  * associated with it, so dst_sec->id == dst_sec->sec_idx == 0.
2182                  */
2183                 glob_sym->sec_id = dst_sec ? dst_sec->id : 0;
2184                 glob_sym->name_off = name_off;
2185                 /* we will fill btf_id in during BTF merging step */
2186                 glob_sym->btf_id = 0;
2187                 glob_sym->is_extern = sym_is_extern;
2188                 glob_sym->is_weak = sym_bind == STB_WEAK;
2189         }
2190
2191         return 0;
2192 }
2193
2194 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj)
2195 {
2196         struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx];
2197         int i, err;
2198
2199         for (i = 1; i < obj->sec_cnt; i++) {
2200                 struct src_sec *src_sec, *src_linked_sec;
2201                 struct dst_sec *dst_sec, *dst_linked_sec;
2202                 Elf64_Rel *src_rel, *dst_rel;
2203                 int j, n;
2204
2205                 src_sec = &obj->secs[i];
2206                 if (!is_relo_sec(src_sec))
2207                         continue;
2208
2209                 /* shdr->sh_info points to relocatable section */
2210                 src_linked_sec = &obj->secs[src_sec->shdr->sh_info];
2211                 if (src_linked_sec->skipped)
2212                         continue;
2213
2214                 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
2215                 if (!dst_sec) {
2216                         dst_sec = add_dst_sec(linker, src_sec->sec_name);
2217                         if (!dst_sec)
2218                                 return -ENOMEM;
2219                         err = init_sec(linker, dst_sec, src_sec);
2220                         if (err) {
2221                                 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
2222                                 return err;
2223                         }
2224                 } else if (!secs_match(dst_sec, src_sec)) {
2225                         pr_warn("sections %s are not compatible\n", src_sec->sec_name);
2226                         return -1;
2227                 }
2228
2229                 /* shdr->sh_link points to SYMTAB */
2230                 dst_sec->shdr->sh_link = linker->symtab_sec_idx;
2231
2232                 /* shdr->sh_info points to relocated section */
2233                 dst_linked_sec = &linker->secs[src_linked_sec->dst_id];
2234                 dst_sec->shdr->sh_info = dst_linked_sec->sec_idx;
2235
2236                 src_sec->dst_id = dst_sec->id;
2237                 err = extend_sec(linker, dst_sec, src_sec);
2238                 if (err)
2239                         return err;
2240
2241                 src_rel = src_sec->data->d_buf;
2242                 dst_rel = dst_sec->raw_data + src_sec->dst_off;
2243                 n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize;
2244                 for (j = 0; j < n; j++, src_rel++, dst_rel++) {
2245                         size_t src_sym_idx, dst_sym_idx, sym_type;
2246                         Elf64_Sym *src_sym;
2247
2248                         src_sym_idx = ELF64_R_SYM(src_rel->r_info);
2249                         src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx;
2250
2251                         dst_sym_idx = obj->sym_map[src_sym_idx];
2252                         dst_rel->r_offset += src_linked_sec->dst_off;
2253                         sym_type = ELF64_R_TYPE(src_rel->r_info);
2254                         dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type);
2255
2256                         if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) {
2257                                 struct src_sec *sec = &obj->secs[src_sym->st_shndx];
2258                                 struct bpf_insn *insn;
2259
2260                                 if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) {
2261                                         /* calls to the very first static function inside
2262                                          * .text section at offset 0 will
2263                                          * reference section symbol, not the
2264                                          * function symbol. Fix that up,
2265                                          * otherwise it won't be possible to
2266                                          * relocate calls to two different
2267                                          * static functions with the same name
2268                                          * (rom two different object files)
2269                                          */
2270                                         insn = dst_linked_sec->raw_data + dst_rel->r_offset;
2271                                         if (insn->code == (BPF_JMP | BPF_CALL))
2272                                                 insn->imm += sec->dst_off / sizeof(struct bpf_insn);
2273                                         else
2274                                                 insn->imm += sec->dst_off;
2275                                 } else {
2276                                         pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n");
2277                                         return -EINVAL;
2278                                 }
2279                         }
2280
2281                 }
2282         }
2283
2284         return 0;
2285 }
2286
2287 static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx,
2288                                    int sym_type, const char *sym_name)
2289 {
2290         struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
2291         Elf64_Sym *sym = symtab->data->d_buf;
2292         int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize;
2293         int str_sec_idx = symtab->shdr->sh_link;
2294         const char *name;
2295
2296         for (i = 0; i < n; i++, sym++) {
2297                 if (sym->st_shndx != sec_idx)
2298                         continue;
2299                 if (ELF64_ST_TYPE(sym->st_info) != sym_type)
2300                         continue;
2301
2302                 name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
2303                 if (!name)
2304                         return NULL;
2305
2306                 if (strcmp(sym_name, name) != 0)
2307                         continue;
2308
2309                 return sym;
2310         }
2311
2312         return NULL;
2313 }
2314
2315 static int linker_fixup_btf(struct src_obj *obj)
2316 {
2317         const char *sec_name;
2318         struct src_sec *sec;
2319         int i, j, n, m;
2320
2321         if (!obj->btf)
2322                 return 0;
2323
2324         n = btf__type_cnt(obj->btf);
2325         for (i = 1; i < n; i++) {
2326                 struct btf_var_secinfo *vi;
2327                 struct btf_type *t;
2328
2329                 t = btf_type_by_id(obj->btf, i);
2330                 if (btf_kind(t) != BTF_KIND_DATASEC)
2331                         continue;
2332
2333                 sec_name = btf__str_by_offset(obj->btf, t->name_off);
2334                 sec = find_src_sec_by_name(obj, sec_name);
2335                 if (sec) {
2336                         /* record actual section size, unless ephemeral */
2337                         if (sec->shdr)
2338                                 t->size = sec->shdr->sh_size;
2339                 } else {
2340                         /* BTF can have some sections that are not represented
2341                          * in ELF, e.g., .kconfig, .ksyms, .extern, which are used
2342                          * for special extern variables.
2343                          *
2344                          * For all but one such special (ephemeral)
2345                          * sections, we pre-create "section shells" to be able
2346                          * to keep track of extra per-section metadata later
2347                          * (e.g., those BTF extern variables).
2348                          *
2349                          * .extern is even more special, though, because it
2350                          * contains extern variables that need to be resolved
2351                          * by static linker, not libbpf and kernel. When such
2352                          * externs are resolved, we are going to remove them
2353                          * from .extern BTF section and might end up not
2354                          * needing it at all. Each resolved extern should have
2355                          * matching non-extern VAR/FUNC in other sections.
2356                          *
2357                          * We do support leaving some of the externs
2358                          * unresolved, though, to support cases of building
2359                          * libraries, which will later be linked against final
2360                          * BPF applications. So if at finalization we still
2361                          * see unresolved externs, we'll create .extern
2362                          * section on our own.
2363                          */
2364                         if (strcmp(sec_name, BTF_EXTERN_SEC) == 0)
2365                                 continue;
2366
2367                         sec = add_src_sec(obj, sec_name);
2368                         if (!sec)
2369                                 return -ENOMEM;
2370
2371                         sec->ephemeral = true;
2372                         sec->sec_idx = 0; /* will match UNDEF shndx in ELF */
2373                 }
2374
2375                 /* remember ELF section and its BTF type ID match */
2376                 sec->sec_type_id = i;
2377
2378                 /* fix up variable offsets */
2379                 vi = btf_var_secinfos(t);
2380                 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
2381                         const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type);
2382                         const char *var_name;
2383                         int var_linkage;
2384                         Elf64_Sym *sym;
2385
2386                         /* could be a variable or function */
2387                         if (!btf_is_var(vt))
2388                                 continue;
2389
2390                         var_name = btf__str_by_offset(obj->btf, vt->name_off);
2391                         var_linkage = btf_var(vt)->linkage;
2392
2393                         /* no need to patch up static or extern vars */
2394                         if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED)
2395                                 continue;
2396
2397                         sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name);
2398                         if (!sym) {
2399                                 pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name);
2400                                 return -ENOENT;
2401                         }
2402
2403                         vi->offset = sym->st_value;
2404                 }
2405         }
2406
2407         return 0;
2408 }
2409
2410 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj)
2411 {
2412         const struct btf_type *t;
2413         int i, j, n, start_id, id, err;
2414         const char *name;
2415
2416         if (!obj->btf)
2417                 return 0;
2418
2419         start_id = btf__type_cnt(linker->btf);
2420         n = btf__type_cnt(obj->btf);
2421
2422         obj->btf_type_map = calloc(n + 1, sizeof(int));
2423         if (!obj->btf_type_map)
2424                 return -ENOMEM;
2425
2426         for (i = 1; i < n; i++) {
2427                 struct glob_sym *glob_sym = NULL;
2428
2429                 t = btf__type_by_id(obj->btf, i);
2430
2431                 /* DATASECs are handled specially below */
2432                 if (btf_kind(t) == BTF_KIND_DATASEC)
2433                         continue;
2434
2435                 if (btf_is_non_static(t)) {
2436                         /* there should be glob_sym already */
2437                         name = btf__str_by_offset(obj->btf, t->name_off);
2438                         glob_sym = find_glob_sym(linker, name);
2439
2440                         /* VARs without corresponding glob_sym are those that
2441                          * belong to skipped/deduplicated sections (i.e.,
2442                          * license and version), so just skip them
2443                          */
2444                         if (!glob_sym)
2445                                 continue;
2446
2447                         /* linker_append_elf_sym() might have requested
2448                          * updating underlying type ID, if extern was resolved
2449                          * to strong symbol or weak got upgraded to non-weak
2450                          */
2451                         if (glob_sym->underlying_btf_id == 0)
2452                                 glob_sym->underlying_btf_id = -t->type;
2453
2454                         /* globals from previous object files that match our
2455                          * VAR/FUNC already have a corresponding associated
2456                          * BTF type, so just make sure to use it
2457                          */
2458                         if (glob_sym->btf_id) {
2459                                 /* reuse existing BTF type for global var/func */
2460                                 obj->btf_type_map[i] = glob_sym->btf_id;
2461                                 continue;
2462                         }
2463                 }
2464
2465                 id = btf__add_type(linker->btf, obj->btf, t);
2466                 if (id < 0) {
2467                         pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename);
2468                         return id;
2469                 }
2470
2471                 obj->btf_type_map[i] = id;
2472
2473                 /* record just appended BTF type for var/func */
2474                 if (glob_sym) {
2475                         glob_sym->btf_id = id;
2476                         glob_sym->underlying_btf_id = -t->type;
2477                 }
2478         }
2479
2480         /* remap all the types except DATASECs */
2481         n = btf__type_cnt(linker->btf);
2482         for (i = start_id; i < n; i++) {
2483                 struct btf_type *dst_t = btf_type_by_id(linker->btf, i);
2484                 struct btf_field_iter it;
2485                 __u32 *type_id;
2486
2487                 err = btf_field_iter_init(&it, dst_t, BTF_FIELD_ITER_IDS);
2488                 if (err)
2489                         return err;
2490
2491                 while ((type_id = btf_field_iter_next(&it))) {
2492                         int new_id = obj->btf_type_map[*type_id];
2493
2494                         /* Error out if the type wasn't remapped. Ignore VOID which stays VOID. */
2495                         if (new_id == 0 && *type_id != 0) {
2496                                 pr_warn("failed to find new ID mapping for original BTF type ID %u\n",
2497                                         *type_id);
2498                                 return -EINVAL;
2499                         }
2500
2501                         *type_id = obj->btf_type_map[*type_id];
2502                 }
2503         }
2504
2505         /* Rewrite VAR/FUNC underlying types (i.e., FUNC's FUNC_PROTO and VAR's
2506          * actual type), if necessary
2507          */
2508         for (i = 0; i < linker->glob_sym_cnt; i++) {
2509                 struct glob_sym *glob_sym = &linker->glob_syms[i];
2510                 struct btf_type *glob_t;
2511
2512                 if (glob_sym->underlying_btf_id >= 0)
2513                         continue;
2514
2515                 glob_sym->underlying_btf_id = obj->btf_type_map[-glob_sym->underlying_btf_id];
2516
2517                 glob_t = btf_type_by_id(linker->btf, glob_sym->btf_id);
2518                 glob_t->type = glob_sym->underlying_btf_id;
2519         }
2520
2521         /* append DATASEC info */
2522         for (i = 1; i < obj->sec_cnt; i++) {
2523                 struct src_sec *src_sec;
2524                 struct dst_sec *dst_sec;
2525                 const struct btf_var_secinfo *src_var;
2526                 struct btf_var_secinfo *dst_var;
2527
2528                 src_sec = &obj->secs[i];
2529                 if (!src_sec->sec_type_id || src_sec->skipped)
2530                         continue;
2531                 dst_sec = &linker->secs[src_sec->dst_id];
2532
2533                 /* Mark section as having BTF regardless of the presence of
2534                  * variables. In some cases compiler might generate empty BTF
2535                  * with no variables information. E.g., when promoting local
2536                  * array/structure variable initial values and BPF object
2537                  * file otherwise has no read-only static variables in
2538                  * .rodata. We need to preserve such empty BTF and just set
2539                  * correct section size.
2540                  */
2541                 dst_sec->has_btf = true;
2542
2543                 t = btf__type_by_id(obj->btf, src_sec->sec_type_id);
2544                 src_var = btf_var_secinfos(t);
2545                 n = btf_vlen(t);
2546                 for (j = 0; j < n; j++, src_var++) {
2547                         void *sec_vars = dst_sec->sec_vars;
2548                         int new_id = obj->btf_type_map[src_var->type];
2549                         struct glob_sym *glob_sym = NULL;
2550
2551                         t = btf_type_by_id(linker->btf, new_id);
2552                         if (btf_is_non_static(t)) {
2553                                 name = btf__str_by_offset(linker->btf, t->name_off);
2554                                 glob_sym = find_glob_sym(linker, name);
2555                                 if (glob_sym->sec_id != dst_sec->id) {
2556                                         pr_warn("global '%s': section mismatch %d vs %d\n",
2557                                                 name, glob_sym->sec_id, dst_sec->id);
2558                                         return -EINVAL;
2559                                 }
2560                         }
2561
2562                         /* If there is already a member (VAR or FUNC) mapped
2563                          * to the same type, don't add a duplicate entry.
2564                          * This will happen when multiple object files define
2565                          * the same extern VARs/FUNCs.
2566                          */
2567                         if (glob_sym && glob_sym->var_idx >= 0) {
2568                                 __s64 sz;
2569
2570                                 /* FUNCs don't have size, nothing to update */
2571                                 if (btf_is_func(t))
2572                                         continue;
2573
2574                                 dst_var = &dst_sec->sec_vars[glob_sym->var_idx];
2575                                 /* Because underlying BTF type might have
2576                                  * changed, so might its size have changed, so
2577                                  * re-calculate and update it in sec_var.
2578                                  */
2579                                 sz = btf__resolve_size(linker->btf, glob_sym->underlying_btf_id);
2580                                 if (sz < 0) {
2581                                         pr_warn("global '%s': failed to resolve size of underlying type: %d\n",
2582                                                 name, (int)sz);
2583                                         return -EINVAL;
2584                                 }
2585                                 dst_var->size = sz;
2586                                 continue;
2587                         }
2588
2589                         sec_vars = libbpf_reallocarray(sec_vars,
2590                                                        dst_sec->sec_var_cnt + 1,
2591                                                        sizeof(*dst_sec->sec_vars));
2592                         if (!sec_vars)
2593                                 return -ENOMEM;
2594
2595                         dst_sec->sec_vars = sec_vars;
2596                         dst_sec->sec_var_cnt++;
2597
2598                         dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1];
2599                         dst_var->type = obj->btf_type_map[src_var->type];
2600                         dst_var->size = src_var->size;
2601                         dst_var->offset = src_sec->dst_off + src_var->offset;
2602
2603                         if (glob_sym)
2604                                 glob_sym->var_idx = dst_sec->sec_var_cnt - 1;
2605                 }
2606         }
2607
2608         return 0;
2609 }
2610
2611 static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec)
2612 {
2613         void *tmp;
2614
2615         tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz);
2616         if (!tmp)
2617                 return NULL;
2618         ext_data->recs = tmp;
2619
2620         tmp += ext_data->rec_cnt * ext_data->rec_sz;
2621         memcpy(tmp, src_rec, ext_data->rec_sz);
2622
2623         ext_data->rec_cnt++;
2624
2625         return tmp;
2626 }
2627
2628 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj)
2629 {
2630         const struct btf_ext_info_sec *ext_sec;
2631         const char *sec_name, *s;
2632         struct src_sec *src_sec;
2633         struct dst_sec *dst_sec;
2634         int rec_sz, str_off, i;
2635
2636         if (!obj->btf_ext)
2637                 return 0;
2638
2639         rec_sz = obj->btf_ext->func_info.rec_size;
2640         for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) {
2641                 struct bpf_func_info_min *src_rec, *dst_rec;
2642
2643                 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2644                 src_sec = find_src_sec_by_name(obj, sec_name);
2645                 if (!src_sec) {
2646                         pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2647                         return -EINVAL;
2648                 }
2649                 dst_sec = &linker->secs[src_sec->dst_id];
2650
2651                 if (dst_sec->func_info.rec_sz == 0)
2652                         dst_sec->func_info.rec_sz = rec_sz;
2653                 if (dst_sec->func_info.rec_sz != rec_sz) {
2654                         pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2655                         return -EINVAL;
2656                 }
2657
2658                 for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) {
2659                         dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec);
2660                         if (!dst_rec)
2661                                 return -ENOMEM;
2662
2663                         dst_rec->insn_off += src_sec->dst_off;
2664                         dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2665                 }
2666         }
2667
2668         rec_sz = obj->btf_ext->line_info.rec_size;
2669         for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) {
2670                 struct bpf_line_info_min *src_rec, *dst_rec;
2671
2672                 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2673                 src_sec = find_src_sec_by_name(obj, sec_name);
2674                 if (!src_sec) {
2675                         pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2676                         return -EINVAL;
2677                 }
2678                 dst_sec = &linker->secs[src_sec->dst_id];
2679
2680                 if (dst_sec->line_info.rec_sz == 0)
2681                         dst_sec->line_info.rec_sz = rec_sz;
2682                 if (dst_sec->line_info.rec_sz != rec_sz) {
2683                         pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2684                         return -EINVAL;
2685                 }
2686
2687                 for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) {
2688                         dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec);
2689                         if (!dst_rec)
2690                                 return -ENOMEM;
2691
2692                         dst_rec->insn_off += src_sec->dst_off;
2693
2694                         s = btf__str_by_offset(obj->btf, src_rec->file_name_off);
2695                         str_off = btf__add_str(linker->btf, s);
2696                         if (str_off < 0)
2697                                 return -ENOMEM;
2698                         dst_rec->file_name_off = str_off;
2699
2700                         s = btf__str_by_offset(obj->btf, src_rec->line_off);
2701                         str_off = btf__add_str(linker->btf, s);
2702                         if (str_off < 0)
2703                                 return -ENOMEM;
2704                         dst_rec->line_off = str_off;
2705
2706                         /* dst_rec->line_col is fine */
2707                 }
2708         }
2709
2710         rec_sz = obj->btf_ext->core_relo_info.rec_size;
2711         for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) {
2712                 struct bpf_core_relo *src_rec, *dst_rec;
2713
2714                 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2715                 src_sec = find_src_sec_by_name(obj, sec_name);
2716                 if (!src_sec) {
2717                         pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2718                         return -EINVAL;
2719                 }
2720                 dst_sec = &linker->secs[src_sec->dst_id];
2721
2722                 if (dst_sec->core_relo_info.rec_sz == 0)
2723                         dst_sec->core_relo_info.rec_sz = rec_sz;
2724                 if (dst_sec->core_relo_info.rec_sz != rec_sz) {
2725                         pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2726                         return -EINVAL;
2727                 }
2728
2729                 for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) {
2730                         dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec);
2731                         if (!dst_rec)
2732                                 return -ENOMEM;
2733
2734                         dst_rec->insn_off += src_sec->dst_off;
2735                         dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2736
2737                         s = btf__str_by_offset(obj->btf, src_rec->access_str_off);
2738                         str_off = btf__add_str(linker->btf, s);
2739                         if (str_off < 0)
2740                                 return -ENOMEM;
2741                         dst_rec->access_str_off = str_off;
2742
2743                         /* dst_rec->kind is fine */
2744                 }
2745         }
2746
2747         return 0;
2748 }
2749
2750 int bpf_linker__finalize(struct bpf_linker *linker)
2751 {
2752         struct dst_sec *sec;
2753         size_t strs_sz;
2754         const void *strs;
2755         int err, i;
2756
2757         if (!linker->elf)
2758                 return libbpf_err(-EINVAL);
2759
2760         err = finalize_btf(linker);
2761         if (err)
2762                 return libbpf_err(err);
2763
2764         /* Finalize strings */
2765         strs_sz = strset__data_size(linker->strtab_strs);
2766         strs = strset__data(linker->strtab_strs);
2767
2768         sec = &linker->secs[linker->strtab_sec_idx];
2769         sec->data->d_align = 1;
2770         sec->data->d_off = 0LL;
2771         sec->data->d_buf = (void *)strs;
2772         sec->data->d_type = ELF_T_BYTE;
2773         sec->data->d_size = strs_sz;
2774         sec->shdr->sh_size = strs_sz;
2775
2776         for (i = 1; i < linker->sec_cnt; i++) {
2777                 sec = &linker->secs[i];
2778
2779                 /* STRTAB is handled specially above */
2780                 if (sec->sec_idx == linker->strtab_sec_idx)
2781                         continue;
2782
2783                 /* special ephemeral sections (.ksyms, .kconfig, etc) */
2784                 if (!sec->scn)
2785                         continue;
2786
2787                 /* restore sections with bpf insns to target byte-order */
2788                 if (linker->swapped_endian && is_exec_sec(sec))
2789                         exec_sec_bswap(sec->raw_data, sec->sec_sz);
2790
2791                 sec->data->d_buf = sec->raw_data;
2792         }
2793
2794         /* Finalize ELF layout */
2795         if (elf_update(linker->elf, ELF_C_NULL) < 0) {
2796                 err = -EINVAL;
2797                 pr_warn_elf("failed to finalize ELF layout");
2798                 return libbpf_err(err);
2799         }
2800
2801         /* Write out final ELF contents */
2802         if (elf_update(linker->elf, ELF_C_WRITE) < 0) {
2803                 err = -EINVAL;
2804                 pr_warn_elf("failed to write ELF contents");
2805                 return libbpf_err(err);
2806         }
2807
2808         elf_end(linker->elf);
2809         linker->elf = NULL;
2810
2811         if (linker->fd_is_owned)
2812                 close(linker->fd);
2813         linker->fd = -1;
2814
2815         return 0;
2816 }
2817
2818 static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name,
2819                              size_t align, const void *raw_data, size_t raw_sz)
2820 {
2821         Elf_Scn *scn;
2822         Elf_Data *data;
2823         Elf64_Shdr *shdr;
2824         int name_off;
2825
2826         name_off = strset__add_str(linker->strtab_strs, sec_name);
2827         if (name_off < 0)
2828                 return name_off;
2829
2830         scn = elf_newscn(linker->elf);
2831         if (!scn)
2832                 return -ENOMEM;
2833         data = elf_newdata(scn);
2834         if (!data)
2835                 return -ENOMEM;
2836         shdr = elf64_getshdr(scn);
2837         if (!shdr)
2838                 return -EINVAL;
2839
2840         shdr->sh_name = name_off;
2841         shdr->sh_type = SHT_PROGBITS;
2842         shdr->sh_flags = 0;
2843         shdr->sh_size = raw_sz;
2844         shdr->sh_link = 0;
2845         shdr->sh_info = 0;
2846         shdr->sh_addralign = align;
2847         shdr->sh_entsize = 0;
2848
2849         data->d_type = ELF_T_BYTE;
2850         data->d_size = raw_sz;
2851         data->d_buf = (void *)raw_data;
2852         data->d_align = align;
2853         data->d_off = 0;
2854
2855         return 0;
2856 }
2857
2858 static int finalize_btf(struct bpf_linker *linker)
2859 {
2860         enum btf_endianness link_endianness;
2861         LIBBPF_OPTS(btf_dedup_opts, opts);
2862         struct btf *btf = linker->btf;
2863         const void *raw_data;
2864         int i, j, id, err;
2865         __u32 raw_sz;
2866
2867         /* bail out if no BTF data was produced */
2868         if (btf__type_cnt(linker->btf) == 1)
2869                 return 0;
2870
2871         for (i = 1; i < linker->sec_cnt; i++) {
2872                 struct dst_sec *sec = &linker->secs[i];
2873
2874                 if (!sec->has_btf)
2875                         continue;
2876
2877                 id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz);
2878                 if (id < 0) {
2879                         pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n",
2880                                 sec->sec_name, id);
2881                         return id;
2882                 }
2883
2884                 for (j = 0; j < sec->sec_var_cnt; j++) {
2885                         struct btf_var_secinfo *vi = &sec->sec_vars[j];
2886
2887                         if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size))
2888                                 return -EINVAL;
2889                 }
2890         }
2891
2892         err = finalize_btf_ext(linker);
2893         if (err) {
2894                 pr_warn(".BTF.ext generation failed: %s\n", errstr(err));
2895                 return err;
2896         }
2897
2898         opts.btf_ext = linker->btf_ext;
2899         err = btf__dedup(linker->btf, &opts);
2900         if (err) {
2901                 pr_warn("BTF dedup failed: %s\n", errstr(err));
2902                 return err;
2903         }
2904
2905         /* Set .BTF and .BTF.ext output byte order */
2906         link_endianness = linker->elf_hdr->e_ident[EI_DATA] == ELFDATA2MSB ?
2907                           BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN;
2908         btf__set_endianness(linker->btf, link_endianness);
2909         if (linker->btf_ext)
2910                 btf_ext__set_endianness(linker->btf_ext, link_endianness);
2911
2912         /* Emit .BTF section */
2913         raw_data = btf__raw_data(linker->btf, &raw_sz);
2914         if (!raw_data)
2915                 return -ENOMEM;
2916
2917         err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz);
2918         if (err) {
2919                 pr_warn("failed to write out .BTF ELF section: %s\n", errstr(err));
2920                 return err;
2921         }
2922
2923         /* Emit .BTF.ext section */
2924         if (linker->btf_ext) {
2925                 raw_data = btf_ext__raw_data(linker->btf_ext, &raw_sz);
2926                 if (!raw_data)
2927                         return -ENOMEM;
2928
2929                 err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz);
2930                 if (err) {
2931                         pr_warn("failed to write out .BTF.ext ELF section: %s\n", errstr(err));
2932                         return err;
2933                 }
2934         }
2935
2936         return 0;
2937 }
2938
2939 static int emit_btf_ext_data(struct bpf_linker *linker, void *output,
2940                              const char *sec_name, struct btf_ext_sec_data *sec_data)
2941 {
2942         struct btf_ext_info_sec *sec_info;
2943         void *cur = output;
2944         int str_off;
2945         size_t sz;
2946
2947         if (!sec_data->rec_cnt)
2948                 return 0;
2949
2950         str_off = btf__add_str(linker->btf, sec_name);
2951         if (str_off < 0)
2952                 return -ENOMEM;
2953
2954         sec_info = cur;
2955         sec_info->sec_name_off = str_off;
2956         sec_info->num_info = sec_data->rec_cnt;
2957         cur += sizeof(struct btf_ext_info_sec);
2958
2959         sz = sec_data->rec_cnt * sec_data->rec_sz;
2960         memcpy(cur, sec_data->recs, sz);
2961         cur += sz;
2962
2963         return cur - output;
2964 }
2965
2966 static int finalize_btf_ext(struct bpf_linker *linker)
2967 {
2968         size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0;
2969         size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0;
2970         struct btf_ext_header *hdr;
2971         void *data, *cur;
2972         int i, err, sz;
2973
2974         /* validate that all sections have the same .BTF.ext record sizes
2975          * and calculate total data size for each type of data (func info,
2976          * line info, core relos)
2977          */
2978         for (i = 1; i < linker->sec_cnt; i++) {
2979                 struct dst_sec *sec = &linker->secs[i];
2980
2981                 if (sec->func_info.rec_cnt) {
2982                         if (func_rec_sz == 0)
2983                                 func_rec_sz = sec->func_info.rec_sz;
2984                         if (func_rec_sz != sec->func_info.rec_sz) {
2985                                 pr_warn("mismatch in func_info record size %zu != %u\n",
2986                                         func_rec_sz, sec->func_info.rec_sz);
2987                                 return -EINVAL;
2988                         }
2989
2990                         funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt;
2991                 }
2992                 if (sec->line_info.rec_cnt) {
2993                         if (line_rec_sz == 0)
2994                                 line_rec_sz = sec->line_info.rec_sz;
2995                         if (line_rec_sz != sec->line_info.rec_sz) {
2996                                 pr_warn("mismatch in line_info record size %zu != %u\n",
2997                                         line_rec_sz, sec->line_info.rec_sz);
2998                                 return -EINVAL;
2999                         }
3000
3001                         lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt;
3002                 }
3003                 if (sec->core_relo_info.rec_cnt) {
3004                         if (core_relo_rec_sz == 0)
3005                                 core_relo_rec_sz = sec->core_relo_info.rec_sz;
3006                         if (core_relo_rec_sz != sec->core_relo_info.rec_sz) {
3007                                 pr_warn("mismatch in core_relo_info record size %zu != %u\n",
3008                                         core_relo_rec_sz, sec->core_relo_info.rec_sz);
3009                                 return -EINVAL;
3010                         }
3011
3012                         core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt;
3013                 }
3014         }
3015
3016         if (!funcs_sz && !lines_sz && !core_relos_sz)
3017                 return 0;
3018
3019         total_sz += sizeof(struct btf_ext_header);
3020         if (funcs_sz) {
3021                 funcs_sz += sizeof(__u32); /* record size prefix */
3022                 total_sz += funcs_sz;
3023         }
3024         if (lines_sz) {
3025                 lines_sz += sizeof(__u32); /* record size prefix */
3026                 total_sz += lines_sz;
3027         }
3028         if (core_relos_sz) {
3029                 core_relos_sz += sizeof(__u32); /* record size prefix */
3030                 total_sz += core_relos_sz;
3031         }
3032
3033         cur = data = calloc(1, total_sz);
3034         if (!data)
3035                 return -ENOMEM;
3036
3037         hdr = cur;
3038         hdr->magic = BTF_MAGIC;
3039         hdr->version = BTF_VERSION;
3040         hdr->flags = 0;
3041         hdr->hdr_len = sizeof(struct btf_ext_header);
3042         cur += sizeof(struct btf_ext_header);
3043
3044         /* All offsets are in bytes relative to the end of this header */
3045         hdr->func_info_off = 0;
3046         hdr->func_info_len = funcs_sz;
3047         hdr->line_info_off = funcs_sz;
3048         hdr->line_info_len = lines_sz;
3049         hdr->core_relo_off = funcs_sz + lines_sz;
3050         hdr->core_relo_len = core_relos_sz;
3051
3052         if (funcs_sz) {
3053                 *(__u32 *)cur = func_rec_sz;
3054                 cur += sizeof(__u32);
3055
3056                 for (i = 1; i < linker->sec_cnt; i++) {
3057                         struct dst_sec *sec = &linker->secs[i];
3058
3059                         sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info);
3060                         if (sz < 0) {
3061                                 err = sz;
3062                                 goto out;
3063                         }
3064
3065                         cur += sz;
3066                 }
3067         }
3068
3069         if (lines_sz) {
3070                 *(__u32 *)cur = line_rec_sz;
3071                 cur += sizeof(__u32);
3072
3073                 for (i = 1; i < linker->sec_cnt; i++) {
3074                         struct dst_sec *sec = &linker->secs[i];
3075
3076                         sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info);
3077                         if (sz < 0) {
3078                                 err = sz;
3079                                 goto out;
3080                         }
3081
3082                         cur += sz;
3083                 }
3084         }
3085
3086         if (core_relos_sz) {
3087                 *(__u32 *)cur = core_relo_rec_sz;
3088                 cur += sizeof(__u32);
3089
3090                 for (i = 1; i < linker->sec_cnt; i++) {
3091                         struct dst_sec *sec = &linker->secs[i];
3092
3093                         sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info);
3094                         if (sz < 0) {
3095                                 err = sz;
3096                                 goto out;
3097                         }
3098
3099                         cur += sz;
3100                 }
3101         }
3102
3103         linker->btf_ext = btf_ext__new(data, total_sz);
3104         err = libbpf_get_error(linker->btf_ext);
3105         if (err) {
3106                 linker->btf_ext = NULL;
3107                 pr_warn("failed to parse final .BTF.ext data: %s\n", errstr(err));
3108                 goto out;
3109         }
3110
3111 out:
3112         free(data);
3113         return err;
3114 }