tools/lib/bpf/linker.c

   1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
   2 /*
   3  * BPF static linker
   4  *
   5  * Copyright (c) 2021 Facebook
   6  */
   7 #include <stdbool.h>
   8 #include <stddef.h>
   9 #include <stdio.h>
  10 #include <stdlib.h>
  11 #include <string.h>
  12 #include <unistd.h>
  13 #include <errno.h>
  14 #include <linux/err.h>
  15 #include <linux/btf.h>
  16 #include <elf.h>
  17 #include <libelf.h>
  18 #include <fcntl.h>
  19 #include "libbpf.h"
  20 #include "btf.h"
  21 #include "libbpf_internal.h"
  22 #include "strset.h"
  23 #include "str_error.h"
  24
  25 #define BTF_EXTERN_SEC ".extern"
  26
  27 struct src_sec {
  28         const char *sec_name;
  29         /* positional (not necessarily ELF) index in an array of sections */
  30         int id;
  31         /* positional (not necessarily ELF) index of a matching section in a final object file */
  32         int dst_id;
  33         /* section data offset in a matching output section */
  34         int dst_off;
  35         /* whether section is omitted from the final ELF file */
  36         bool skipped;
  37         /* whether section is an ephemeral section, not mapped to an ELF section */
  38         bool ephemeral;
  39
  40         /* ELF info */
  41         size_t sec_idx;
  42         Elf_Scn *scn;
  43         Elf64_Shdr *shdr;
  44         Elf_Data *data;
  45
  46         /* corresponding BTF DATASEC type ID */
  47         int sec_type_id;
  48 };
  49
  50 struct src_obj {
  51         const char *filename;
  52         int fd;
  53         Elf *elf;
  54         /* Section header strings section index */
  55         size_t shstrs_sec_idx;
  56         /* SYMTAB section index */
  57         size_t symtab_sec_idx;
  58
  59         struct btf *btf;
  60         struct btf_ext *btf_ext;
  61
  62         /* List of sections (including ephemeral). Slot zero is unused. */
  63         struct src_sec *secs;
  64         int sec_cnt;
  65
  66         /* mapping of symbol indices from src to dst ELF */
  67         int *sym_map;
  68         /* mapping from the src BTF type IDs to dst ones */
  69         int *btf_type_map;
  70 };
  71
  72 /* single .BTF.ext data section */
  73 struct btf_ext_sec_data {
  74         size_t rec_cnt;
  75         __u32 rec_sz;
  76         void *recs;
  77 };
  78
  79 struct glob_sym {
  80         /* ELF symbol index */
  81         int sym_idx;
  82         /* associated section id for .ksyms, .kconfig, etc, but not .extern */
  83         int sec_id;
  84         /* extern name offset in STRTAB */
  85         int name_off;
  86         /* optional associated BTF type ID */
  87         int btf_id;
  88         /* BTF type ID to which VAR/FUNC type is pointing to; used for
  89          * rewriting types when extern VAR/FUNC is resolved to a concrete
  90          * definition
  91          */
  92         int underlying_btf_id;
  93         /* sec_var index in the corresponding dst_sec, if exists */
  94         int var_idx;
  95
  96         /* extern or resolved/global symbol */
  97         bool is_extern;
  98         /* weak or strong symbol, never goes back from strong to weak */
  99         bool is_weak;
 100 };
 101
 102 struct dst_sec {
 103         char *sec_name;
 104         /* positional (not necessarily ELF) index in an array of sections */
 105         int id;
 106
 107         bool ephemeral;
 108
 109         /* ELF info */
 110         size_t sec_idx;
 111         Elf_Scn *scn;
 112         Elf64_Shdr *shdr;
 113         Elf_Data *data;
 114
 115         /* final output section size */
 116         int sec_sz;
 117         /* final output contents of the section */
 118         void *raw_data;
 119
 120         /* corresponding STT_SECTION symbol index in SYMTAB */
 121         int sec_sym_idx;
 122
 123         /* section's DATASEC variable info, emitted on BTF finalization */
 124         bool has_btf;
 125         int sec_var_cnt;
 126         struct btf_var_secinfo *sec_vars;
 127
 128         /* section's .BTF.ext data */
 129         struct btf_ext_sec_data func_info;
 130         struct btf_ext_sec_data line_info;
 131         struct btf_ext_sec_data core_relo_info;
 132 };
 133
 134 struct bpf_linker {
 135         char *filename;
 136         int fd;
 137         Elf *elf;
 138         Elf64_Ehdr *elf_hdr;
 139         bool swapped_endian;
 140
 141         /* Output sections metadata */
 142         struct dst_sec *secs;
 143         int sec_cnt;
 144
 145         struct strset *strtab_strs; /* STRTAB unique strings */
 146         size_t strtab_sec_idx; /* STRTAB section index */
 147         size_t symtab_sec_idx; /* SYMTAB section index */
 148
 149         struct btf *btf;
 150         struct btf_ext *btf_ext;
 151
 152         /* global (including extern) ELF symbols */
 153         int glob_sym_cnt;
 154         struct glob_sym *glob_syms;
 155 };
 156
 157 #define pr_warn_elf(fmt, ...)                                                                   \
 158         libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1))
 159
 160 static int init_output_elf(struct bpf_linker *linker, const char *file);
 161
 162 static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
 163                                 const struct bpf_linker_file_opts *opts,
 164                                 struct src_obj *obj);
 165 static int linker_sanity_check_elf(struct src_obj *obj);
 166 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec);
 167 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec);
 168 static int linker_sanity_check_btf(struct src_obj *obj);
 169 static int linker_sanity_check_btf_ext(struct src_obj *obj);
 170 static int linker_fixup_btf(struct src_obj *obj);
 171 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj);
 172 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj);
 173 static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
 174                                  Elf64_Sym *sym, const char *sym_name, int src_sym_idx);
 175 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj);
 176 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj);
 177 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj);
 178
 179 static int finalize_btf(struct bpf_linker *linker);
 180 static int finalize_btf_ext(struct bpf_linker *linker);
 181
 182 void bpf_linker__free(struct bpf_linker *linker)
 183 {
 184         int i;
 185
 186         if (!linker)
 187                 return;
 188
 189         free(linker->filename);
 190
 191         if (linker->elf)
 192                 elf_end(linker->elf);
 193
 194         if (linker->fd >= 0)
 195                 close(linker->fd);
 196
 197         strset__free(linker->strtab_strs);
 198
 199         btf__free(linker->btf);
 200         btf_ext__free(linker->btf_ext);
 201
 202         for (i = 1; i < linker->sec_cnt; i++) {
 203                 struct dst_sec *sec = &linker->secs[i];
 204
 205                 free(sec->sec_name);
 206                 free(sec->raw_data);
 207                 free(sec->sec_vars);
 208
 209                 free(sec->func_info.recs);
 210                 free(sec->line_info.recs);
 211                 free(sec->core_relo_info.recs);
 212         }
 213         free(linker->secs);
 214
 215         free(linker->glob_syms);
 216         free(linker);
 217 }
 218
 219 struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts)
 220 {
 221         struct bpf_linker *linker;
 222         int err;
 223
 224         if (!OPTS_VALID(opts, bpf_linker_opts))
 225                 return errno = EINVAL, NULL;
 226
 227         if (elf_version(EV_CURRENT) == EV_NONE) {
 228                 pr_warn_elf("libelf initialization failed");
 229                 return errno = EINVAL, NULL;
 230         }
 231
 232         linker = calloc(1, sizeof(*linker));
 233         if (!linker)
 234                 return errno = ENOMEM, NULL;
 235
 236         linker->fd = -1;
 237
 238         err = init_output_elf(linker, filename);
 239         if (err)
 240                 goto err_out;
 241
 242         return linker;
 243
 244 err_out:
 245         bpf_linker__free(linker);
 246         return errno = -err, NULL;
 247 }
 248
 249 static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name)
 250 {
 251         struct dst_sec *secs = linker->secs, *sec;
 252         size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2;
 253
 254         secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
 255         if (!secs)
 256                 return NULL;
 257
 258         /* zero out newly allocated memory */
 259         memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs));
 260
 261         linker->secs = secs;
 262         linker->sec_cnt = new_cnt;
 263
 264         sec = &linker->secs[new_cnt - 1];
 265         sec->id = new_cnt - 1;
 266         sec->sec_name = strdup(sec_name);
 267         if (!sec->sec_name)
 268                 return NULL;
 269
 270         return sec;
 271 }
 272
 273 static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx)
 274 {
 275         struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
 276         Elf64_Sym *syms, *sym;
 277         size_t sym_cnt = symtab->sec_sz / sizeof(*sym);
 278
 279         syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym));
 280         if (!syms)
 281                 return NULL;
 282
 283         sym = &syms[sym_cnt];
 284         memset(sym, 0, sizeof(*sym));
 285
 286         symtab->raw_data = syms;
 287         symtab->sec_sz += sizeof(*sym);
 288         symtab->shdr->sh_size += sizeof(*sym);
 289         symtab->data->d_size += sizeof(*sym);
 290
 291         if (sym_idx)
 292                 *sym_idx = sym_cnt;
 293
 294         return sym;
 295 }
 296
 297 static int init_output_elf(struct bpf_linker *linker, const char *file)
 298 {
 299         int err, str_off;
 300         Elf64_Sym *init_sym;
 301         struct dst_sec *sec;
 302
 303         linker->filename = strdup(file);
 304         if (!linker->filename)
 305                 return -ENOMEM;
 306
 307         linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);
 308         if (linker->fd < 0) {
 309                 err = -errno;
 310                 pr_warn("failed to create '%s': %s\n", file, errstr(err));
 311                 return err;
 312         }
 313
 314         linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL);
 315         if (!linker->elf) {
 316                 pr_warn_elf("failed to create ELF object");
 317                 return -EINVAL;
 318         }
 319
 320         /* ELF header */
 321         linker->elf_hdr = elf64_newehdr(linker->elf);
 322         if (!linker->elf_hdr) {
 323                 pr_warn_elf("failed to create ELF header");
 324                 return -EINVAL;
 325         }
 326
 327         linker->elf_hdr->e_machine = EM_BPF;
 328         linker->elf_hdr->e_type = ET_REL;
 329         /* Set unknown ELF endianness, assign later from input files */
 330         linker->elf_hdr->e_ident[EI_DATA] = ELFDATANONE;
 331
 332         /* STRTAB */
 333         /* initialize strset with an empty string to conform to ELF */
 334         linker->strtab_strs = strset__new(INT_MAX, "", sizeof(""));
 335         if (libbpf_get_error(linker->strtab_strs))
 336                 return libbpf_get_error(linker->strtab_strs);
 337
 338         sec = add_dst_sec(linker, ".strtab");
 339         if (!sec)
 340                 return -ENOMEM;
 341
 342         sec->scn = elf_newscn(linker->elf);
 343         if (!sec->scn) {
 344                 pr_warn_elf("failed to create STRTAB section");
 345                 return -EINVAL;
 346         }
 347
 348         sec->shdr = elf64_getshdr(sec->scn);
 349         if (!sec->shdr)
 350                 return -EINVAL;
 351
 352         sec->data = elf_newdata(sec->scn);
 353         if (!sec->data) {
 354                 pr_warn_elf("failed to create STRTAB data");
 355                 return -EINVAL;
 356         }
 357
 358         str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
 359         if (str_off < 0)
 360                 return str_off;
 361
 362         sec->sec_idx = elf_ndxscn(sec->scn);
 363         linker->elf_hdr->e_shstrndx = sec->sec_idx;
 364         linker->strtab_sec_idx = sec->sec_idx;
 365
 366         sec->shdr->sh_name = str_off;
 367         sec->shdr->sh_type = SHT_STRTAB;
 368         sec->shdr->sh_flags = SHF_STRINGS;
 369         sec->shdr->sh_offset = 0;
 370         sec->shdr->sh_link = 0;
 371         sec->shdr->sh_info = 0;
 372         sec->shdr->sh_addralign = 1;
 373         sec->shdr->sh_size = sec->sec_sz = 0;
 374         sec->shdr->sh_entsize = 0;
 375
 376         /* SYMTAB */
 377         sec = add_dst_sec(linker, ".symtab");
 378         if (!sec)
 379                 return -ENOMEM;
 380
 381         sec->scn = elf_newscn(linker->elf);
 382         if (!sec->scn) {
 383                 pr_warn_elf("failed to create SYMTAB section");
 384                 return -EINVAL;
 385         }
 386
 387         sec->shdr = elf64_getshdr(sec->scn);
 388         if (!sec->shdr)
 389                 return -EINVAL;
 390
 391         sec->data = elf_newdata(sec->scn);
 392         if (!sec->data) {
 393                 pr_warn_elf("failed to create SYMTAB data");
 394                 return -EINVAL;
 395         }
 396         /* Ensure libelf translates byte-order of symbol records */
 397         sec->data->d_type = ELF_T_SYM;
 398
 399         str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
 400         if (str_off < 0)
 401                 return str_off;
 402
 403         sec->sec_idx = elf_ndxscn(sec->scn);
 404         linker->symtab_sec_idx = sec->sec_idx;
 405
 406         sec->shdr->sh_name = str_off;
 407         sec->shdr->sh_type = SHT_SYMTAB;
 408         sec->shdr->sh_flags = 0;
 409         sec->shdr->sh_offset = 0;
 410         sec->shdr->sh_link = linker->strtab_sec_idx;
 411         /* sh_info should be one greater than the index of the last local
 412          * symbol (i.e., binding is STB_LOCAL). But why and who cares?
 413          */
 414         sec->shdr->sh_info = 0;
 415         sec->shdr->sh_addralign = 8;
 416         sec->shdr->sh_entsize = sizeof(Elf64_Sym);
 417
 418         /* .BTF */
 419         linker->btf = btf__new_empty();
 420         err = libbpf_get_error(linker->btf);
 421         if (err)
 422                 return err;
 423
 424         /* add the special all-zero symbol */
 425         init_sym = add_new_sym(linker, NULL);
 426         if (!init_sym)
 427                 return -EINVAL;
 428
 429         init_sym->st_name = 0;
 430         init_sym->st_info = 0;
 431         init_sym->st_other = 0;
 432         init_sym->st_shndx = SHN_UNDEF;
 433         init_sym->st_value = 0;
 434         init_sym->st_size = 0;
 435
 436         return 0;
 437 }
 438
 439 int bpf_linker__add_file(struct bpf_linker *linker, const char *filename,
 440                          const struct bpf_linker_file_opts *opts)
 441 {
 442         struct src_obj obj = {};
 443         int err = 0;
 444
 445         if (!OPTS_VALID(opts, bpf_linker_file_opts))
 446                 return libbpf_err(-EINVAL);
 447
 448         if (!linker->elf)
 449                 return libbpf_err(-EINVAL);
 450
 451         err = err ?: linker_load_obj_file(linker, filename, opts, &obj);
 452         err = err ?: linker_append_sec_data(linker, &obj);
 453         err = err ?: linker_append_elf_syms(linker, &obj);
 454         err = err ?: linker_append_elf_relos(linker, &obj);
 455         err = err ?: linker_append_btf(linker, &obj);
 456         err = err ?: linker_append_btf_ext(linker, &obj);
 457
 458         /* free up src_obj resources */
 459         free(obj.btf_type_map);
 460         btf__free(obj.btf);
 461         btf_ext__free(obj.btf_ext);
 462         free(obj.secs);
 463         free(obj.sym_map);
 464         if (obj.elf)
 465                 elf_end(obj.elf);
 466         if (obj.fd >= 0)
 467                 close(obj.fd);
 468
 469         return libbpf_err(err);
 470 }
 471
 472 static bool is_dwarf_sec_name(const char *name)
 473 {
 474         /* approximation, but the actual list is too long */
 475         return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
 476 }
 477
 478 static bool is_ignored_sec(struct src_sec *sec)
 479 {
 480         Elf64_Shdr *shdr = sec->shdr;
 481         const char *name = sec->sec_name;
 482
 483         /* no special handling of .strtab */
 484         if (shdr->sh_type == SHT_STRTAB)
 485                 return true;
 486
 487         /* ignore .llvm_addrsig section as well */
 488         if (shdr->sh_type == SHT_LLVM_ADDRSIG)
 489                 return true;
 490
 491         /* no subprograms will lead to an empty .text section, ignore it */
 492         if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 &&
 493             strcmp(sec->sec_name, ".text") == 0)
 494                 return true;
 495
 496         /* DWARF sections */
 497         if (is_dwarf_sec_name(sec->sec_name))
 498                 return true;
 499
 500         if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
 501                 name += sizeof(".rel") - 1;
 502                 /* DWARF section relocations */
 503                 if (is_dwarf_sec_name(name))
 504                         return true;
 505
 506                 /* .BTF and .BTF.ext don't need relocations */
 507                 if (strcmp(name, BTF_ELF_SEC) == 0 ||
 508                     strcmp(name, BTF_EXT_ELF_SEC) == 0)
 509                         return true;
 510         }
 511
 512         return false;
 513 }
 514
 515 static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name)
 516 {
 517         struct src_sec *secs = obj->secs, *sec;
 518         size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2;
 519
 520         secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
 521         if (!secs)
 522                 return NULL;
 523
 524         /* zero out newly allocated memory */
 525         memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs));
 526
 527         obj->secs = secs;
 528         obj->sec_cnt = new_cnt;
 529
 530         sec = &obj->secs[new_cnt - 1];
 531         sec->id = new_cnt - 1;
 532         sec->sec_name = sec_name;
 533
 534         return sec;
 535 }
 536
 537 static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
 538                                 const struct bpf_linker_file_opts *opts,
 539                                 struct src_obj *obj)
 540 {
 541         int err = 0;
 542         Elf_Scn *scn;
 543         Elf_Data *data;
 544         Elf64_Ehdr *ehdr;
 545         Elf64_Shdr *shdr;
 546         struct src_sec *sec;
 547         unsigned char obj_byteorder;
 548         unsigned char link_byteorder = linker->elf_hdr->e_ident[EI_DATA];
 549 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 550         const unsigned char host_byteorder = ELFDATA2LSB;
 551 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
 552         const unsigned char host_byteorder = ELFDATA2MSB;
 553 #else
 554 #error "Unknown __BYTE_ORDER__"
 555 #endif
 556
 557         pr_debug("linker: adding object file '%s'...\n", filename);
 558
 559         obj->filename = filename;
 560
 561         obj->fd = open(filename, O_RDONLY | O_CLOEXEC);
 562         if (obj->fd < 0) {
 563                 err = -errno;
 564                 pr_warn("failed to open file '%s': %s\n", filename, errstr(err));
 565                 return err;
 566         }
 567         obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL);
 568         if (!obj->elf) {
 569                 err = -errno;
 570                 pr_warn_elf("failed to parse ELF file '%s'", filename);
 571                 return err;
 572         }
 573
 574         /* Sanity check ELF file high-level properties */
 575         ehdr = elf64_getehdr(obj->elf);
 576         if (!ehdr) {
 577                 err = -errno;
 578                 pr_warn_elf("failed to get ELF header for %s", filename);
 579                 return err;
 580         }
 581
 582         /* Linker output endianness set by first input object */
 583         obj_byteorder = ehdr->e_ident[EI_DATA];
 584         if (obj_byteorder != ELFDATA2LSB && obj_byteorder != ELFDATA2MSB) {
 585                 err = -EOPNOTSUPP;
 586                 pr_warn("unknown byte order of ELF file %s\n", filename);
 587                 return err;
 588         }
 589         if (link_byteorder == ELFDATANONE) {
 590                 linker->elf_hdr->e_ident[EI_DATA] = obj_byteorder;
 591                 linker->swapped_endian = obj_byteorder != host_byteorder;
 592                 pr_debug("linker: set %s-endian output byte order\n",
 593                          obj_byteorder == ELFDATA2MSB ? "big" : "little");
 594         } else if (link_byteorder != obj_byteorder) {
 595                 err = -EOPNOTSUPP;
 596                 pr_warn("byte order mismatch with ELF file %s\n", filename);
 597                 return err;
 598         }
 599
 600         if (ehdr->e_type != ET_REL
 601             || ehdr->e_machine != EM_BPF
 602             || ehdr->e_ident[EI_CLASS] != ELFCLASS64) {
 603                 err = -EOPNOTSUPP;
 604                 pr_warn_elf("unsupported kind of ELF file %s", filename);
 605                 return err;
 606         }
 607
 608         if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) {
 609                 err = -errno;
 610                 pr_warn_elf("failed to get SHSTRTAB section index for %s", filename);
 611                 return err;
 612         }
 613
 614         scn = NULL;
 615         while ((scn = elf_nextscn(obj->elf, scn)) != NULL) {
 616                 size_t sec_idx = elf_ndxscn(scn);
 617                 const char *sec_name;
 618
 619                 shdr = elf64_getshdr(scn);
 620                 if (!shdr) {
 621                         err = -errno;
 622                         pr_warn_elf("failed to get section #%zu header for %s",
 623                                     sec_idx, filename);
 624                         return err;
 625                 }
 626
 627                 sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name);
 628                 if (!sec_name) {
 629                         err = -errno;
 630                         pr_warn_elf("failed to get section #%zu name for %s",
 631                                     sec_idx, filename);
 632                         return err;
 633                 }
 634
 635                 data = elf_getdata(scn, 0);
 636                 if (!data) {
 637                         err = -errno;
 638                         pr_warn_elf("failed to get section #%zu (%s) data from %s",
 639                                     sec_idx, sec_name, filename);
 640                         return err;
 641                 }
 642
 643                 sec = add_src_sec(obj, sec_name);
 644                 if (!sec)
 645                         return -ENOMEM;
 646
 647                 sec->scn = scn;
 648                 sec->shdr = shdr;
 649                 sec->data = data;
 650                 sec->sec_idx = elf_ndxscn(scn);
 651
 652                 if (is_ignored_sec(sec)) {
 653                         sec->skipped = true;
 654                         continue;
 655                 }
 656
 657                 switch (shdr->sh_type) {
 658                 case SHT_SYMTAB:
 659                         if (obj->symtab_sec_idx) {
 660                                 err = -EOPNOTSUPP;
 661                                 pr_warn("multiple SYMTAB sections found, not supported\n");
 662                                 return err;
 663                         }
 664                         obj->symtab_sec_idx = sec_idx;
 665                         break;
 666                 case SHT_STRTAB:
 667                         /* we'll construct our own string table */
 668                         break;
 669                 case SHT_PROGBITS:
 670                         if (strcmp(sec_name, BTF_ELF_SEC) == 0) {
 671                                 obj->btf = btf__new(data->d_buf, shdr->sh_size);
 672                                 err = libbpf_get_error(obj->btf);
 673                                 if (err) {
 674                                         pr_warn("failed to parse .BTF from %s: %s\n",
 675                                                 filename, errstr(err));
 676                                         return err;
 677                                 }
 678                                 sec->skipped = true;
 679                                 continue;
 680                         }
 681                         if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) {
 682                                 obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size);
 683                                 err = libbpf_get_error(obj->btf_ext);
 684                                 if (err) {
 685                                         pr_warn("failed to parse .BTF.ext from '%s': %s\n",
 686                                                 filename, errstr(err));
 687                                         return err;
 688                                 }
 689                                 sec->skipped = true;
 690                                 continue;
 691                         }
 692
 693                         /* data & code */
 694                         break;
 695                 case SHT_NOBITS:
 696                         /* BSS */
 697                         break;
 698                 case SHT_REL:
 699                         /* relocations */
 700                         break;
 701                 default:
 702                         pr_warn("unrecognized section #%zu (%s) in %s\n",
 703                                 sec_idx, sec_name, filename);
 704                         err = -EINVAL;
 705                         return err;
 706                 }
 707         }
 708
 709         err = err ?: linker_sanity_check_elf(obj);
 710         err = err ?: linker_sanity_check_btf(obj);
 711         err = err ?: linker_sanity_check_btf_ext(obj);
 712         err = err ?: linker_fixup_btf(obj);
 713
 714         return err;
 715 }
 716
 717 static int linker_sanity_check_elf(struct src_obj *obj)
 718 {
 719         struct src_sec *sec;
 720         int i, err;
 721
 722         if (!obj->symtab_sec_idx) {
 723                 pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename);
 724                 return -EINVAL;
 725         }
 726         if (!obj->shstrs_sec_idx) {
 727                 pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename);
 728                 return -EINVAL;
 729         }
 730
 731         for (i = 1; i < obj->sec_cnt; i++) {
 732                 sec = &obj->secs[i];
 733
 734                 if (sec->sec_name[0] == '\0') {
 735                         pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename);
 736                         return -EINVAL;
 737                 }
 738
 739                 if (is_dwarf_sec_name(sec->sec_name))
 740                         continue;
 741
 742                 if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign)) {
 743                         pr_warn("ELF section #%zu alignment %llu is non pow-of-2 alignment in %s\n",
 744                                 sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign,
 745                                 obj->filename);
 746                         return -EINVAL;
 747                 }
 748                 if (sec->shdr->sh_addralign != sec->data->d_align) {
 749                         pr_warn("ELF section #%zu has inconsistent alignment addr=%llu != d=%llu in %s\n",
 750                                 sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign,
 751                                 (long long unsigned)sec->data->d_align, obj->filename);
 752                         return -EINVAL;
 753                 }
 754
 755                 if (sec->shdr->sh_size != sec->data->d_size) {
 756                         pr_warn("ELF section #%zu has inconsistent section size sh=%llu != d=%llu in %s\n",
 757                                 sec->sec_idx, (long long unsigned)sec->shdr->sh_size,
 758                                 (long long unsigned)sec->data->d_size, obj->filename);
 759                         return -EINVAL;
 760                 }
 761
 762                 switch (sec->shdr->sh_type) {
 763                 case SHT_SYMTAB:
 764                         err = linker_sanity_check_elf_symtab(obj, sec);
 765                         if (err)
 766                                 return err;
 767                         break;
 768                 case SHT_STRTAB:
 769                         break;
 770                 case SHT_PROGBITS:
 771                         if (sec->shdr->sh_flags & SHF_EXECINSTR) {
 772                                 if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0) {
 773                                         pr_warn("ELF section #%zu has unexpected size alignment %llu in %s\n",
 774                                                 sec->sec_idx, (long long unsigned)sec->shdr->sh_size,
 775                                                 obj->filename);
 776                                         return -EINVAL;
 777                                 }
 778                         }
 779                         break;
 780                 case SHT_NOBITS:
 781                         break;
 782                 case SHT_REL:
 783                         err = linker_sanity_check_elf_relos(obj, sec);
 784                         if (err)
 785                                 return err;
 786                         break;
 787                 case SHT_LLVM_ADDRSIG:
 788                         break;
 789                 default:
 790                         pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n",
 791                                 sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename);
 792                         return -EINVAL;
 793                 }
 794         }
 795
 796         return 0;
 797 }
 798
 799 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec)
 800 {
 801         struct src_sec *link_sec;
 802         Elf64_Sym *sym;
 803         int i, n;
 804
 805         if (sec->shdr->sh_entsize != sizeof(Elf64_Sym))
 806                 return -EINVAL;
 807         if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
 808                 return -EINVAL;
 809
 810         if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) {
 811                 pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n",
 812                         sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
 813                 return -EINVAL;
 814         }
 815         link_sec = &obj->secs[sec->shdr->sh_link];
 816         if (link_sec->shdr->sh_type != SHT_STRTAB) {
 817                 pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n",
 818                         sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
 819                 return -EINVAL;
 820         }
 821
 822         n = sec->shdr->sh_size / sec->shdr->sh_entsize;
 823         sym = sec->data->d_buf;
 824         for (i = 0; i < n; i++, sym++) {
 825                 int sym_type = ELF64_ST_TYPE(sym->st_info);
 826                 int sym_bind = ELF64_ST_BIND(sym->st_info);
 827                 int sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
 828
 829                 if (i == 0) {
 830                         if (sym->st_name != 0 || sym->st_info != 0
 831                             || sym->st_other != 0 || sym->st_shndx != 0
 832                             || sym->st_value != 0 || sym->st_size != 0) {
 833                                 pr_warn("ELF sym #0 is invalid in %s\n", obj->filename);
 834                                 return -EINVAL;
 835                         }
 836                         continue;
 837                 }
 838                 if (sym_bind != STB_LOCAL && sym_bind != STB_GLOBAL && sym_bind != STB_WEAK) {
 839                         pr_warn("ELF sym #%d in section #%zu has unsupported symbol binding %d\n",
 840                                 i, sec->sec_idx, sym_bind);
 841                         return -EINVAL;
 842                 }
 843                 if (sym_vis != STV_DEFAULT && sym_vis != STV_HIDDEN) {
 844                         pr_warn("ELF sym #%d in section #%zu has unsupported symbol visibility %d\n",
 845                                 i, sec->sec_idx, sym_vis);
 846                         return -EINVAL;
 847                 }
 848                 if (sym->st_shndx == 0) {
 849                         if (sym_type != STT_NOTYPE || sym_bind == STB_LOCAL
 850                             || sym->st_value != 0 || sym->st_size != 0) {
 851                                 pr_warn("ELF sym #%d is invalid extern symbol in %s\n",
 852                                         i, obj->filename);
 853
 854                                 return -EINVAL;
 855                         }
 856                         continue;
 857                 }
 858                 if (sym->st_shndx < SHN_LORESERVE && sym->st_shndx >= obj->sec_cnt) {
 859                         pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n",
 860                                 i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename);
 861                         return -EINVAL;
 862                 }
 863                 if (sym_type == STT_SECTION) {
 864                         if (sym->st_value != 0)
 865                                 return -EINVAL;
 866                         continue;
 867                 }
 868         }
 869
 870         return 0;
 871 }
 872
 873 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec)
 874 {
 875         struct src_sec *link_sec, *sym_sec;
 876         Elf64_Rel *relo;
 877         int i, n;
 878
 879         if (sec->shdr->sh_entsize != sizeof(Elf64_Rel))
 880                 return -EINVAL;
 881         if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
 882                 return -EINVAL;
 883
 884         /* SHT_REL's sh_link should point to SYMTAB */
 885         if (sec->shdr->sh_link != obj->symtab_sec_idx) {
 886                 pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n",
 887                         sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
 888                 return -EINVAL;
 889         }
 890
 891         /* SHT_REL's sh_info points to relocated section */
 892         if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) {
 893                 pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n",
 894                         sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
 895                 return -EINVAL;
 896         }
 897         link_sec = &obj->secs[sec->shdr->sh_info];
 898
 899         /* .rel<secname> -> <secname> pattern is followed */
 900         if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0
 901             || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) {
 902                 pr_warn("ELF relo section #%zu name has invalid name in %s\n",
 903                         sec->sec_idx, obj->filename);
 904                 return -EINVAL;
 905         }
 906
 907         /* don't further validate relocations for ignored sections */
 908         if (link_sec->skipped)
 909                 return 0;
 910
 911         /* relocatable section is data or instructions */
 912         if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) {
 913                 pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n",
 914                         sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
 915                 return -EINVAL;
 916         }
 917
 918         /* check sanity of each relocation */
 919         n = sec->shdr->sh_size / sec->shdr->sh_entsize;
 920         relo = sec->data->d_buf;
 921         sym_sec = &obj->secs[obj->symtab_sec_idx];
 922         for (i = 0; i < n; i++, relo++) {
 923                 size_t sym_idx = ELF64_R_SYM(relo->r_info);
 924                 size_t sym_type = ELF64_R_TYPE(relo->r_info);
 925
 926                 if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32 &&
 927                     sym_type != R_BPF_64_ABS64 && sym_type != R_BPF_64_ABS32) {
 928                         pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n",
 929                                 i, sec->sec_idx, sym_type, obj->filename);
 930                         return -EINVAL;
 931                 }
 932
 933                 if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) {
 934                         pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n",
 935                                 i, sec->sec_idx, sym_idx, obj->filename);
 936                         return -EINVAL;
 937                 }
 938
 939                 if (link_sec->shdr->sh_flags & SHF_EXECINSTR) {
 940                         if (relo->r_offset % sizeof(struct bpf_insn) != 0) {
 941                                 pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n",
 942                                         i, sec->sec_idx, sym_idx, obj->filename);
 943                                 return -EINVAL;
 944                         }
 945                 }
 946         }
 947
 948         return 0;
 949 }
 950
 951 static int check_btf_type_id(__u32 *type_id, void *ctx)
 952 {
 953         struct btf *btf = ctx;
 954
 955         if (*type_id >= btf__type_cnt(btf))
 956                 return -EINVAL;
 957
 958         return 0;
 959 }
 960
 961 static int check_btf_str_off(__u32 *str_off, void *ctx)
 962 {
 963         struct btf *btf = ctx;
 964         const char *s;
 965
 966         s = btf__str_by_offset(btf, *str_off);
 967
 968         if (!s)
 969                 return -EINVAL;
 970
 971         return 0;
 972 }
 973
 974 static int linker_sanity_check_btf(struct src_obj *obj)
 975 {
 976         struct btf_type *t;
 977         int i, n, err;
 978
 979         if (!obj->btf)
 980                 return 0;
 981
 982         n = btf__type_cnt(obj->btf);
 983         for (i = 1; i < n; i++) {
 984                 struct btf_field_iter it;
 985                 __u32 *type_id, *str_off;
 986
 987                 t = btf_type_by_id(obj->btf, i);
 988
 989                 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS);
 990                 if (err)
 991                         return err;
 992                 while ((type_id = btf_field_iter_next(&it))) {
 993                         if (*type_id >= n)
 994                                 return -EINVAL;
 995                 }
 996
 997                 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS);
 998                 if (err)
 999                         return err;
1000                 while ((str_off = btf_field_iter_next(&it))) {
1001                         if (!btf__str_by_offset(obj->btf, *str_off))
1002                                 return -EINVAL;
1003                 }
1004         }
1005
1006         return 0;
1007 }
1008
1009 static int linker_sanity_check_btf_ext(struct src_obj *obj)
1010 {
1011         int err = 0;
1012
1013         if (!obj->btf_ext)
1014                 return 0;
1015
1016         /* can't use .BTF.ext without .BTF */
1017         if (!obj->btf)
1018                 return -EINVAL;
1019
1020         err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf);
1021         err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf);
1022         if (err)
1023                 return err;
1024
1025         return 0;
1026 }
1027
1028 static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec)
1029 {
1030         Elf_Scn *scn;
1031         Elf_Data *data;
1032         Elf64_Shdr *shdr;
1033         int name_off;
1034
1035         dst_sec->sec_sz = 0;
1036         dst_sec->sec_idx = 0;
1037         dst_sec->ephemeral = src_sec->ephemeral;
1038
1039         /* ephemeral sections are just thin section shells lacking most parts */
1040         if (src_sec->ephemeral)
1041                 return 0;
1042
1043         scn = elf_newscn(linker->elf);
1044         if (!scn)
1045                 return -ENOMEM;
1046         data = elf_newdata(scn);
1047         if (!data)
1048                 return -ENOMEM;
1049         shdr = elf64_getshdr(scn);
1050         if (!shdr)
1051                 return -ENOMEM;
1052
1053         dst_sec->scn = scn;
1054         dst_sec->shdr = shdr;
1055         dst_sec->data = data;
1056         dst_sec->sec_idx = elf_ndxscn(scn);
1057
1058         name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name);
1059         if (name_off < 0)
1060                 return name_off;
1061
1062         shdr->sh_name = name_off;
1063         shdr->sh_type = src_sec->shdr->sh_type;
1064         shdr->sh_flags = src_sec->shdr->sh_flags;
1065         shdr->sh_size = 0;
1066         /* sh_link and sh_info have different meaning for different types of
1067          * sections, so we leave it up to the caller code to fill them in, if
1068          * necessary
1069          */
1070         shdr->sh_link = 0;
1071         shdr->sh_info = 0;
1072         shdr->sh_addralign = src_sec->shdr->sh_addralign;
1073         shdr->sh_entsize = src_sec->shdr->sh_entsize;
1074
1075         data->d_type = src_sec->data->d_type;
1076         data->d_size = 0;
1077         data->d_buf = NULL;
1078         data->d_align = src_sec->data->d_align;
1079         data->d_off = 0;
1080
1081         return 0;
1082 }
1083
1084 static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name)
1085 {
1086         struct dst_sec *sec;
1087         int i;
1088
1089         for (i = 1; i < linker->sec_cnt; i++) {
1090                 sec = &linker->secs[i];
1091
1092                 if (strcmp(sec->sec_name, sec_name) == 0)
1093                         return sec;
1094         }
1095
1096         return NULL;
1097 }
1098
1099 static bool secs_match(struct dst_sec *dst, struct src_sec *src)
1100 {
1101         if (dst->ephemeral || src->ephemeral)
1102                 return true;
1103
1104         if (dst->shdr->sh_type != src->shdr->sh_type) {
1105                 pr_warn("sec %s types mismatch\n", dst->sec_name);
1106                 return false;
1107         }
1108         if (dst->shdr->sh_flags != src->shdr->sh_flags) {
1109                 pr_warn("sec %s flags mismatch\n", dst->sec_name);
1110                 return false;
1111         }
1112         if (dst->shdr->sh_entsize != src->shdr->sh_entsize) {
1113                 pr_warn("sec %s entsize mismatch\n", dst->sec_name);
1114                 return false;
1115         }
1116
1117         return true;
1118 }
1119
1120 static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec)
1121 {
1122         if (dst_sec->sec_sz != src_sec->shdr->sh_size)
1123                 return false;
1124         if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0)
1125                 return false;
1126         return true;
1127 }
1128
1129 static bool is_exec_sec(struct dst_sec *sec)
1130 {
1131         if (!sec || sec->ephemeral)
1132                 return false;
1133         return (sec->shdr->sh_type == SHT_PROGBITS) &&
1134                (sec->shdr->sh_flags & SHF_EXECINSTR);
1135 }
1136
1137 static void exec_sec_bswap(void *raw_data, int size)
1138 {
1139         const int insn_cnt = size / sizeof(struct bpf_insn);
1140         struct bpf_insn *insn = raw_data;
1141         int i;
1142
1143         for (i = 0; i < insn_cnt; i++, insn++)
1144                 bpf_insn_bswap(insn);
1145 }
1146
1147 static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src_sec *src)
1148 {
1149         void *tmp;
1150         size_t dst_align, src_align;
1151         size_t dst_align_sz, dst_final_sz;
1152         int err;
1153
1154         /* Ephemeral source section doesn't contribute anything to ELF
1155          * section data.
1156          */
1157         if (src->ephemeral)
1158                 return 0;
1159
1160         /* Some sections (like .maps) can contain both externs (and thus be
1161          * ephemeral) and non-externs (map definitions). So it's possible that
1162          * it has to be "upgraded" from ephemeral to non-ephemeral when the
1163          * first non-ephemeral entity appears. In such case, we add ELF
1164          * section, data, etc.
1165          */
1166         if (dst->ephemeral) {
1167                 err = init_sec(linker, dst, src);
1168                 if (err)
1169                         return err;
1170         }
1171
1172         dst_align = dst->shdr->sh_addralign;
1173         src_align = src->shdr->sh_addralign;
1174         if (dst_align == 0)
1175                 dst_align = 1;
1176         if (dst_align < src_align)
1177                 dst_align = src_align;
1178
1179         dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align;
1180
1181         /* no need to re-align final size */
1182         dst_final_sz = dst_align_sz + src->shdr->sh_size;
1183
1184         if (src->shdr->sh_type != SHT_NOBITS) {
1185                 tmp = realloc(dst->raw_data, dst_final_sz);
1186                 /* If dst_align_sz == 0, realloc() behaves in a special way:
1187                  * 1. When dst->raw_data is NULL it returns:
1188                  *    "either NULL or a pointer suitable to be passed to free()" [1].
1189                  * 2. When dst->raw_data is not-NULL it frees dst->raw_data and returns NULL,
1190                  *    thus invalidating any "pointer suitable to be passed to free()" obtained
1191                  *    at step (1).
1192                  *
1193                  * The dst_align_sz > 0 check avoids error exit after (2), otherwise
1194                  * dst->raw_data would be freed again in bpf_linker__free().
1195                  *
1196                  * [1] man 3 realloc
1197                  */
1198                 if (!tmp && dst_align_sz > 0)
1199                         return -ENOMEM;
1200                 dst->raw_data = tmp;
1201
1202                 /* pad dst section, if it's alignment forced size increase */
1203                 memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz);
1204                 /* now copy src data at a properly aligned offset */
1205                 memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size);
1206
1207                 /* convert added bpf insns to native byte-order */
1208                 if (linker->swapped_endian && is_exec_sec(dst))
1209                         exec_sec_bswap(dst->raw_data + dst_align_sz, src->shdr->sh_size);
1210         }
1211
1212         dst->sec_sz = dst_final_sz;
1213         dst->shdr->sh_size = dst_final_sz;
1214         dst->data->d_size = dst_final_sz;
1215
1216         dst->shdr->sh_addralign = dst_align;
1217         dst->data->d_align = dst_align;
1218
1219         src->dst_off = dst_align_sz;
1220
1221         return 0;
1222 }
1223
1224 static bool is_data_sec(struct src_sec *sec)
1225 {
1226         if (!sec || sec->skipped)
1227                 return false;
1228         /* ephemeral sections are data sections, e.g., .kconfig, .ksyms */
1229         if (sec->ephemeral)
1230                 return true;
1231         return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS;
1232 }
1233
1234 static bool is_relo_sec(struct src_sec *sec)
1235 {
1236         if (!sec || sec->skipped || sec->ephemeral)
1237                 return false;
1238         return sec->shdr->sh_type == SHT_REL;
1239 }
1240
1241 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj)
1242 {
1243         int i, err;
1244
1245         for (i = 1; i < obj->sec_cnt; i++) {
1246                 struct src_sec *src_sec;
1247                 struct dst_sec *dst_sec;
1248
1249                 src_sec = &obj->secs[i];
1250                 if (!is_data_sec(src_sec))
1251                         continue;
1252
1253                 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
1254                 if (!dst_sec) {
1255                         dst_sec = add_dst_sec(linker, src_sec->sec_name);
1256                         if (!dst_sec)
1257                                 return -ENOMEM;
1258                         err = init_sec(linker, dst_sec, src_sec);
1259                         if (err) {
1260                                 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
1261                                 return err;
1262                         }
1263                 } else {
1264                         if (!secs_match(dst_sec, src_sec)) {
1265                                 pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name);
1266                                 return -1;
1267                         }
1268
1269                         /* "license" and "version" sections are deduped */
1270                         if (strcmp(src_sec->sec_name, "license") == 0
1271                             || strcmp(src_sec->sec_name, "version") == 0) {
1272                                 if (!sec_content_is_same(dst_sec, src_sec)) {
1273                                         pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name);
1274                                         return -EINVAL;
1275                                 }
1276                                 src_sec->skipped = true;
1277                                 src_sec->dst_id = dst_sec->id;
1278                                 continue;
1279                         }
1280                 }
1281
1282                 /* record mapped section index */
1283                 src_sec->dst_id = dst_sec->id;
1284
1285                 err = extend_sec(linker, dst_sec, src_sec);
1286                 if (err)
1287                         return err;
1288         }
1289
1290         return 0;
1291 }
1292
1293 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj)
1294 {
1295         struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
1296         Elf64_Sym *sym = symtab->data->d_buf;
1297         int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize, err;
1298         int str_sec_idx = symtab->shdr->sh_link;
1299         const char *sym_name;
1300
1301         obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map));
1302         if (!obj->sym_map)
1303                 return -ENOMEM;
1304
1305         for (i = 0; i < n; i++, sym++) {
1306                 /* We already validated all-zero symbol #0 and we already
1307                  * appended it preventively to the final SYMTAB, so skip it.
1308                  */
1309                 if (i == 0)
1310                         continue;
1311
1312                 sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
1313                 if (!sym_name) {
1314                         pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename);
1315                         return -EINVAL;
1316                 }
1317
1318                 err = linker_append_elf_sym(linker, obj, sym, sym_name, i);
1319                 if (err)
1320                         return err;
1321         }
1322
1323         return 0;
1324 }
1325
1326 static Elf64_Sym *get_sym_by_idx(struct bpf_linker *linker, size_t sym_idx)
1327 {
1328         struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
1329         Elf64_Sym *syms = symtab->raw_data;
1330
1331         return &syms[sym_idx];
1332 }
1333
1334 static struct glob_sym *find_glob_sym(struct bpf_linker *linker, const char *sym_name)
1335 {
1336         struct glob_sym *glob_sym;
1337         const char *name;
1338         int i;
1339
1340         for (i = 0; i < linker->glob_sym_cnt; i++) {
1341                 glob_sym = &linker->glob_syms[i];
1342                 name = strset__data(linker->strtab_strs) + glob_sym->name_off;
1343
1344                 if (strcmp(name, sym_name) == 0)
1345                         return glob_sym;
1346         }
1347
1348         return NULL;
1349 }
1350
1351 static struct glob_sym *add_glob_sym(struct bpf_linker *linker)
1352 {
1353         struct glob_sym *syms, *sym;
1354
1355         syms = libbpf_reallocarray(linker->glob_syms, linker->glob_sym_cnt + 1,
1356                                    sizeof(*linker->glob_syms));
1357         if (!syms)
1358                 return NULL;
1359
1360         sym = &syms[linker->glob_sym_cnt];
1361         memset(sym, 0, sizeof(*sym));
1362         sym->var_idx = -1;
1363
1364         linker->glob_syms = syms;
1365         linker->glob_sym_cnt++;
1366
1367         return sym;
1368 }
1369
1370 static bool glob_sym_btf_matches(const char *sym_name, bool exact,
1371                                  const struct btf *btf1, __u32 id1,
1372                                  const struct btf *btf2, __u32 id2)
1373 {
1374         const struct btf_type *t1, *t2;
1375         bool is_static1, is_static2;
1376         const char *n1, *n2;
1377         int i, n;
1378
1379 recur:
1380         n1 = n2 = NULL;
1381         t1 = skip_mods_and_typedefs(btf1, id1, &id1);
1382         t2 = skip_mods_and_typedefs(btf2, id2, &id2);
1383
1384         /* check if only one side is FWD, otherwise handle with common logic */
1385         if (!exact && btf_is_fwd(t1) != btf_is_fwd(t2)) {
1386                 n1 = btf__str_by_offset(btf1, t1->name_off);
1387                 n2 = btf__str_by_offset(btf2, t2->name_off);
1388                 if (strcmp(n1, n2) != 0) {
1389                         pr_warn("global '%s': incompatible forward declaration names '%s' and '%s'\n",
1390                                 sym_name, n1, n2);
1391                         return false;
1392                 }
1393                 /* validate if FWD kind matches concrete kind */
1394                 if (btf_is_fwd(t1)) {
1395                         if (btf_kflag(t1) && btf_is_union(t2))
1396                                 return true;
1397                         if (!btf_kflag(t1) && btf_is_struct(t2))
1398                                 return true;
1399                         pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1400                                 sym_name, btf_kflag(t1) ? "union" : "struct", btf_kind_str(t2));
1401                 } else {
1402                         if (btf_kflag(t2) && btf_is_union(t1))
1403                                 return true;
1404                         if (!btf_kflag(t2) && btf_is_struct(t1))
1405                                 return true;
1406                         pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1407                                 sym_name, btf_kflag(t2) ? "union" : "struct", btf_kind_str(t1));
1408                 }
1409                 return false;
1410         }
1411
1412         if (btf_kind(t1) != btf_kind(t2)) {
1413                 pr_warn("global '%s': incompatible BTF kinds %s and %s\n",
1414                         sym_name, btf_kind_str(t1), btf_kind_str(t2));
1415                 return false;
1416         }
1417
1418         switch (btf_kind(t1)) {
1419         case BTF_KIND_STRUCT:
1420         case BTF_KIND_UNION:
1421         case BTF_KIND_ENUM:
1422         case BTF_KIND_ENUM64:
1423         case BTF_KIND_FWD:
1424         case BTF_KIND_FUNC:
1425         case BTF_KIND_VAR:
1426                 n1 = btf__str_by_offset(btf1, t1->name_off);
1427                 n2 = btf__str_by_offset(btf2, t2->name_off);
1428                 if (strcmp(n1, n2) != 0) {
1429                         pr_warn("global '%s': incompatible %s names '%s' and '%s'\n",
1430                                 sym_name, btf_kind_str(t1), n1, n2);
1431                         return false;
1432                 }
1433                 break;
1434         default:
1435                 break;
1436         }
1437
1438         switch (btf_kind(t1)) {
1439         case BTF_KIND_UNKN: /* void */
1440         case BTF_KIND_FWD:
1441                 return true;
1442         case BTF_KIND_INT:
1443         case BTF_KIND_FLOAT:
1444         case BTF_KIND_ENUM:
1445         case BTF_KIND_ENUM64:
1446                 /* ignore encoding for int and enum values for enum */
1447                 if (t1->size != t2->size) {
1448                         pr_warn("global '%s': incompatible %s '%s' size %u and %u\n",
1449                                 sym_name, btf_kind_str(t1), n1, t1->size, t2->size);
1450                         return false;
1451                 }
1452                 return true;
1453         case BTF_KIND_PTR:
1454                 /* just validate overall shape of the referenced type, so no
1455                  * contents comparison for struct/union, and allowed fwd vs
1456                  * struct/union
1457                  */
1458                 exact = false;
1459                 id1 = t1->type;
1460                 id2 = t2->type;
1461                 goto recur;
1462         case BTF_KIND_ARRAY:
1463                 /* ignore index type and array size */
1464                 id1 = btf_array(t1)->type;
1465                 id2 = btf_array(t2)->type;
1466                 goto recur;
1467         case BTF_KIND_FUNC:
1468                 /* extern and global linkages are compatible */
1469                 is_static1 = btf_func_linkage(t1) == BTF_FUNC_STATIC;
1470                 is_static2 = btf_func_linkage(t2) == BTF_FUNC_STATIC;
1471                 if (is_static1 != is_static2) {
1472                         pr_warn("global '%s': incompatible func '%s' linkage\n", sym_name, n1);
1473                         return false;
1474                 }
1475
1476                 id1 = t1->type;
1477                 id2 = t2->type;
1478                 goto recur;
1479         case BTF_KIND_VAR:
1480                 /* extern and global linkages are compatible */
1481                 is_static1 = btf_var(t1)->linkage == BTF_VAR_STATIC;
1482                 is_static2 = btf_var(t2)->linkage == BTF_VAR_STATIC;
1483                 if (is_static1 != is_static2) {
1484                         pr_warn("global '%s': incompatible var '%s' linkage\n", sym_name, n1);
1485                         return false;
1486                 }
1487
1488                 id1 = t1->type;
1489                 id2 = t2->type;
1490                 goto recur;
1491         case BTF_KIND_STRUCT:
1492         case BTF_KIND_UNION: {
1493                 const struct btf_member *m1, *m2;
1494
1495                 if (!exact)
1496                         return true;
1497
1498                 if (btf_vlen(t1) != btf_vlen(t2)) {
1499                         pr_warn("global '%s': incompatible number of %s fields %u and %u\n",
1500                                 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1501                         return false;
1502                 }
1503
1504                 n = btf_vlen(t1);
1505                 m1 = btf_members(t1);
1506                 m2 = btf_members(t2);
1507                 for (i = 0; i < n; i++, m1++, m2++) {
1508                         n1 = btf__str_by_offset(btf1, m1->name_off);
1509                         n2 = btf__str_by_offset(btf2, m2->name_off);
1510                         if (strcmp(n1, n2) != 0) {
1511                                 pr_warn("global '%s': incompatible field #%d names '%s' and '%s'\n",
1512                                         sym_name, i, n1, n2);
1513                                 return false;
1514                         }
1515                         if (m1->offset != m2->offset) {
1516                                 pr_warn("global '%s': incompatible field #%d ('%s') offsets\n",
1517                                         sym_name, i, n1);
1518                                 return false;
1519                         }
1520                         if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1521                                 return false;
1522                 }
1523
1524                 return true;
1525         }
1526         case BTF_KIND_FUNC_PROTO: {
1527                 const struct btf_param *m1, *m2;
1528
1529                 if (btf_vlen(t1) != btf_vlen(t2)) {
1530                         pr_warn("global '%s': incompatible number of %s params %u and %u\n",
1531                                 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1532                         return false;
1533                 }
1534
1535                 n = btf_vlen(t1);
1536                 m1 = btf_params(t1);
1537                 m2 = btf_params(t2);
1538                 for (i = 0; i < n; i++, m1++, m2++) {
1539                         /* ignore func arg names */
1540                         if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1541                                 return false;
1542                 }
1543
1544                 /* now check return type as well */
1545                 id1 = t1->type;
1546                 id2 = t2->type;
1547                 goto recur;
1548         }
1549
1550         /* skip_mods_and_typedefs() make this impossible */
1551         case BTF_KIND_TYPEDEF:
1552         case BTF_KIND_VOLATILE:
1553         case BTF_KIND_CONST:
1554         case BTF_KIND_RESTRICT:
1555         /* DATASECs are never compared with each other */
1556         case BTF_KIND_DATASEC:
1557         default:
1558                 pr_warn("global '%s': unsupported BTF kind %s\n",
1559                         sym_name, btf_kind_str(t1));
1560                 return false;
1561         }
1562 }
1563
1564 static bool map_defs_match(const char *sym_name,
1565                            const struct btf *main_btf,
1566                            const struct btf_map_def *main_def,
1567                            const struct btf_map_def *main_inner_def,
1568                            const struct btf *extra_btf,
1569                            const struct btf_map_def *extra_def,
1570                            const struct btf_map_def *extra_inner_def)
1571 {
1572         const char *reason;
1573
1574         if (main_def->map_type != extra_def->map_type) {
1575                 reason = "type";
1576                 goto mismatch;
1577         }
1578
1579         /* check key type/size match */
1580         if (main_def->key_size != extra_def->key_size) {
1581                 reason = "key_size";
1582                 goto mismatch;
1583         }
1584         if (!!main_def->key_type_id != !!extra_def->key_type_id) {
1585                 reason = "key type";
1586                 goto mismatch;
1587         }
1588         if ((main_def->parts & MAP_DEF_KEY_TYPE)
1589              && !glob_sym_btf_matches(sym_name, true /*exact*/,
1590                                       main_btf, main_def->key_type_id,
1591                                       extra_btf, extra_def->key_type_id)) {
1592                 reason = "key type";
1593                 goto mismatch;
1594         }
1595
1596         /* validate value type/size match */
1597         if (main_def->value_size != extra_def->value_size) {
1598                 reason = "value_size";
1599                 goto mismatch;
1600         }
1601         if (!!main_def->value_type_id != !!extra_def->value_type_id) {
1602                 reason = "value type";
1603                 goto mismatch;
1604         }
1605         if ((main_def->parts & MAP_DEF_VALUE_TYPE)
1606              && !glob_sym_btf_matches(sym_name, true /*exact*/,
1607                                       main_btf, main_def->value_type_id,
1608                                       extra_btf, extra_def->value_type_id)) {
1609                 reason = "key type";
1610                 goto mismatch;
1611         }
1612
1613         if (main_def->max_entries != extra_def->max_entries) {
1614                 reason = "max_entries";
1615                 goto mismatch;
1616         }
1617         if (main_def->map_flags != extra_def->map_flags) {
1618                 reason = "map_flags";
1619                 goto mismatch;
1620         }
1621         if (main_def->numa_node != extra_def->numa_node) {
1622                 reason = "numa_node";
1623                 goto mismatch;
1624         }
1625         if (main_def->pinning != extra_def->pinning) {
1626                 reason = "pinning";
1627                 goto mismatch;
1628         }
1629
1630         if ((main_def->parts & MAP_DEF_INNER_MAP) != (extra_def->parts & MAP_DEF_INNER_MAP)) {
1631                 reason = "inner map";
1632                 goto mismatch;
1633         }
1634
1635         if (main_def->parts & MAP_DEF_INNER_MAP) {
1636                 char inner_map_name[128];
1637
1638                 snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", sym_name);
1639
1640                 return map_defs_match(inner_map_name,
1641                                       main_btf, main_inner_def, NULL,
1642                                       extra_btf, extra_inner_def, NULL);
1643         }
1644
1645         return true;
1646
1647 mismatch:
1648         pr_warn("global '%s': map %s mismatch\n", sym_name, reason);
1649         return false;
1650 }
1651
1652 static bool glob_map_defs_match(const char *sym_name,
1653                                 struct bpf_linker *linker, struct glob_sym *glob_sym,
1654                                 struct src_obj *obj, Elf64_Sym *sym, int btf_id)
1655 {
1656         struct btf_map_def dst_def = {}, dst_inner_def = {};
1657         struct btf_map_def src_def = {}, src_inner_def = {};
1658         const struct btf_type *t;
1659         int err;
1660
1661         t = btf__type_by_id(obj->btf, btf_id);
1662         if (!btf_is_var(t)) {
1663                 pr_warn("global '%s': invalid map definition type [%d]\n", sym_name, btf_id);
1664                 return false;
1665         }
1666         t = skip_mods_and_typedefs(obj->btf, t->type, NULL);
1667
1668         err = parse_btf_map_def(sym_name, obj->btf, t, true /*strict*/, &src_def, &src_inner_def);
1669         if (err) {
1670                 pr_warn("global '%s': invalid map definition\n", sym_name);
1671                 return false;
1672         }
1673
1674         /* re-parse existing map definition */
1675         t = btf__type_by_id(linker->btf, glob_sym->btf_id);
1676         t = skip_mods_and_typedefs(linker->btf, t->type, NULL);
1677         err = parse_btf_map_def(sym_name, linker->btf, t, true /*strict*/, &dst_def, &dst_inner_def);
1678         if (err) {
1679                 /* this should not happen, because we already validated it */
1680                 pr_warn("global '%s': invalid dst map definition\n", sym_name);
1681                 return false;
1682         }
1683
1684         /* Currently extern map definition has to be complete and match
1685          * concrete map definition exactly. This restriction might be lifted
1686          * in the future.
1687          */
1688         return map_defs_match(sym_name, linker->btf, &dst_def, &dst_inner_def,
1689                               obj->btf, &src_def, &src_inner_def);
1690 }
1691
1692 static bool glob_syms_match(const char *sym_name,
1693                             struct bpf_linker *linker, struct glob_sym *glob_sym,
1694                             struct src_obj *obj, Elf64_Sym *sym, size_t sym_idx, int btf_id)
1695 {
1696         const struct btf_type *src_t;
1697
1698         /* if we are dealing with externs, BTF types describing both global
1699          * and extern VARs/FUNCs should be completely present in all files
1700          */
1701         if (!glob_sym->btf_id || !btf_id) {
1702                 pr_warn("BTF info is missing for global symbol '%s'\n", sym_name);
1703                 return false;
1704         }
1705
1706         src_t = btf__type_by_id(obj->btf, btf_id);
1707         if (!btf_is_var(src_t) && !btf_is_func(src_t)) {
1708                 pr_warn("only extern variables and functions are supported, but got '%s' for '%s'\n",
1709                         btf_kind_str(src_t), sym_name);
1710                 return false;
1711         }
1712
1713         /* deal with .maps definitions specially */
1714         if (glob_sym->sec_id && strcmp(linker->secs[glob_sym->sec_id].sec_name, MAPS_ELF_SEC) == 0)
1715                 return glob_map_defs_match(sym_name, linker, glob_sym, obj, sym, btf_id);
1716
1717         if (!glob_sym_btf_matches(sym_name, true /*exact*/,
1718                                   linker->btf, glob_sym->btf_id, obj->btf, btf_id))
1719                 return false;
1720
1721         return true;
1722 }
1723
1724 static bool btf_is_non_static(const struct btf_type *t)
1725 {
1726         return (btf_is_var(t) && btf_var(t)->linkage != BTF_VAR_STATIC)
1727                || (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_STATIC);
1728 }
1729
1730 static int find_glob_sym_btf(struct src_obj *obj, Elf64_Sym *sym, const char *sym_name,
1731                              int *out_btf_sec_id, int *out_btf_id)
1732 {
1733         int i, j, n, m, btf_id = 0;
1734         const struct btf_type *t;
1735         const struct btf_var_secinfo *vi;
1736         const char *name;
1737
1738         if (!obj->btf) {
1739                 pr_warn("failed to find BTF info for object '%s'\n", obj->filename);
1740                 return -EINVAL;
1741         }
1742
1743         n = btf__type_cnt(obj->btf);
1744         for (i = 1; i < n; i++) {
1745                 t = btf__type_by_id(obj->btf, i);
1746
1747                 /* some global and extern FUNCs and VARs might not be associated with any
1748                  * DATASEC, so try to detect them in the same pass
1749                  */
1750                 if (btf_is_non_static(t)) {
1751                         name = btf__str_by_offset(obj->btf, t->name_off);
1752                         if (strcmp(name, sym_name) != 0)
1753                                 continue;
1754
1755                         /* remember and still try to find DATASEC */
1756                         btf_id = i;
1757                         continue;
1758                 }
1759
1760                 if (!btf_is_datasec(t))
1761                         continue;
1762
1763                 vi = btf_var_secinfos(t);
1764                 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
1765                         t = btf__type_by_id(obj->btf, vi->type);
1766                         name = btf__str_by_offset(obj->btf, t->name_off);
1767
1768                         if (strcmp(name, sym_name) != 0)
1769                                 continue;
1770                         if (btf_is_var(t) && btf_var(t)->linkage == BTF_VAR_STATIC)
1771                                 continue;
1772                         if (btf_is_func(t) && btf_func_linkage(t) == BTF_FUNC_STATIC)
1773                                 continue;
1774
1775                         if (btf_id && btf_id != vi->type) {
1776                                 pr_warn("global/extern '%s' BTF is ambiguous: both types #%d and #%u match\n",
1777                                         sym_name, btf_id, vi->type);
1778                                 return -EINVAL;
1779                         }
1780
1781                         *out_btf_sec_id = i;
1782                         *out_btf_id = vi->type;
1783
1784                         return 0;
1785                 }
1786         }
1787
1788         /* free-floating extern or global FUNC */
1789         if (btf_id) {
1790                 *out_btf_sec_id = 0;
1791                 *out_btf_id = btf_id;
1792                 return 0;
1793         }
1794
1795         pr_warn("failed to find BTF info for global/extern symbol '%s'\n", sym_name);
1796         return -ENOENT;
1797 }
1798
1799 static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name)
1800 {
1801         struct src_sec *sec;
1802         int i;
1803
1804         for (i = 1; i < obj->sec_cnt; i++) {
1805                 sec = &obj->secs[i];
1806
1807                 if (strcmp(sec->sec_name, sec_name) == 0)
1808                         return sec;
1809         }
1810
1811         return NULL;
1812 }
1813
1814 static int complete_extern_btf_info(struct btf *dst_btf, int dst_id,
1815                                     struct btf *src_btf, int src_id)
1816 {
1817         struct btf_type *dst_t = btf_type_by_id(dst_btf, dst_id);
1818         struct btf_type *src_t = btf_type_by_id(src_btf, src_id);
1819         struct btf_param *src_p, *dst_p;
1820         const char *s;
1821         int i, n, off;
1822
1823         /* We already made sure that source and destination types (FUNC or
1824          * VAR) match in terms of types and argument names.
1825          */
1826         if (btf_is_var(dst_t)) {
1827                 btf_var(dst_t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
1828                 return 0;
1829         }
1830
1831         dst_t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_GLOBAL, 0);
1832
1833         /* now onto FUNC_PROTO types */
1834         src_t = btf_type_by_id(src_btf, src_t->type);
1835         dst_t = btf_type_by_id(dst_btf, dst_t->type);
1836
1837         /* Fill in all the argument names, which for extern FUNCs are missing.
1838          * We'll end up with two copies of FUNCs/VARs for externs, but that
1839          * will be taken care of by BTF dedup at the very end.
1840          * It might be that BTF types for extern in one file has less/more BTF
1841          * information (e.g., FWD instead of full STRUCT/UNION information),
1842          * but that should be (in most cases, subject to BTF dedup rules)
1843          * handled and resolved by BTF dedup algorithm as well, so we won't
1844          * worry about it. Our only job is to make sure that argument names
1845          * are populated on both sides, otherwise BTF dedup will pedantically
1846          * consider them different.
1847          */
1848         src_p = btf_params(src_t);
1849         dst_p = btf_params(dst_t);
1850         for (i = 0, n = btf_vlen(dst_t); i < n; i++, src_p++, dst_p++) {
1851                 if (!src_p->name_off)
1852                         continue;
1853
1854                 /* src_btf has more complete info, so add name to dst_btf */
1855                 s = btf__str_by_offset(src_btf, src_p->name_off);
1856                 off = btf__add_str(dst_btf, s);
1857                 if (off < 0)
1858                         return off;
1859                 dst_p->name_off = off;
1860         }
1861         return 0;
1862 }
1863
1864 static void sym_update_bind(Elf64_Sym *sym, int sym_bind)
1865 {
1866         sym->st_info = ELF64_ST_INFO(sym_bind, ELF64_ST_TYPE(sym->st_info));
1867 }
1868
1869 static void sym_update_type(Elf64_Sym *sym, int sym_type)
1870 {
1871         sym->st_info = ELF64_ST_INFO(ELF64_ST_BIND(sym->st_info), sym_type);
1872 }
1873
1874 static void sym_update_visibility(Elf64_Sym *sym, int sym_vis)
1875 {
1876         /* libelf doesn't provide setters for ST_VISIBILITY,
1877          * but it is stored in the lower 2 bits of st_other
1878          */
1879         sym->st_other &= ~0x03;
1880         sym->st_other |= sym_vis;
1881 }
1882
1883 static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
1884                                  Elf64_Sym *sym, const char *sym_name, int src_sym_idx)
1885 {
1886         struct src_sec *src_sec = NULL;
1887         struct dst_sec *dst_sec = NULL;
1888         struct glob_sym *glob_sym = NULL;
1889         int name_off, sym_type, sym_bind, sym_vis, err;
1890         int btf_sec_id = 0, btf_id = 0;
1891         size_t dst_sym_idx;
1892         Elf64_Sym *dst_sym;
1893         bool sym_is_extern;
1894
1895         sym_type = ELF64_ST_TYPE(sym->st_info);
1896         sym_bind = ELF64_ST_BIND(sym->st_info);
1897         sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
1898         sym_is_extern = sym->st_shndx == SHN_UNDEF;
1899
1900         if (sym_is_extern) {
1901                 if (!obj->btf) {
1902                         pr_warn("externs without BTF info are not supported\n");
1903                         return -ENOTSUP;
1904                 }
1905         } else if (sym->st_shndx < SHN_LORESERVE) {
1906                 src_sec = &obj->secs[sym->st_shndx];
1907                 if (src_sec->skipped)
1908                         return 0;
1909                 dst_sec = &linker->secs[src_sec->dst_id];
1910
1911                 /* allow only one STT_SECTION symbol per section */
1912                 if (sym_type == STT_SECTION && dst_sec->sec_sym_idx) {
1913                         obj->sym_map[src_sym_idx] = dst_sec->sec_sym_idx;
1914                         return 0;
1915                 }
1916         }
1917
1918         if (sym_bind == STB_LOCAL)
1919                 goto add_sym;
1920
1921         /* find matching BTF info */
1922         err = find_glob_sym_btf(obj, sym, sym_name, &btf_sec_id, &btf_id);
1923         if (err)
1924                 return err;
1925
1926         if (sym_is_extern && btf_sec_id) {
1927                 const char *sec_name = NULL;
1928                 const struct btf_type *t;
1929
1930                 t = btf__type_by_id(obj->btf, btf_sec_id);
1931                 sec_name = btf__str_by_offset(obj->btf, t->name_off);
1932
1933                 /* Clang puts unannotated extern vars into
1934                  * '.extern' BTF DATASEC. Treat them the same
1935                  * as unannotated extern funcs (which are
1936                  * currently not put into any DATASECs).
1937                  * Those don't have associated src_sec/dst_sec.
1938                  */
1939                 if (strcmp(sec_name, BTF_EXTERN_SEC) != 0) {
1940                         src_sec = find_src_sec_by_name(obj, sec_name);
1941                         if (!src_sec) {
1942                                 pr_warn("failed to find matching ELF sec '%s'\n", sec_name);
1943                                 return -ENOENT;
1944                         }
1945                         dst_sec = &linker->secs[src_sec->dst_id];
1946                 }
1947         }
1948
1949         glob_sym = find_glob_sym(linker, sym_name);
1950         if (glob_sym) {
1951                 /* Preventively resolve to existing symbol. This is
1952                  * needed for further relocation symbol remapping in
1953                  * the next step of linking.
1954                  */
1955                 obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
1956
1957                 /* If both symbols are non-externs, at least one of
1958                  * them has to be STB_WEAK, otherwise they are in
1959                  * a conflict with each other.
1960                  */
1961                 if (!sym_is_extern && !glob_sym->is_extern
1962                     && !glob_sym->is_weak && sym_bind != STB_WEAK) {
1963                         pr_warn("conflicting non-weak symbol #%d (%s) definition in '%s'\n",
1964                                 src_sym_idx, sym_name, obj->filename);
1965                         return -EINVAL;
1966                 }
1967
1968                 if (!glob_syms_match(sym_name, linker, glob_sym, obj, sym, src_sym_idx, btf_id))
1969                         return -EINVAL;
1970
1971                 dst_sym = get_sym_by_idx(linker, glob_sym->sym_idx);
1972
1973                 /* If new symbol is strong, then force dst_sym to be strong as
1974                  * well; this way a mix of weak and non-weak extern
1975                  * definitions will end up being strong.
1976                  */
1977                 if (sym_bind == STB_GLOBAL) {
1978                         /* We still need to preserve type (NOTYPE or
1979                          * OBJECT/FUNC, depending on whether the symbol is
1980                          * extern or not)
1981                          */
1982                         sym_update_bind(dst_sym, STB_GLOBAL);
1983                         glob_sym->is_weak = false;
1984                 }
1985
1986                 /* Non-default visibility is "contaminating", with stricter
1987                  * visibility overwriting more permissive ones, even if more
1988                  * permissive visibility comes from just an extern definition.
1989                  * Currently only STV_DEFAULT and STV_HIDDEN are allowed and
1990                  * ensured by ELF symbol sanity checks above.
1991                  */
1992                 if (sym_vis > ELF64_ST_VISIBILITY(dst_sym->st_other))
1993                         sym_update_visibility(dst_sym, sym_vis);
1994
1995                 /* If the new symbol is extern, then regardless if
1996                  * existing symbol is extern or resolved global, just
1997                  * keep the existing one untouched.
1998                  */
1999                 if (sym_is_extern)
2000                         return 0;
2001
2002                 /* If existing symbol is a strong resolved symbol, bail out,
2003                  * because we lost resolution battle have nothing to
2004                  * contribute. We already checked above that there is no
2005                  * strong-strong conflict. We also already tightened binding
2006                  * and visibility, so nothing else to contribute at that point.
2007                  */
2008                 if (!glob_sym->is_extern && sym_bind == STB_WEAK)
2009                         return 0;
2010
2011                 /* At this point, new symbol is strong non-extern,
2012                  * so overwrite glob_sym with new symbol information.
2013                  * Preserve binding and visibility.
2014                  */
2015                 sym_update_type(dst_sym, sym_type);
2016                 dst_sym->st_shndx = dst_sec->sec_idx;
2017                 dst_sym->st_value = src_sec->dst_off + sym->st_value;
2018                 dst_sym->st_size = sym->st_size;
2019
2020                 /* see comment below about dst_sec->id vs dst_sec->sec_idx */
2021                 glob_sym->sec_id = dst_sec->id;
2022                 glob_sym->is_extern = false;
2023
2024                 if (complete_extern_btf_info(linker->btf, glob_sym->btf_id,
2025                                              obj->btf, btf_id))
2026                         return -EINVAL;
2027
2028                 /* request updating VAR's/FUNC's underlying BTF type when appending BTF type */
2029                 glob_sym->underlying_btf_id = 0;
2030
2031                 obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
2032                 return 0;
2033         }
2034
2035 add_sym:
2036         name_off = strset__add_str(linker->strtab_strs, sym_name);
2037         if (name_off < 0)
2038                 return name_off;
2039
2040         dst_sym = add_new_sym(linker, &dst_sym_idx);
2041         if (!dst_sym)
2042                 return -ENOMEM;
2043
2044         dst_sym->st_name = name_off;
2045         dst_sym->st_info = sym->st_info;
2046         dst_sym->st_other = sym->st_other;
2047         dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx;
2048         dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value;
2049         dst_sym->st_size = sym->st_size;
2050
2051         obj->sym_map[src_sym_idx] = dst_sym_idx;
2052
2053         if (sym_type == STT_SECTION && dst_sym) {
2054                 dst_sec->sec_sym_idx = dst_sym_idx;
2055                 dst_sym->st_value = 0;
2056         }
2057
2058         if (sym_bind != STB_LOCAL) {
2059                 glob_sym = add_glob_sym(linker);
2060                 if (!glob_sym)
2061                         return -ENOMEM;
2062
2063                 glob_sym->sym_idx = dst_sym_idx;
2064                 /* we use dst_sec->id (and not dst_sec->sec_idx), because
2065                  * ephemeral sections (.kconfig, .ksyms, etc) don't have
2066                  * sec_idx (as they don't have corresponding ELF section), but
2067                  * still have id. .extern doesn't have even ephemeral section
2068                  * associated with it, so dst_sec->id == dst_sec->sec_idx == 0.
2069                  */
2070                 glob_sym->sec_id = dst_sec ? dst_sec->id : 0;
2071                 glob_sym->name_off = name_off;
2072                 /* we will fill btf_id in during BTF merging step */
2073                 glob_sym->btf_id = 0;
2074                 glob_sym->is_extern = sym_is_extern;
2075                 glob_sym->is_weak = sym_bind == STB_WEAK;
2076         }
2077
2078         return 0;
2079 }
2080
2081 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj)
2082 {
2083         struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx];
2084         int i, err;
2085
2086         for (i = 1; i < obj->sec_cnt; i++) {
2087                 struct src_sec *src_sec, *src_linked_sec;
2088                 struct dst_sec *dst_sec, *dst_linked_sec;
2089                 Elf64_Rel *src_rel, *dst_rel;
2090                 int j, n;
2091
2092                 src_sec = &obj->secs[i];
2093                 if (!is_relo_sec(src_sec))
2094                         continue;
2095
2096                 /* shdr->sh_info points to relocatable section */
2097                 src_linked_sec = &obj->secs[src_sec->shdr->sh_info];
2098                 if (src_linked_sec->skipped)
2099                         continue;
2100
2101                 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
2102                 if (!dst_sec) {
2103                         dst_sec = add_dst_sec(linker, src_sec->sec_name);
2104                         if (!dst_sec)
2105                                 return -ENOMEM;
2106                         err = init_sec(linker, dst_sec, src_sec);
2107                         if (err) {
2108                                 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
2109                                 return err;
2110                         }
2111                 } else if (!secs_match(dst_sec, src_sec)) {
2112                         pr_warn("sections %s are not compatible\n", src_sec->sec_name);
2113                         return -1;
2114                 }
2115
2116                 /* shdr->sh_link points to SYMTAB */
2117                 dst_sec->shdr->sh_link = linker->symtab_sec_idx;
2118
2119                 /* shdr->sh_info points to relocated section */
2120                 dst_linked_sec = &linker->secs[src_linked_sec->dst_id];
2121                 dst_sec->shdr->sh_info = dst_linked_sec->sec_idx;
2122
2123                 src_sec->dst_id = dst_sec->id;
2124                 err = extend_sec(linker, dst_sec, src_sec);
2125                 if (err)
2126                         return err;
2127
2128                 src_rel = src_sec->data->d_buf;
2129                 dst_rel = dst_sec->raw_data + src_sec->dst_off;
2130                 n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize;
2131                 for (j = 0; j < n; j++, src_rel++, dst_rel++) {
2132                         size_t src_sym_idx, dst_sym_idx, sym_type;
2133                         Elf64_Sym *src_sym;
2134
2135                         src_sym_idx = ELF64_R_SYM(src_rel->r_info);
2136                         src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx;
2137
2138                         dst_sym_idx = obj->sym_map[src_sym_idx];
2139                         dst_rel->r_offset += src_linked_sec->dst_off;
2140                         sym_type = ELF64_R_TYPE(src_rel->r_info);
2141                         dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type);
2142
2143                         if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) {
2144                                 struct src_sec *sec = &obj->secs[src_sym->st_shndx];
2145                                 struct bpf_insn *insn;
2146
2147                                 if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) {
2148                                         /* calls to the very first static function inside
2149                                          * .text section at offset 0 will
2150                                          * reference section symbol, not the
2151                                          * function symbol. Fix that up,
2152                                          * otherwise it won't be possible to
2153                                          * relocate calls to two different
2154                                          * static functions with the same name
2155                                          * (rom two different object files)
2156                                          */
2157                                         insn = dst_linked_sec->raw_data + dst_rel->r_offset;
2158                                         if (insn->code == (BPF_JMP | BPF_CALL))
2159                                                 insn->imm += sec->dst_off / sizeof(struct bpf_insn);
2160                                         else
2161                                                 insn->imm += sec->dst_off;
2162                                 } else {
2163                                         pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n");
2164                                         return -EINVAL;
2165                                 }
2166                         }
2167
2168                 }
2169         }
2170
2171         return 0;
2172 }
2173
2174 static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx,
2175                                    int sym_type, const char *sym_name)
2176 {
2177         struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
2178         Elf64_Sym *sym = symtab->data->d_buf;
2179         int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize;
2180         int str_sec_idx = symtab->shdr->sh_link;
2181         const char *name;
2182
2183         for (i = 0; i < n; i++, sym++) {
2184                 if (sym->st_shndx != sec_idx)
2185                         continue;
2186                 if (ELF64_ST_TYPE(sym->st_info) != sym_type)
2187                         continue;
2188
2189                 name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
2190                 if (!name)
2191                         return NULL;
2192
2193                 if (strcmp(sym_name, name) != 0)
2194                         continue;
2195
2196                 return sym;
2197         }
2198
2199         return NULL;
2200 }
2201
2202 static int linker_fixup_btf(struct src_obj *obj)
2203 {
2204         const char *sec_name;
2205         struct src_sec *sec;
2206         int i, j, n, m;
2207
2208         if (!obj->btf)
2209                 return 0;
2210
2211         n = btf__type_cnt(obj->btf);
2212         for (i = 1; i < n; i++) {
2213                 struct btf_var_secinfo *vi;
2214                 struct btf_type *t;
2215
2216                 t = btf_type_by_id(obj->btf, i);
2217                 if (btf_kind(t) != BTF_KIND_DATASEC)
2218                         continue;
2219
2220                 sec_name = btf__str_by_offset(obj->btf, t->name_off);
2221                 sec = find_src_sec_by_name(obj, sec_name);
2222                 if (sec) {
2223                         /* record actual section size, unless ephemeral */
2224                         if (sec->shdr)
2225                                 t->size = sec->shdr->sh_size;
2226                 } else {
2227                         /* BTF can have some sections that are not represented
2228                          * in ELF, e.g., .kconfig, .ksyms, .extern, which are used
2229                          * for special extern variables.
2230                          *
2231                          * For all but one such special (ephemeral)
2232                          * sections, we pre-create "section shells" to be able
2233                          * to keep track of extra per-section metadata later
2234                          * (e.g., those BTF extern variables).
2235                          *
2236                          * .extern is even more special, though, because it
2237                          * contains extern variables that need to be resolved
2238                          * by static linker, not libbpf and kernel. When such
2239                          * externs are resolved, we are going to remove them
2240                          * from .extern BTF section and might end up not
2241                          * needing it at all. Each resolved extern should have
2242                          * matching non-extern VAR/FUNC in other sections.
2243                          *
2244                          * We do support leaving some of the externs
2245                          * unresolved, though, to support cases of building
2246                          * libraries, which will later be linked against final
2247                          * BPF applications. So if at finalization we still
2248                          * see unresolved externs, we'll create .extern
2249                          * section on our own.
2250                          */
2251                         if (strcmp(sec_name, BTF_EXTERN_SEC) == 0)
2252                                 continue;
2253
2254                         sec = add_src_sec(obj, sec_name);
2255                         if (!sec)
2256                                 return -ENOMEM;
2257
2258                         sec->ephemeral = true;
2259                         sec->sec_idx = 0; /* will match UNDEF shndx in ELF */
2260                 }
2261
2262                 /* remember ELF section and its BTF type ID match */
2263                 sec->sec_type_id = i;
2264
2265                 /* fix up variable offsets */
2266                 vi = btf_var_secinfos(t);
2267                 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
2268                         const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type);
2269                         const char *var_name;
2270                         int var_linkage;
2271                         Elf64_Sym *sym;
2272
2273                         /* could be a variable or function */
2274                         if (!btf_is_var(vt))
2275                                 continue;
2276
2277                         var_name = btf__str_by_offset(obj->btf, vt->name_off);
2278                         var_linkage = btf_var(vt)->linkage;
2279
2280                         /* no need to patch up static or extern vars */
2281                         if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED)
2282                                 continue;
2283
2284                         sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name);
2285                         if (!sym) {
2286                                 pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name);
2287                                 return -ENOENT;
2288                         }
2289
2290                         vi->offset = sym->st_value;
2291                 }
2292         }
2293
2294         return 0;
2295 }
2296
2297 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj)
2298 {
2299         const struct btf_type *t;
2300         int i, j, n, start_id, id, err;
2301         const char *name;
2302
2303         if (!obj->btf)
2304                 return 0;
2305
2306         start_id = btf__type_cnt(linker->btf);
2307         n = btf__type_cnt(obj->btf);
2308
2309         obj->btf_type_map = calloc(n + 1, sizeof(int));
2310         if (!obj->btf_type_map)
2311                 return -ENOMEM;
2312
2313         for (i = 1; i < n; i++) {
2314                 struct glob_sym *glob_sym = NULL;
2315
2316                 t = btf__type_by_id(obj->btf, i);
2317
2318                 /* DATASECs are handled specially below */
2319                 if (btf_kind(t) == BTF_KIND_DATASEC)
2320                         continue;
2321
2322                 if (btf_is_non_static(t)) {
2323                         /* there should be glob_sym already */
2324                         name = btf__str_by_offset(obj->btf, t->name_off);
2325                         glob_sym = find_glob_sym(linker, name);
2326
2327                         /* VARs without corresponding glob_sym are those that
2328                          * belong to skipped/deduplicated sections (i.e.,
2329                          * license and version), so just skip them
2330                          */
2331                         if (!glob_sym)
2332                                 continue;
2333
2334                         /* linker_append_elf_sym() might have requested
2335                          * updating underlying type ID, if extern was resolved
2336                          * to strong symbol or weak got upgraded to non-weak
2337                          */
2338                         if (glob_sym->underlying_btf_id == 0)
2339                                 glob_sym->underlying_btf_id = -t->type;
2340
2341                         /* globals from previous object files that match our
2342                          * VAR/FUNC already have a corresponding associated
2343                          * BTF type, so just make sure to use it
2344                          */
2345                         if (glob_sym->btf_id) {
2346                                 /* reuse existing BTF type for global var/func */
2347                                 obj->btf_type_map[i] = glob_sym->btf_id;
2348                                 continue;
2349                         }
2350                 }
2351
2352                 id = btf__add_type(linker->btf, obj->btf, t);
2353                 if (id < 0) {
2354                         pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename);
2355                         return id;
2356                 }
2357
2358                 obj->btf_type_map[i] = id;
2359
2360                 /* record just appended BTF type for var/func */
2361                 if (glob_sym) {
2362                         glob_sym->btf_id = id;
2363                         glob_sym->underlying_btf_id = -t->type;
2364                 }
2365         }
2366
2367         /* remap all the types except DATASECs */
2368         n = btf__type_cnt(linker->btf);
2369         for (i = start_id; i < n; i++) {
2370                 struct btf_type *dst_t = btf_type_by_id(linker->btf, i);
2371                 struct btf_field_iter it;
2372                 __u32 *type_id;
2373
2374                 err = btf_field_iter_init(&it, dst_t, BTF_FIELD_ITER_IDS);
2375                 if (err)
2376                         return err;
2377
2378                 while ((type_id = btf_field_iter_next(&it))) {
2379                         int new_id = obj->btf_type_map[*type_id];
2380
2381                         /* Error out if the type wasn't remapped. Ignore VOID which stays VOID. */
2382                         if (new_id == 0 && *type_id != 0) {
2383                                 pr_warn("failed to find new ID mapping for original BTF type ID %u\n",
2384                                         *type_id);
2385                                 return -EINVAL;
2386                         }
2387
2388                         *type_id = obj->btf_type_map[*type_id];
2389                 }
2390         }
2391
2392         /* Rewrite VAR/FUNC underlying types (i.e., FUNC's FUNC_PROTO and VAR's
2393          * actual type), if necessary
2394          */
2395         for (i = 0; i < linker->glob_sym_cnt; i++) {
2396                 struct glob_sym *glob_sym = &linker->glob_syms[i];
2397                 struct btf_type *glob_t;
2398
2399                 if (glob_sym->underlying_btf_id >= 0)
2400                         continue;
2401
2402                 glob_sym->underlying_btf_id = obj->btf_type_map[-glob_sym->underlying_btf_id];
2403
2404                 glob_t = btf_type_by_id(linker->btf, glob_sym->btf_id);
2405                 glob_t->type = glob_sym->underlying_btf_id;
2406         }
2407
2408         /* append DATASEC info */
2409         for (i = 1; i < obj->sec_cnt; i++) {
2410                 struct src_sec *src_sec;
2411                 struct dst_sec *dst_sec;
2412                 const struct btf_var_secinfo *src_var;
2413                 struct btf_var_secinfo *dst_var;
2414
2415                 src_sec = &obj->secs[i];
2416                 if (!src_sec->sec_type_id || src_sec->skipped)
2417                         continue;
2418                 dst_sec = &linker->secs[src_sec->dst_id];
2419
2420                 /* Mark section as having BTF regardless of the presence of
2421                  * variables. In some cases compiler might generate empty BTF
2422                  * with no variables information. E.g., when promoting local
2423                  * array/structure variable initial values and BPF object
2424                  * file otherwise has no read-only static variables in
2425                  * .rodata. We need to preserve such empty BTF and just set
2426                  * correct section size.
2427                  */
2428                 dst_sec->has_btf = true;
2429
2430                 t = btf__type_by_id(obj->btf, src_sec->sec_type_id);
2431                 src_var = btf_var_secinfos(t);
2432                 n = btf_vlen(t);
2433                 for (j = 0; j < n; j++, src_var++) {
2434                         void *sec_vars = dst_sec->sec_vars;
2435                         int new_id = obj->btf_type_map[src_var->type];
2436                         struct glob_sym *glob_sym = NULL;
2437
2438                         t = btf_type_by_id(linker->btf, new_id);
2439                         if (btf_is_non_static(t)) {
2440                                 name = btf__str_by_offset(linker->btf, t->name_off);
2441                                 glob_sym = find_glob_sym(linker, name);
2442                                 if (glob_sym->sec_id != dst_sec->id) {
2443                                         pr_warn("global '%s': section mismatch %d vs %d\n",
2444                                                 name, glob_sym->sec_id, dst_sec->id);
2445                                         return -EINVAL;
2446                                 }
2447                         }
2448
2449                         /* If there is already a member (VAR or FUNC) mapped
2450                          * to the same type, don't add a duplicate entry.
2451                          * This will happen when multiple object files define
2452                          * the same extern VARs/FUNCs.
2453                          */
2454                         if (glob_sym && glob_sym->var_idx >= 0) {
2455                                 __s64 sz;
2456
2457                                 /* FUNCs don't have size, nothing to update */
2458                                 if (btf_is_func(t))
2459                                         continue;
2460
2461                                 dst_var = &dst_sec->sec_vars[glob_sym->var_idx];
2462                                 /* Because underlying BTF type might have
2463                                  * changed, so might its size have changed, so
2464                                  * re-calculate and update it in sec_var.
2465                                  */
2466                                 sz = btf__resolve_size(linker->btf, glob_sym->underlying_btf_id);
2467                                 if (sz < 0) {
2468                                         pr_warn("global '%s': failed to resolve size of underlying type: %d\n",
2469                                                 name, (int)sz);
2470                                         return -EINVAL;
2471                                 }
2472                                 dst_var->size = sz;
2473                                 continue;
2474                         }
2475
2476                         sec_vars = libbpf_reallocarray(sec_vars,
2477                                                        dst_sec->sec_var_cnt + 1,
2478                                                        sizeof(*dst_sec->sec_vars));
2479                         if (!sec_vars)
2480                                 return -ENOMEM;
2481
2482                         dst_sec->sec_vars = sec_vars;
2483                         dst_sec->sec_var_cnt++;
2484
2485                         dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1];
2486                         dst_var->type = obj->btf_type_map[src_var->type];
2487                         dst_var->size = src_var->size;
2488                         dst_var->offset = src_sec->dst_off + src_var->offset;
2489
2490                         if (glob_sym)
2491                                 glob_sym->var_idx = dst_sec->sec_var_cnt - 1;
2492                 }
2493         }
2494
2495         return 0;
2496 }
2497
2498 static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec)
2499 {
2500         void *tmp;
2501
2502         tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz);
2503         if (!tmp)
2504                 return NULL;
2505         ext_data->recs = tmp;
2506
2507         tmp += ext_data->rec_cnt * ext_data->rec_sz;
2508         memcpy(tmp, src_rec, ext_data->rec_sz);
2509
2510         ext_data->rec_cnt++;
2511
2512         return tmp;
2513 }
2514
2515 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj)
2516 {
2517         const struct btf_ext_info_sec *ext_sec;
2518         const char *sec_name, *s;
2519         struct src_sec *src_sec;
2520         struct dst_sec *dst_sec;
2521         int rec_sz, str_off, i;
2522
2523         if (!obj->btf_ext)
2524                 return 0;
2525
2526         rec_sz = obj->btf_ext->func_info.rec_size;
2527         for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) {
2528                 struct bpf_func_info_min *src_rec, *dst_rec;
2529
2530                 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2531                 src_sec = find_src_sec_by_name(obj, sec_name);
2532                 if (!src_sec) {
2533                         pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2534                         return -EINVAL;
2535                 }
2536                 dst_sec = &linker->secs[src_sec->dst_id];
2537
2538                 if (dst_sec->func_info.rec_sz == 0)
2539                         dst_sec->func_info.rec_sz = rec_sz;
2540                 if (dst_sec->func_info.rec_sz != rec_sz) {
2541                         pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2542                         return -EINVAL;
2543                 }
2544
2545                 for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) {
2546                         dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec);
2547                         if (!dst_rec)
2548                                 return -ENOMEM;
2549
2550                         dst_rec->insn_off += src_sec->dst_off;
2551                         dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2552                 }
2553         }
2554
2555         rec_sz = obj->btf_ext->line_info.rec_size;
2556         for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) {
2557                 struct bpf_line_info_min *src_rec, *dst_rec;
2558
2559                 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2560                 src_sec = find_src_sec_by_name(obj, sec_name);
2561                 if (!src_sec) {
2562                         pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2563                         return -EINVAL;
2564                 }
2565                 dst_sec = &linker->secs[src_sec->dst_id];
2566
2567                 if (dst_sec->line_info.rec_sz == 0)
2568                         dst_sec->line_info.rec_sz = rec_sz;
2569                 if (dst_sec->line_info.rec_sz != rec_sz) {
2570                         pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2571                         return -EINVAL;
2572                 }
2573
2574                 for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) {
2575                         dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec);
2576                         if (!dst_rec)
2577                                 return -ENOMEM;
2578
2579                         dst_rec->insn_off += src_sec->dst_off;
2580
2581                         s = btf__str_by_offset(obj->btf, src_rec->file_name_off);
2582                         str_off = btf__add_str(linker->btf, s);
2583                         if (str_off < 0)
2584                                 return -ENOMEM;
2585                         dst_rec->file_name_off = str_off;
2586
2587                         s = btf__str_by_offset(obj->btf, src_rec->line_off);
2588                         str_off = btf__add_str(linker->btf, s);
2589                         if (str_off < 0)
2590                                 return -ENOMEM;
2591                         dst_rec->line_off = str_off;
2592
2593                         /* dst_rec->line_col is fine */
2594                 }
2595         }
2596
2597         rec_sz = obj->btf_ext->core_relo_info.rec_size;
2598         for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) {
2599                 struct bpf_core_relo *src_rec, *dst_rec;
2600
2601                 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2602                 src_sec = find_src_sec_by_name(obj, sec_name);
2603                 if (!src_sec) {
2604                         pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2605                         return -EINVAL;
2606                 }
2607                 dst_sec = &linker->secs[src_sec->dst_id];
2608
2609                 if (dst_sec->core_relo_info.rec_sz == 0)
2610                         dst_sec->core_relo_info.rec_sz = rec_sz;
2611                 if (dst_sec->core_relo_info.rec_sz != rec_sz) {
2612                         pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2613                         return -EINVAL;
2614                 }
2615
2616                 for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) {
2617                         dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec);
2618                         if (!dst_rec)
2619                                 return -ENOMEM;
2620
2621                         dst_rec->insn_off += src_sec->dst_off;
2622                         dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2623
2624                         s = btf__str_by_offset(obj->btf, src_rec->access_str_off);
2625                         str_off = btf__add_str(linker->btf, s);
2626                         if (str_off < 0)
2627                                 return -ENOMEM;
2628                         dst_rec->access_str_off = str_off;
2629
2630                         /* dst_rec->kind is fine */
2631                 }
2632         }
2633
2634         return 0;
2635 }
2636
2637 int bpf_linker__finalize(struct bpf_linker *linker)
2638 {
2639         struct dst_sec *sec;
2640         size_t strs_sz;
2641         const void *strs;
2642         int err, i;
2643
2644         if (!linker->elf)
2645                 return libbpf_err(-EINVAL);
2646
2647         err = finalize_btf(linker);
2648         if (err)
2649                 return libbpf_err(err);
2650
2651         /* Finalize strings */
2652         strs_sz = strset__data_size(linker->strtab_strs);
2653         strs = strset__data(linker->strtab_strs);
2654
2655         sec = &linker->secs[linker->strtab_sec_idx];
2656         sec->data->d_align = 1;
2657         sec->data->d_off = 0LL;
2658         sec->data->d_buf = (void *)strs;
2659         sec->data->d_type = ELF_T_BYTE;
2660         sec->data->d_size = strs_sz;
2661         sec->shdr->sh_size = strs_sz;
2662
2663         for (i = 1; i < linker->sec_cnt; i++) {
2664                 sec = &linker->secs[i];
2665
2666                 /* STRTAB is handled specially above */
2667                 if (sec->sec_idx == linker->strtab_sec_idx)
2668                         continue;
2669
2670                 /* special ephemeral sections (.ksyms, .kconfig, etc) */
2671                 if (!sec->scn)
2672                         continue;
2673
2674                 /* restore sections with bpf insns to target byte-order */
2675                 if (linker->swapped_endian && is_exec_sec(sec))
2676                         exec_sec_bswap(sec->raw_data, sec->sec_sz);
2677
2678                 sec->data->d_buf = sec->raw_data;
2679         }
2680
2681         /* Finalize ELF layout */
2682         if (elf_update(linker->elf, ELF_C_NULL) < 0) {
2683                 err = -errno;
2684                 pr_warn_elf("failed to finalize ELF layout");
2685                 return libbpf_err(err);
2686         }
2687
2688         /* Write out final ELF contents */
2689         if (elf_update(linker->elf, ELF_C_WRITE) < 0) {
2690                 err = -errno;
2691                 pr_warn_elf("failed to write ELF contents");
2692                 return libbpf_err(err);
2693         }
2694
2695         elf_end(linker->elf);
2696         close(linker->fd);
2697
2698         linker->elf = NULL;
2699         linker->fd = -1;
2700
2701         return 0;
2702 }
2703
2704 static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name,
2705                              size_t align, const void *raw_data, size_t raw_sz)
2706 {
2707         Elf_Scn *scn;
2708         Elf_Data *data;
2709         Elf64_Shdr *shdr;
2710         int name_off;
2711
2712         name_off = strset__add_str(linker->strtab_strs, sec_name);
2713         if (name_off < 0)
2714                 return name_off;
2715
2716         scn = elf_newscn(linker->elf);
2717         if (!scn)
2718                 return -ENOMEM;
2719         data = elf_newdata(scn);
2720         if (!data)
2721                 return -ENOMEM;
2722         shdr = elf64_getshdr(scn);
2723         if (!shdr)
2724                 return -EINVAL;
2725
2726         shdr->sh_name = name_off;
2727         shdr->sh_type = SHT_PROGBITS;
2728         shdr->sh_flags = 0;
2729         shdr->sh_size = raw_sz;
2730         shdr->sh_link = 0;
2731         shdr->sh_info = 0;
2732         shdr->sh_addralign = align;
2733         shdr->sh_entsize = 0;
2734
2735         data->d_type = ELF_T_BYTE;
2736         data->d_size = raw_sz;
2737         data->d_buf = (void *)raw_data;
2738         data->d_align = align;
2739         data->d_off = 0;
2740
2741         return 0;
2742 }
2743
2744 static int finalize_btf(struct bpf_linker *linker)
2745 {
2746         enum btf_endianness link_endianness;
2747         LIBBPF_OPTS(btf_dedup_opts, opts);
2748         struct btf *btf = linker->btf;
2749         const void *raw_data;
2750         int i, j, id, err;
2751         __u32 raw_sz;
2752
2753         /* bail out if no BTF data was produced */
2754         if (btf__type_cnt(linker->btf) == 1)
2755                 return 0;
2756
2757         for (i = 1; i < linker->sec_cnt; i++) {
2758                 struct dst_sec *sec = &linker->secs[i];
2759
2760                 if (!sec->has_btf)
2761                         continue;
2762
2763                 id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz);
2764                 if (id < 0) {
2765                         pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n",
2766                                 sec->sec_name, id);
2767                         return id;
2768                 }
2769
2770                 for (j = 0; j < sec->sec_var_cnt; j++) {
2771                         struct btf_var_secinfo *vi = &sec->sec_vars[j];
2772
2773                         if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size))
2774                                 return -EINVAL;
2775                 }
2776         }
2777
2778         err = finalize_btf_ext(linker);
2779         if (err) {
2780                 pr_warn(".BTF.ext generation failed: %s\n", errstr(err));
2781                 return err;
2782         }
2783
2784         opts.btf_ext = linker->btf_ext;
2785         err = btf__dedup(linker->btf, &opts);
2786         if (err) {
2787                 pr_warn("BTF dedup failed: %s\n", errstr(err));
2788                 return err;
2789         }
2790
2791         /* Set .BTF and .BTF.ext output byte order */
2792         link_endianness = linker->elf_hdr->e_ident[EI_DATA] == ELFDATA2MSB ?
2793                           BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN;
2794         btf__set_endianness(linker->btf, link_endianness);
2795         if (linker->btf_ext)
2796                 btf_ext__set_endianness(linker->btf_ext, link_endianness);
2797
2798         /* Emit .BTF section */
2799         raw_data = btf__raw_data(linker->btf, &raw_sz);
2800         if (!raw_data)
2801                 return -ENOMEM;
2802
2803         err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz);
2804         if (err) {
2805                 pr_warn("failed to write out .BTF ELF section: %s\n", errstr(err));
2806                 return err;
2807         }
2808
2809         /* Emit .BTF.ext section */
2810         if (linker->btf_ext) {
2811                 raw_data = btf_ext__raw_data(linker->btf_ext, &raw_sz);
2812                 if (!raw_data)
2813                         return -ENOMEM;
2814
2815                 err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz);
2816                 if (err) {
2817                         pr_warn("failed to write out .BTF.ext ELF section: %s\n", errstr(err));
2818                         return err;
2819                 }
2820         }
2821
2822         return 0;
2823 }
2824
2825 static int emit_btf_ext_data(struct bpf_linker *linker, void *output,
2826                              const char *sec_name, struct btf_ext_sec_data *sec_data)
2827 {
2828         struct btf_ext_info_sec *sec_info;
2829         void *cur = output;
2830         int str_off;
2831         size_t sz;
2832
2833         if (!sec_data->rec_cnt)
2834                 return 0;
2835
2836         str_off = btf__add_str(linker->btf, sec_name);
2837         if (str_off < 0)
2838                 return -ENOMEM;
2839
2840         sec_info = cur;
2841         sec_info->sec_name_off = str_off;
2842         sec_info->num_info = sec_data->rec_cnt;
2843         cur += sizeof(struct btf_ext_info_sec);
2844
2845         sz = sec_data->rec_cnt * sec_data->rec_sz;
2846         memcpy(cur, sec_data->recs, sz);
2847         cur += sz;
2848
2849         return cur - output;
2850 }
2851
2852 static int finalize_btf_ext(struct bpf_linker *linker)
2853 {
2854         size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0;
2855         size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0;
2856         struct btf_ext_header *hdr;
2857         void *data, *cur;
2858         int i, err, sz;
2859
2860         /* validate that all sections have the same .BTF.ext record sizes
2861          * and calculate total data size for each type of data (func info,
2862          * line info, core relos)
2863          */
2864         for (i = 1; i < linker->sec_cnt; i++) {
2865                 struct dst_sec *sec = &linker->secs[i];
2866
2867                 if (sec->func_info.rec_cnt) {
2868                         if (func_rec_sz == 0)
2869                                 func_rec_sz = sec->func_info.rec_sz;
2870                         if (func_rec_sz != sec->func_info.rec_sz) {
2871                                 pr_warn("mismatch in func_info record size %zu != %u\n",
2872                                         func_rec_sz, sec->func_info.rec_sz);
2873                                 return -EINVAL;
2874                         }
2875
2876                         funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt;
2877                 }
2878                 if (sec->line_info.rec_cnt) {
2879                         if (line_rec_sz == 0)
2880                                 line_rec_sz = sec->line_info.rec_sz;
2881                         if (line_rec_sz != sec->line_info.rec_sz) {
2882                                 pr_warn("mismatch in line_info record size %zu != %u\n",
2883                                         line_rec_sz, sec->line_info.rec_sz);
2884                                 return -EINVAL;
2885                         }
2886
2887                         lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt;
2888                 }
2889                 if (sec->core_relo_info.rec_cnt) {
2890                         if (core_relo_rec_sz == 0)
2891                                 core_relo_rec_sz = sec->core_relo_info.rec_sz;
2892                         if (core_relo_rec_sz != sec->core_relo_info.rec_sz) {
2893                                 pr_warn("mismatch in core_relo_info record size %zu != %u\n",
2894                                         core_relo_rec_sz, sec->core_relo_info.rec_sz);
2895                                 return -EINVAL;
2896                         }
2897
2898                         core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt;
2899                 }
2900         }
2901
2902         if (!funcs_sz && !lines_sz && !core_relos_sz)
2903                 return 0;
2904
2905         total_sz += sizeof(struct btf_ext_header);
2906         if (funcs_sz) {
2907                 funcs_sz += sizeof(__u32); /* record size prefix */
2908                 total_sz += funcs_sz;
2909         }
2910         if (lines_sz) {
2911                 lines_sz += sizeof(__u32); /* record size prefix */
2912                 total_sz += lines_sz;
2913         }
2914         if (core_relos_sz) {
2915                 core_relos_sz += sizeof(__u32); /* record size prefix */
2916                 total_sz += core_relos_sz;
2917         }
2918
2919         cur = data = calloc(1, total_sz);
2920         if (!data)
2921                 return -ENOMEM;
2922
2923         hdr = cur;
2924         hdr->magic = BTF_MAGIC;
2925         hdr->version = BTF_VERSION;
2926         hdr->flags = 0;
2927         hdr->hdr_len = sizeof(struct btf_ext_header);
2928         cur += sizeof(struct btf_ext_header);
2929
2930         /* All offsets are in bytes relative to the end of this header */
2931         hdr->func_info_off = 0;
2932         hdr->func_info_len = funcs_sz;
2933         hdr->line_info_off = funcs_sz;
2934         hdr->line_info_len = lines_sz;
2935         hdr->core_relo_off = funcs_sz + lines_sz;
2936         hdr->core_relo_len = core_relos_sz;
2937
2938         if (funcs_sz) {
2939                 *(__u32 *)cur = func_rec_sz;
2940                 cur += sizeof(__u32);
2941
2942                 for (i = 1; i < linker->sec_cnt; i++) {
2943                         struct dst_sec *sec = &linker->secs[i];
2944
2945                         sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info);
2946                         if (sz < 0) {
2947                                 err = sz;
2948                                 goto out;
2949                         }
2950
2951                         cur += sz;
2952                 }
2953         }
2954
2955         if (lines_sz) {
2956                 *(__u32 *)cur = line_rec_sz;
2957                 cur += sizeof(__u32);
2958
2959                 for (i = 1; i < linker->sec_cnt; i++) {
2960                         struct dst_sec *sec = &linker->secs[i];
2961
2962                         sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info);
2963                         if (sz < 0) {
2964                                 err = sz;
2965                                 goto out;
2966                         }
2967
2968                         cur += sz;
2969                 }
2970         }
2971
2972         if (core_relos_sz) {
2973                 *(__u32 *)cur = core_relo_rec_sz;
2974                 cur += sizeof(__u32);
2975
2976                 for (i = 1; i < linker->sec_cnt; i++) {
2977                         struct dst_sec *sec = &linker->secs[i];
2978
2979                         sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info);
2980                         if (sz < 0) {
2981                                 err = sz;
2982                                 goto out;
2983                         }
2984
2985                         cur += sz;
2986                 }
2987         }
2988
2989         linker->btf_ext = btf_ext__new(data, total_sz);
2990         err = libbpf_get_error(linker->btf_ext);
2991         if (err) {
2992                 linker->btf_ext = NULL;
2993                 pr_warn("failed to parse final .BTF.ext data: %s\n", errstr(err));
2994                 goto out;
2995         }
2996
2997 out:
2998         free(data);
2999         return err;
3000 }