tools/lib/bpf/libbpf.c

   1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
   2
   3 /*
   4  * Common eBPF ELF object loading operations.
   5  *
   6  * Copyright (C) 2013-2015 Alexei Starovoitov <[email protected]>
   7  * Copyright (C) 2015 Wang Nan <[email protected]>
   8  * Copyright (C) 2015 Huawei Inc.
   9  * Copyright (C) 2017 Nicira, Inc.
  10  */
  11
  12 #define _GNU_SOURCE
  13 #include <stdlib.h>
  14 #include <stdio.h>
  15 #include <stdarg.h>
  16 #include <libgen.h>
  17 #include <inttypes.h>
  18 #include <string.h>
  19 #include <unistd.h>
  20 #include <fcntl.h>
  21 #include <errno.h>
  22 #include <asm/unistd.h>
  23 #include <linux/err.h>
  24 #include <linux/kernel.h>
  25 #include <linux/bpf.h>
  26 #include <linux/btf.h>
  27 #include <linux/list.h>
  28 #include <linux/limits.h>
  29 #include <linux/perf_event.h>
  30 #include <linux/ring_buffer.h>
  31 #include <sys/stat.h>
  32 #include <sys/types.h>
  33 #include <sys/vfs.h>
  34 #include <tools/libc_compat.h>
  35 #include <libelf.h>
  36 #include <gelf.h>
  37
  38 #include "libbpf.h"
  39 #include "bpf.h"
  40 #include "btf.h"
  41 #include "str_error.h"
  42
  43 #ifndef EM_BPF
  44 #define EM_BPF 247
  45 #endif
  46
  47 #ifndef BPF_FS_MAGIC
  48 #define BPF_FS_MAGIC            0xcafe4a11
  49 #endif
  50
  51 #define __printf(a, b)  __attribute__((format(printf, a, b)))
  52
  53 __printf(1, 2)
  54 static int __base_pr(const char *format, ...)
  55 {
  56         va_list args;
  57         int err;
  58
  59         va_start(args, format);
  60         err = vfprintf(stderr, format, args);
  61         va_end(args);
  62         return err;
  63 }
  64
  65 static __printf(1, 2) libbpf_print_fn_t __pr_warning = __base_pr;
  66 static __printf(1, 2) libbpf_print_fn_t __pr_info = __base_pr;
  67 static __printf(1, 2) libbpf_print_fn_t __pr_debug;
  68
  69 #define __pr(func, fmt, ...)    \
  70 do {                            \
  71         if ((func))             \
  72                 (func)("libbpf: " fmt, ##__VA_ARGS__); \
  73 } while (0)
  74
  75 #define pr_warning(fmt, ...)    __pr(__pr_warning, fmt, ##__VA_ARGS__)
  76 #define pr_info(fmt, ...)       __pr(__pr_info, fmt, ##__VA_ARGS__)
  77 #define pr_debug(fmt, ...)      __pr(__pr_debug, fmt, ##__VA_ARGS__)
  78
  79 void libbpf_set_print(libbpf_print_fn_t warn,
  80                       libbpf_print_fn_t info,
  81                       libbpf_print_fn_t debug)
  82 {
  83         __pr_warning = warn;
  84         __pr_info = info;
  85         __pr_debug = debug;
  86 }
  87
  88 #define STRERR_BUFSIZE  128
  89
  90 #define CHECK_ERR(action, err, out) do {        \
  91         err = action;                   \
  92         if (err)                        \
  93                 goto out;               \
  94 } while(0)
  95
  96
  97 /* Copied from tools/perf/util/util.h */
  98 #ifndef zfree
  99 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
 100 #endif
 101
 102 #ifndef zclose
 103 # define zclose(fd) ({                  \
 104         int ___err = 0;                 \
 105         if ((fd) >= 0)                  \
 106                 ___err = close((fd));   \
 107         fd = -1;                        \
 108         ___err; })
 109 #endif
 110
 111 #ifdef HAVE_LIBELF_MMAP_SUPPORT
 112 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP
 113 #else
 114 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ
 115 #endif
 116
 117 /*
 118  * bpf_prog should be a better name but it has been used in
 119  * linux/filter.h.
 120  */
 121 struct bpf_program {
 122         /* Index in elf obj file, for relocation use. */
 123         int idx;
 124         char *name;
 125         int prog_ifindex;
 126         char *section_name;
 127         struct bpf_insn *insns;
 128         size_t insns_cnt, main_prog_cnt;
 129         enum bpf_prog_type type;
 130
 131         struct reloc_desc {
 132                 enum {
 133                         RELO_LD64,
 134                         RELO_CALL,
 135                 } type;
 136                 int insn_idx;
 137                 union {
 138                         int map_idx;
 139                         int text_off;
 140                 };
 141         } *reloc_desc;
 142         int nr_reloc;
 143
 144         struct {
 145                 int nr;
 146                 int *fds;
 147         } instances;
 148         bpf_program_prep_t preprocessor;
 149
 150         struct bpf_object *obj;
 151         void *priv;
 152         bpf_program_clear_priv_t clear_priv;
 153
 154         enum bpf_attach_type expected_attach_type;
 155 };
 156
 157 struct bpf_map {
 158         int fd;
 159         char *name;
 160         size_t offset;
 161         int map_ifindex;
 162         struct bpf_map_def def;
 163         __u32 btf_key_type_id;
 164         __u32 btf_value_type_id;
 165         void *priv;
 166         bpf_map_clear_priv_t clear_priv;
 167 };
 168
 169 static LIST_HEAD(bpf_objects_list);
 170
 171 struct bpf_object {
 172         char license[64];
 173         __u32 kern_version;
 174
 175         struct bpf_program *programs;
 176         size_t nr_programs;
 177         struct bpf_map *maps;
 178         size_t nr_maps;
 179
 180         bool loaded;
 181         bool has_pseudo_calls;
 182
 183         /*
 184          * Information when doing elf related work. Only valid if fd
 185          * is valid.
 186          */
 187         struct {
 188                 int fd;
 189                 void *obj_buf;
 190                 size_t obj_buf_sz;
 191                 Elf *elf;
 192                 GElf_Ehdr ehdr;
 193                 Elf_Data *symbols;
 194                 size_t strtabidx;
 195                 struct {
 196                         GElf_Shdr shdr;
 197                         Elf_Data *data;
 198                 } *reloc;
 199                 int nr_reloc;
 200                 int maps_shndx;
 201                 int text_shndx;
 202         } efile;
 203         /*
 204          * All loaded bpf_object is linked in a list, which is
 205          * hidden to caller. bpf_objects__<func> handlers deal with
 206          * all objects.
 207          */
 208         struct list_head list;
 209
 210         struct btf *btf;
 211
 212         void *priv;
 213         bpf_object_clear_priv_t clear_priv;
 214
 215         char path[];
 216 };
 217 #define obj_elf_valid(o)        ((o)->efile.elf)
 218
 219 void bpf_program__unload(struct bpf_program *prog)
 220 {
 221         int i;
 222
 223         if (!prog)
 224                 return;
 225
 226         /*
 227          * If the object is opened but the program was never loaded,
 228          * it is possible that prog->instances.nr == -1.
 229          */
 230         if (prog->instances.nr > 0) {
 231                 for (i = 0; i < prog->instances.nr; i++)
 232                         zclose(prog->instances.fds[i]);
 233         } else if (prog->instances.nr != -1) {
 234                 pr_warning("Internal error: instances.nr is %d\n",
 235                            prog->instances.nr);
 236         }
 237
 238         prog->instances.nr = -1;
 239         zfree(&prog->instances.fds);
 240 }
 241
 242 static void bpf_program__exit(struct bpf_program *prog)
 243 {
 244         if (!prog)
 245                 return;
 246
 247         if (prog->clear_priv)
 248                 prog->clear_priv(prog, prog->priv);
 249
 250         prog->priv = NULL;
 251         prog->clear_priv = NULL;
 252
 253         bpf_program__unload(prog);
 254         zfree(&prog->name);
 255         zfree(&prog->section_name);
 256         zfree(&prog->insns);
 257         zfree(&prog->reloc_desc);
 258
 259         prog->nr_reloc = 0;
 260         prog->insns_cnt = 0;
 261         prog->idx = -1;
 262 }
 263
 264 static int
 265 bpf_program__init(void *data, size_t size, char *section_name, int idx,
 266                   struct bpf_program *prog)
 267 {
 268         if (size < sizeof(struct bpf_insn)) {
 269                 pr_warning("corrupted section '%s'\n", section_name);
 270                 return -EINVAL;
 271         }
 272
 273         bzero(prog, sizeof(*prog));
 274
 275         prog->section_name = strdup(section_name);
 276         if (!prog->section_name) {
 277                 pr_warning("failed to alloc name for prog under section(%d) %s\n",
 278                            idx, section_name);
 279                 goto errout;
 280         }
 281
 282         prog->insns = malloc(size);
 283         if (!prog->insns) {
 284                 pr_warning("failed to alloc insns for prog under section %s\n",
 285                            section_name);
 286                 goto errout;
 287         }
 288         prog->insns_cnt = size / sizeof(struct bpf_insn);
 289         memcpy(prog->insns, data,
 290                prog->insns_cnt * sizeof(struct bpf_insn));
 291         prog->idx = idx;
 292         prog->instances.fds = NULL;
 293         prog->instances.nr = -1;
 294         prog->type = BPF_PROG_TYPE_KPROBE;
 295
 296         return 0;
 297 errout:
 298         bpf_program__exit(prog);
 299         return -ENOMEM;
 300 }
 301
 302 static int
 303 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size,
 304                         char *section_name, int idx)
 305 {
 306         struct bpf_program prog, *progs;
 307         int nr_progs, err;
 308
 309         err = bpf_program__init(data, size, section_name, idx, &prog);
 310         if (err)
 311                 return err;
 312
 313         progs = obj->programs;
 314         nr_progs = obj->nr_programs;
 315
 316         progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0]));
 317         if (!progs) {
 318                 /*
 319                  * In this case the original obj->programs
 320                  * is still valid, so don't need special treat for
 321                  * bpf_close_object().
 322                  */
 323                 pr_warning("failed to alloc a new program under section '%s'\n",
 324                            section_name);
 325                 bpf_program__exit(&prog);
 326                 return -ENOMEM;
 327         }
 328
 329         pr_debug("found program %s\n", prog.section_name);
 330         obj->programs = progs;
 331         obj->nr_programs = nr_progs + 1;
 332         prog.obj = obj;
 333         progs[nr_progs] = prog;
 334         return 0;
 335 }
 336
 337 static int
 338 bpf_object__init_prog_names(struct bpf_object *obj)
 339 {
 340         Elf_Data *symbols = obj->efile.symbols;
 341         struct bpf_program *prog;
 342         size_t pi, si;
 343
 344         for (pi = 0; pi < obj->nr_programs; pi++) {
 345                 const char *name = NULL;
 346
 347                 prog = &obj->programs[pi];
 348
 349                 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name;
 350                      si++) {
 351                         GElf_Sym sym;
 352
 353                         if (!gelf_getsym(symbols, si, &sym))
 354                                 continue;
 355                         if (sym.st_shndx != prog->idx)
 356                                 continue;
 357                         if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL)
 358                                 continue;
 359
 360                         name = elf_strptr(obj->efile.elf,
 361                                           obj->efile.strtabidx,
 362                                           sym.st_name);
 363                         if (!name) {
 364                                 pr_warning("failed to get sym name string for prog %s\n",
 365                                            prog->section_name);
 366                                 return -LIBBPF_ERRNO__LIBELF;
 367                         }
 368                 }
 369
 370                 if (!name && prog->idx == obj->efile.text_shndx)
 371                         name = ".text";
 372
 373                 if (!name) {
 374                         pr_warning("failed to find sym for prog %s\n",
 375                                    prog->section_name);
 376                         return -EINVAL;
 377                 }
 378
 379                 prog->name = strdup(name);
 380                 if (!prog->name) {
 381                         pr_warning("failed to allocate memory for prog sym %s\n",
 382                                    name);
 383                         return -ENOMEM;
 384                 }
 385         }
 386
 387         return 0;
 388 }
 389
 390 static struct bpf_object *bpf_object__new(const char *path,
 391                                           void *obj_buf,
 392                                           size_t obj_buf_sz)
 393 {
 394         struct bpf_object *obj;
 395
 396         obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
 397         if (!obj) {
 398                 pr_warning("alloc memory failed for %s\n", path);
 399                 return ERR_PTR(-ENOMEM);
 400         }
 401
 402         strcpy(obj->path, path);
 403         obj->efile.fd = -1;
 404
 405         /*
 406          * Caller of this function should also calls
 407          * bpf_object__elf_finish() after data collection to return
 408          * obj_buf to user. If not, we should duplicate the buffer to
 409          * avoid user freeing them before elf finish.
 410          */
 411         obj->efile.obj_buf = obj_buf;
 412         obj->efile.obj_buf_sz = obj_buf_sz;
 413         obj->efile.maps_shndx = -1;
 414
 415         obj->loaded = false;
 416
 417         INIT_LIST_HEAD(&obj->list);
 418         list_add(&obj->list, &bpf_objects_list);
 419         return obj;
 420 }
 421
 422 static void bpf_object__elf_finish(struct bpf_object *obj)
 423 {
 424         if (!obj_elf_valid(obj))
 425                 return;
 426
 427         if (obj->efile.elf) {
 428                 elf_end(obj->efile.elf);
 429                 obj->efile.elf = NULL;
 430         }
 431         obj->efile.symbols = NULL;
 432
 433         zfree(&obj->efile.reloc);
 434         obj->efile.nr_reloc = 0;
 435         zclose(obj->efile.fd);
 436         obj->efile.obj_buf = NULL;
 437         obj->efile.obj_buf_sz = 0;
 438 }
 439
 440 static int bpf_object__elf_init(struct bpf_object *obj)
 441 {
 442         int err = 0;
 443         GElf_Ehdr *ep;
 444
 445         if (obj_elf_valid(obj)) {
 446                 pr_warning("elf init: internal error\n");
 447                 return -LIBBPF_ERRNO__LIBELF;
 448         }
 449
 450         if (obj->efile.obj_buf_sz > 0) {
 451                 /*
 452                  * obj_buf should have been validated by
 453                  * bpf_object__open_buffer().
 454                  */
 455                 obj->efile.elf = elf_memory(obj->efile.obj_buf,
 456                                             obj->efile.obj_buf_sz);
 457         } else {
 458                 obj->efile.fd = open(obj->path, O_RDONLY);
 459                 if (obj->efile.fd < 0) {
 460                         char errmsg[STRERR_BUFSIZE];
 461                         char *cp = libbpf_strerror_r(errno, errmsg,
 462                                                      sizeof(errmsg));
 463
 464                         pr_warning("failed to open %s: %s\n", obj->path, cp);
 465                         return -errno;
 466                 }
 467
 468                 obj->efile.elf = elf_begin(obj->efile.fd,
 469                                 LIBBPF_ELF_C_READ_MMAP,
 470                                 NULL);
 471         }
 472
 473         if (!obj->efile.elf) {
 474                 pr_warning("failed to open %s as ELF file\n",
 475                                 obj->path);
 476                 err = -LIBBPF_ERRNO__LIBELF;
 477                 goto errout;
 478         }
 479
 480         if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) {
 481                 pr_warning("failed to get EHDR from %s\n",
 482                                 obj->path);
 483                 err = -LIBBPF_ERRNO__FORMAT;
 484                 goto errout;
 485         }
 486         ep = &obj->efile.ehdr;
 487
 488         /* Old LLVM set e_machine to EM_NONE */
 489         if ((ep->e_type != ET_REL) || (ep->e_machine && (ep->e_machine != EM_BPF))) {
 490                 pr_warning("%s is not an eBPF object file\n",
 491                         obj->path);
 492                 err = -LIBBPF_ERRNO__FORMAT;
 493                 goto errout;
 494         }
 495
 496         return 0;
 497 errout:
 498         bpf_object__elf_finish(obj);
 499         return err;
 500 }
 501
 502 static int
 503 bpf_object__check_endianness(struct bpf_object *obj)
 504 {
 505         static unsigned int const endian = 1;
 506
 507         switch (obj->efile.ehdr.e_ident[EI_DATA]) {
 508         case ELFDATA2LSB:
 509                 /* We are big endian, BPF obj is little endian. */
 510                 if (*(unsigned char const *)&endian != 1)
 511                         goto mismatch;
 512                 break;
 513
 514         case ELFDATA2MSB:
 515                 /* We are little endian, BPF obj is big endian. */
 516                 if (*(unsigned char const *)&endian != 0)
 517                         goto mismatch;
 518                 break;
 519         default:
 520                 return -LIBBPF_ERRNO__ENDIAN;
 521         }
 522
 523         return 0;
 524
 525 mismatch:
 526         pr_warning("Error: endianness mismatch.\n");
 527         return -LIBBPF_ERRNO__ENDIAN;
 528 }
 529
 530 static int
 531 bpf_object__init_license(struct bpf_object *obj,
 532                          void *data, size_t size)
 533 {
 534         memcpy(obj->license, data,
 535                min(size, sizeof(obj->license) - 1));
 536         pr_debug("license of %s is %s\n", obj->path, obj->license);
 537         return 0;
 538 }
 539
 540 static int
 541 bpf_object__init_kversion(struct bpf_object *obj,
 542                           void *data, size_t size)
 543 {
 544         __u32 kver;
 545
 546         if (size != sizeof(kver)) {
 547                 pr_warning("invalid kver section in %s\n", obj->path);
 548                 return -LIBBPF_ERRNO__FORMAT;
 549         }
 550         memcpy(&kver, data, sizeof(kver));
 551         obj->kern_version = kver;
 552         pr_debug("kernel version of %s is %x\n", obj->path,
 553                  obj->kern_version);
 554         return 0;
 555 }
 556
 557 static int compare_bpf_map(const void *_a, const void *_b)
 558 {
 559         const struct bpf_map *a = _a;
 560         const struct bpf_map *b = _b;
 561
 562         return a->offset - b->offset;
 563 }
 564
 565 static int
 566 bpf_object__init_maps(struct bpf_object *obj, int flags)
 567 {
 568         bool strict = !(flags & MAPS_RELAX_COMPAT);
 569         int i, map_idx, map_def_sz, nr_maps = 0;
 570         Elf_Scn *scn;
 571         Elf_Data *data;
 572         Elf_Data *symbols = obj->efile.symbols;
 573
 574         if (obj->efile.maps_shndx < 0)
 575                 return -EINVAL;
 576         if (!symbols)
 577                 return -EINVAL;
 578
 579         scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx);
 580         if (scn)
 581                 data = elf_getdata(scn, NULL);
 582         if (!scn || !data) {
 583                 pr_warning("failed to get Elf_Data from map section %d\n",
 584                            obj->efile.maps_shndx);
 585                 return -EINVAL;
 586         }
 587
 588         /*
 589          * Count number of maps. Each map has a name.
 590          * Array of maps is not supported: only the first element is
 591          * considered.
 592          *
 593          * TODO: Detect array of map and report error.
 594          */
 595         for (i = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) {
 596                 GElf_Sym sym;
 597
 598                 if (!gelf_getsym(symbols, i, &sym))
 599                         continue;
 600                 if (sym.st_shndx != obj->efile.maps_shndx)
 601                         continue;
 602                 nr_maps++;
 603         }
 604
 605         /* Alloc obj->maps and fill nr_maps. */
 606         pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path,
 607                  nr_maps, data->d_size);
 608
 609         if (!nr_maps)
 610                 return 0;
 611
 612         /* Assume equally sized map definitions */
 613         map_def_sz = data->d_size / nr_maps;
 614         if (!data->d_size || (data->d_size % nr_maps) != 0) {
 615                 pr_warning("unable to determine map definition size "
 616                            "section %s, %d maps in %zd bytes\n",
 617                            obj->path, nr_maps, data->d_size);
 618                 return -EINVAL;
 619         }
 620
 621         obj->maps = calloc(nr_maps, sizeof(obj->maps[0]));
 622         if (!obj->maps) {
 623                 pr_warning("alloc maps for object failed\n");
 624                 return -ENOMEM;
 625         }
 626         obj->nr_maps = nr_maps;
 627
 628         /*
 629          * fill all fd with -1 so won't close incorrect
 630          * fd (fd=0 is stdin) when failure (zclose won't close
 631          * negative fd)).
 632          */
 633         for (i = 0; i < nr_maps; i++)
 634                 obj->maps[i].fd = -1;
 635
 636         /*
 637          * Fill obj->maps using data in "maps" section.
 638          */
 639         for (i = 0, map_idx = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) {
 640                 GElf_Sym sym;
 641                 const char *map_name;
 642                 struct bpf_map_def *def;
 643
 644                 if (!gelf_getsym(symbols, i, &sym))
 645                         continue;
 646                 if (sym.st_shndx != obj->efile.maps_shndx)
 647                         continue;
 648
 649                 map_name = elf_strptr(obj->efile.elf,
 650                                       obj->efile.strtabidx,
 651                                       sym.st_name);
 652                 obj->maps[map_idx].offset = sym.st_value;
 653                 if (sym.st_value + map_def_sz > data->d_size) {
 654                         pr_warning("corrupted maps section in %s: last map \"%s\" too small\n",
 655                                    obj->path, map_name);
 656                         return -EINVAL;
 657                 }
 658
 659                 obj->maps[map_idx].name = strdup(map_name);
 660                 if (!obj->maps[map_idx].name) {
 661                         pr_warning("failed to alloc map name\n");
 662                         return -ENOMEM;
 663                 }
 664                 pr_debug("map %d is \"%s\"\n", map_idx,
 665                          obj->maps[map_idx].name);
 666                 def = (struct bpf_map_def *)(data->d_buf + sym.st_value);
 667                 /*
 668                  * If the definition of the map in the object file fits in
 669                  * bpf_map_def, copy it.  Any extra fields in our version
 670                  * of bpf_map_def will default to zero as a result of the
 671                  * calloc above.
 672                  */
 673                 if (map_def_sz <= sizeof(struct bpf_map_def)) {
 674                         memcpy(&obj->maps[map_idx].def, def, map_def_sz);
 675                 } else {
 676                         /*
 677                          * Here the map structure being read is bigger than what
 678                          * we expect, truncate if the excess bits are all zero.
 679                          * If they are not zero, reject this map as
 680                          * incompatible.
 681                          */
 682                         char *b;
 683                         for (b = ((char *)def) + sizeof(struct bpf_map_def);
 684                              b < ((char *)def) + map_def_sz; b++) {
 685                                 if (*b != 0) {
 686                                         pr_warning("maps section in %s: \"%s\" "
 687                                                    "has unrecognized, non-zero "
 688                                                    "options\n",
 689                                                    obj->path, map_name);
 690                                         if (strict)
 691                                                 return -EINVAL;
 692                                 }
 693                         }
 694                         memcpy(&obj->maps[map_idx].def, def,
 695                                sizeof(struct bpf_map_def));
 696                 }
 697                 map_idx++;
 698         }
 699
 700         qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), compare_bpf_map);
 701         return 0;
 702 }
 703
 704 static bool section_have_execinstr(struct bpf_object *obj, int idx)
 705 {
 706         Elf_Scn *scn;
 707         GElf_Shdr sh;
 708
 709         scn = elf_getscn(obj->efile.elf, idx);
 710         if (!scn)
 711                 return false;
 712
 713         if (gelf_getshdr(scn, &sh) != &sh)
 714                 return false;
 715
 716         if (sh.sh_flags & SHF_EXECINSTR)
 717                 return true;
 718
 719         return false;
 720 }
 721
 722 static int bpf_object__elf_collect(struct bpf_object *obj, int flags)
 723 {
 724         Elf *elf = obj->efile.elf;
 725         GElf_Ehdr *ep = &obj->efile.ehdr;
 726         Elf_Scn *scn = NULL;
 727         int idx = 0, err = 0;
 728
 729         /* Elf is corrupted/truncated, avoid calling elf_strptr. */
 730         if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) {
 731                 pr_warning("failed to get e_shstrndx from %s\n",
 732                            obj->path);
 733                 return -LIBBPF_ERRNO__FORMAT;
 734         }
 735
 736         while ((scn = elf_nextscn(elf, scn)) != NULL) {
 737                 char *name;
 738                 GElf_Shdr sh;
 739                 Elf_Data *data;
 740
 741                 idx++;
 742                 if (gelf_getshdr(scn, &sh) != &sh) {
 743                         pr_warning("failed to get section(%d) header from %s\n",
 744                                    idx, obj->path);
 745                         err = -LIBBPF_ERRNO__FORMAT;
 746                         goto out;
 747                 }
 748
 749                 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name);
 750                 if (!name) {
 751                         pr_warning("failed to get section(%d) name from %s\n",
 752                                    idx, obj->path);
 753                         err = -LIBBPF_ERRNO__FORMAT;
 754                         goto out;
 755                 }
 756
 757                 data = elf_getdata(scn, 0);
 758                 if (!data) {
 759                         pr_warning("failed to get section(%d) data from %s(%s)\n",
 760                                    idx, name, obj->path);
 761                         err = -LIBBPF_ERRNO__FORMAT;
 762                         goto out;
 763                 }
 764                 pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
 765                          idx, name, (unsigned long)data->d_size,
 766                          (int)sh.sh_link, (unsigned long)sh.sh_flags,
 767                          (int)sh.sh_type);
 768
 769                 if (strcmp(name, "license") == 0)
 770                         err = bpf_object__init_license(obj,
 771                                                        data->d_buf,
 772                                                        data->d_size);
 773                 else if (strcmp(name, "version") == 0)
 774                         err = bpf_object__init_kversion(obj,
 775                                                         data->d_buf,
 776                                                         data->d_size);
 777                 else if (strcmp(name, "maps") == 0)
 778                         obj->efile.maps_shndx = idx;
 779                 else if (strcmp(name, BTF_ELF_SEC) == 0) {
 780                         obj->btf = btf__new(data->d_buf, data->d_size,
 781                                             __pr_debug);
 782                         if (IS_ERR(obj->btf)) {
 783                                 pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n",
 784                                            BTF_ELF_SEC, PTR_ERR(obj->btf));
 785                                 obj->btf = NULL;
 786                         }
 787                 } else if (sh.sh_type == SHT_SYMTAB) {
 788                         if (obj->efile.symbols) {
 789                                 pr_warning("bpf: multiple SYMTAB in %s\n",
 790                                            obj->path);
 791                                 err = -LIBBPF_ERRNO__FORMAT;
 792                         } else {
 793                                 obj->efile.symbols = data;
 794                                 obj->efile.strtabidx = sh.sh_link;
 795                         }
 796                 } else if ((sh.sh_type == SHT_PROGBITS) &&
 797                            (sh.sh_flags & SHF_EXECINSTR) &&
 798                            (data->d_size > 0)) {
 799                         if (strcmp(name, ".text") == 0)
 800                                 obj->efile.text_shndx = idx;
 801                         err = bpf_object__add_program(obj, data->d_buf,
 802                                                       data->d_size, name, idx);
 803                         if (err) {
 804                                 char errmsg[STRERR_BUFSIZE];
 805                                 char *cp = libbpf_strerror_r(-err, errmsg,
 806                                                              sizeof(errmsg));
 807
 808                                 pr_warning("failed to alloc program %s (%s): %s",
 809                                            name, obj->path, cp);
 810                         }
 811                 } else if (sh.sh_type == SHT_REL) {
 812                         void *reloc = obj->efile.reloc;
 813                         int nr_reloc = obj->efile.nr_reloc + 1;
 814                         int sec = sh.sh_info; /* points to other section */
 815
 816                         /* Only do relo for section with exec instructions */
 817                         if (!section_have_execinstr(obj, sec)) {
 818                                 pr_debug("skip relo %s(%d) for section(%d)\n",
 819                                          name, idx, sec);
 820                                 continue;
 821                         }
 822
 823                         reloc = reallocarray(reloc, nr_reloc,
 824                                              sizeof(*obj->efile.reloc));
 825                         if (!reloc) {
 826                                 pr_warning("realloc failed\n");
 827                                 err = -ENOMEM;
 828                         } else {
 829                                 int n = nr_reloc - 1;
 830
 831                                 obj->efile.reloc = reloc;
 832                                 obj->efile.nr_reloc = nr_reloc;
 833
 834                                 obj->efile.reloc[n].shdr = sh;
 835                                 obj->efile.reloc[n].data = data;
 836                         }
 837                 } else {
 838                         pr_debug("skip section(%d) %s\n", idx, name);
 839                 }
 840                 if (err)
 841                         goto out;
 842         }
 843
 844         if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) {
 845                 pr_warning("Corrupted ELF file: index of strtab invalid\n");
 846                 return LIBBPF_ERRNO__FORMAT;
 847         }
 848         if (obj->efile.maps_shndx >= 0) {
 849                 err = bpf_object__init_maps(obj, flags);
 850                 if (err)
 851                         goto out;
 852         }
 853         err = bpf_object__init_prog_names(obj);
 854 out:
 855         return err;
 856 }
 857
 858 static struct bpf_program *
 859 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx)
 860 {
 861         struct bpf_program *prog;
 862         size_t i;
 863
 864         for (i = 0; i < obj->nr_programs; i++) {
 865                 prog = &obj->programs[i];
 866                 if (prog->idx == idx)
 867                         return prog;
 868         }
 869         return NULL;
 870 }
 871
 872 struct bpf_program *
 873 bpf_object__find_program_by_title(struct bpf_object *obj, const char *title)
 874 {
 875         struct bpf_program *pos;
 876
 877         bpf_object__for_each_program(pos, obj) {
 878                 if (pos->section_name && !strcmp(pos->section_name, title))
 879                         return pos;
 880         }
 881         return NULL;
 882 }
 883
 884 static int
 885 bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr,
 886                            Elf_Data *data, struct bpf_object *obj)
 887 {
 888         Elf_Data *symbols = obj->efile.symbols;
 889         int text_shndx = obj->efile.text_shndx;
 890         int maps_shndx = obj->efile.maps_shndx;
 891         struct bpf_map *maps = obj->maps;
 892         size_t nr_maps = obj->nr_maps;
 893         int i, nrels;
 894
 895         pr_debug("collecting relocating info for: '%s'\n",
 896                  prog->section_name);
 897         nrels = shdr->sh_size / shdr->sh_entsize;
 898
 899         prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels);
 900         if (!prog->reloc_desc) {
 901                 pr_warning("failed to alloc memory in relocation\n");
 902                 return -ENOMEM;
 903         }
 904         prog->nr_reloc = nrels;
 905
 906         for (i = 0; i < nrels; i++) {
 907                 GElf_Sym sym;
 908                 GElf_Rel rel;
 909                 unsigned int insn_idx;
 910                 struct bpf_insn *insns = prog->insns;
 911                 size_t map_idx;
 912
 913                 if (!gelf_getrel(data, i, &rel)) {
 914                         pr_warning("relocation: failed to get %d reloc\n", i);
 915                         return -LIBBPF_ERRNO__FORMAT;
 916                 }
 917
 918                 if (!gelf_getsym(symbols,
 919                                  GELF_R_SYM(rel.r_info),
 920                                  &sym)) {
 921                         pr_warning("relocation: symbol %"PRIx64" not found\n",
 922                                    GELF_R_SYM(rel.r_info));
 923                         return -LIBBPF_ERRNO__FORMAT;
 924                 }
 925                 pr_debug("relo for %lld value %lld name %d\n",
 926                          (long long) (rel.r_info >> 32),
 927                          (long long) sym.st_value, sym.st_name);
 928
 929                 if (sym.st_shndx != maps_shndx && sym.st_shndx != text_shndx) {
 930                         pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n",
 931                                    prog->section_name, sym.st_shndx);
 932                         return -LIBBPF_ERRNO__RELOC;
 933                 }
 934
 935                 insn_idx = rel.r_offset / sizeof(struct bpf_insn);
 936                 pr_debug("relocation: insn_idx=%u\n", insn_idx);
 937
 938                 if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) {
 939                         if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) {
 940                                 pr_warning("incorrect bpf_call opcode\n");
 941                                 return -LIBBPF_ERRNO__RELOC;
 942                         }
 943                         prog->reloc_desc[i].type = RELO_CALL;
 944                         prog->reloc_desc[i].insn_idx = insn_idx;
 945                         prog->reloc_desc[i].text_off = sym.st_value;
 946                         obj->has_pseudo_calls = true;
 947                         continue;
 948                 }
 949
 950                 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) {
 951                         pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n",
 952                                    insn_idx, insns[insn_idx].code);
 953                         return -LIBBPF_ERRNO__RELOC;
 954                 }
 955
 956                 /* TODO: 'maps' is sorted. We can use bsearch to make it faster. */
 957                 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
 958                         if (maps[map_idx].offset == sym.st_value) {
 959                                 pr_debug("relocation: find map %zd (%s) for insn %u\n",
 960                                          map_idx, maps[map_idx].name, insn_idx);
 961                                 break;
 962                         }
 963                 }
 964
 965                 if (map_idx >= nr_maps) {
 966                         pr_warning("bpf relocation: map_idx %d large than %d\n",
 967                                    (int)map_idx, (int)nr_maps - 1);
 968                         return -LIBBPF_ERRNO__RELOC;
 969                 }
 970
 971                 prog->reloc_desc[i].type = RELO_LD64;
 972                 prog->reloc_desc[i].insn_idx = insn_idx;
 973                 prog->reloc_desc[i].map_idx = map_idx;
 974         }
 975         return 0;
 976 }
 977
 978 static int bpf_map_find_btf_info(struct bpf_map *map, const struct btf *btf)
 979 {
 980         const struct btf_type *container_type;
 981         const struct btf_member *key, *value;
 982         struct bpf_map_def *def = &map->def;
 983         const size_t max_name = 256;
 984         char container_name[max_name];
 985         __s64 key_size, value_size;
 986         __s32 container_id;
 987
 988         if (snprintf(container_name, max_name, "____btf_map_%s", map->name) ==
 989             max_name) {
 990                 pr_warning("map:%s length of '____btf_map_%s' is too long\n",
 991                            map->name, map->name);
 992                 return -EINVAL;
 993         }
 994
 995         container_id = btf__find_by_name(btf, container_name);
 996         if (container_id < 0) {
 997                 pr_debug("map:%s container_name:%s cannot be found in BTF. Missing BPF_ANNOTATE_KV_PAIR?\n",
 998                          map->name, container_name);
 999                 return container_id;
1000         }
1001
1002         container_type = btf__type_by_id(btf, container_id);
1003         if (!container_type) {
1004                 pr_warning("map:%s cannot find BTF type for container_id:%u\n",
1005                            map->name, container_id);
1006                 return -EINVAL;
1007         }
1008
1009         if (BTF_INFO_KIND(container_type->info) != BTF_KIND_STRUCT ||
1010             BTF_INFO_VLEN(container_type->info) < 2) {
1011                 pr_warning("map:%s container_name:%s is an invalid container struct\n",
1012                            map->name, container_name);
1013                 return -EINVAL;
1014         }
1015
1016         key = (struct btf_member *)(container_type + 1);
1017         value = key + 1;
1018
1019         key_size = btf__resolve_size(btf, key->type);
1020         if (key_size < 0) {
1021                 pr_warning("map:%s invalid BTF key_type_size\n",
1022                            map->name);
1023                 return key_size;
1024         }
1025
1026         if (def->key_size != key_size) {
1027                 pr_warning("map:%s btf_key_type_size:%u != map_def_key_size:%u\n",
1028                            map->name, (__u32)key_size, def->key_size);
1029                 return -EINVAL;
1030         }
1031
1032         value_size = btf__resolve_size(btf, value->type);
1033         if (value_size < 0) {
1034                 pr_warning("map:%s invalid BTF value_type_size\n", map->name);
1035                 return value_size;
1036         }
1037
1038         if (def->value_size != value_size) {
1039                 pr_warning("map:%s btf_value_type_size:%u != map_def_value_size:%u\n",
1040                            map->name, (__u32)value_size, def->value_size);
1041                 return -EINVAL;
1042         }
1043
1044         map->btf_key_type_id = key->type;
1045         map->btf_value_type_id = value->type;
1046
1047         return 0;
1048 }
1049
1050 int bpf_map__reuse_fd(struct bpf_map *map, int fd)
1051 {
1052         struct bpf_map_info info = {};
1053         __u32 len = sizeof(info);
1054         int new_fd, err;
1055         char *new_name;
1056
1057         err = bpf_obj_get_info_by_fd(fd, &info, &len);
1058         if (err)
1059                 return err;
1060
1061         new_name = strdup(info.name);
1062         if (!new_name)
1063                 return -errno;
1064
1065         new_fd = open("/", O_RDONLY | O_CLOEXEC);
1066         if (new_fd < 0)
1067                 goto err_free_new_name;
1068
1069         new_fd = dup3(fd, new_fd, O_CLOEXEC);
1070         if (new_fd < 0)
1071                 goto err_close_new_fd;
1072
1073         err = zclose(map->fd);
1074         if (err)
1075                 goto err_close_new_fd;
1076         free(map->name);
1077
1078         map->fd = new_fd;
1079         map->name = new_name;
1080         map->def.type = info.type;
1081         map->def.key_size = info.key_size;
1082         map->def.value_size = info.value_size;
1083         map->def.max_entries = info.max_entries;
1084         map->def.map_flags = info.map_flags;
1085         map->btf_key_type_id = info.btf_key_type_id;
1086         map->btf_value_type_id = info.btf_value_type_id;
1087
1088         return 0;
1089
1090 err_close_new_fd:
1091         close(new_fd);
1092 err_free_new_name:
1093         free(new_name);
1094         return -errno;
1095 }
1096
1097 static int
1098 bpf_object__create_maps(struct bpf_object *obj)
1099 {
1100         struct bpf_create_map_attr create_attr = {};
1101         unsigned int i;
1102         int err;
1103
1104         for (i = 0; i < obj->nr_maps; i++) {
1105                 struct bpf_map *map = &obj->maps[i];
1106                 struct bpf_map_def *def = &map->def;
1107                 char *cp, errmsg[STRERR_BUFSIZE];
1108                 int *pfd = &map->fd;
1109
1110                 if (map->fd >= 0) {
1111                         pr_debug("skip map create (preset) %s: fd=%d\n",
1112                                  map->name, map->fd);
1113                         continue;
1114                 }
1115
1116                 create_attr.name = map->name;
1117                 create_attr.map_ifindex = map->map_ifindex;
1118                 create_attr.map_type = def->type;
1119                 create_attr.map_flags = def->map_flags;
1120                 create_attr.key_size = def->key_size;
1121                 create_attr.value_size = def->value_size;
1122                 create_attr.max_entries = def->max_entries;
1123                 create_attr.btf_fd = 0;
1124                 create_attr.btf_key_type_id = 0;
1125                 create_attr.btf_value_type_id = 0;
1126
1127                 if (obj->btf && !bpf_map_find_btf_info(map, obj->btf)) {
1128                         create_attr.btf_fd = btf__fd(obj->btf);
1129                         create_attr.btf_key_type_id = map->btf_key_type_id;
1130                         create_attr.btf_value_type_id = map->btf_value_type_id;
1131                 }
1132
1133                 *pfd = bpf_create_map_xattr(&create_attr);
1134                 if (*pfd < 0 && create_attr.btf_key_type_id) {
1135                         cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1136                         pr_warning("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
1137                                    map->name, cp, errno);
1138                         create_attr.btf_fd = 0;
1139                         create_attr.btf_key_type_id = 0;
1140                         create_attr.btf_value_type_id = 0;
1141                         map->btf_key_type_id = 0;
1142                         map->btf_value_type_id = 0;
1143                         *pfd = bpf_create_map_xattr(&create_attr);
1144                 }
1145
1146                 if (*pfd < 0) {
1147                         size_t j;
1148
1149                         err = *pfd;
1150                         cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1151                         pr_warning("failed to create map (name: '%s'): %s\n",
1152                                    map->name, cp);
1153                         for (j = 0; j < i; j++)
1154                                 zclose(obj->maps[j].fd);
1155                         return err;
1156                 }
1157                 pr_debug("create map %s: fd=%d\n", map->name, *pfd);
1158         }
1159
1160         return 0;
1161 }
1162
1163 static int
1164 bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj,
1165                         struct reloc_desc *relo)
1166 {
1167         struct bpf_insn *insn, *new_insn;
1168         struct bpf_program *text;
1169         size_t new_cnt;
1170
1171         if (relo->type != RELO_CALL)
1172                 return -LIBBPF_ERRNO__RELOC;
1173
1174         if (prog->idx == obj->efile.text_shndx) {
1175                 pr_warning("relo in .text insn %d into off %d\n",
1176                            relo->insn_idx, relo->text_off);
1177                 return -LIBBPF_ERRNO__RELOC;
1178         }
1179
1180         if (prog->main_prog_cnt == 0) {
1181                 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx);
1182                 if (!text) {
1183                         pr_warning("no .text section found yet relo into text exist\n");
1184                         return -LIBBPF_ERRNO__RELOC;
1185                 }
1186                 new_cnt = prog->insns_cnt + text->insns_cnt;
1187                 new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn));
1188                 if (!new_insn) {
1189                         pr_warning("oom in prog realloc\n");
1190                         return -ENOMEM;
1191                 }
1192                 memcpy(new_insn + prog->insns_cnt, text->insns,
1193                        text->insns_cnt * sizeof(*insn));
1194                 prog->insns = new_insn;
1195                 prog->main_prog_cnt = prog->insns_cnt;
1196                 prog->insns_cnt = new_cnt;
1197                 pr_debug("added %zd insn from %s to prog %s\n",
1198                          text->insns_cnt, text->section_name,
1199                          prog->section_name);
1200         }
1201         insn = &prog->insns[relo->insn_idx];
1202         insn->imm += prog->main_prog_cnt - relo->insn_idx;
1203         return 0;
1204 }
1205
1206 static int
1207 bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj)
1208 {
1209         int i, err;
1210
1211         if (!prog || !prog->reloc_desc)
1212                 return 0;
1213
1214         for (i = 0; i < prog->nr_reloc; i++) {
1215                 if (prog->reloc_desc[i].type == RELO_LD64) {
1216                         struct bpf_insn *insns = prog->insns;
1217                         int insn_idx, map_idx;
1218
1219                         insn_idx = prog->reloc_desc[i].insn_idx;
1220                         map_idx = prog->reloc_desc[i].map_idx;
1221
1222                         if (insn_idx >= (int)prog->insns_cnt) {
1223                                 pr_warning("relocation out of range: '%s'\n",
1224                                            prog->section_name);
1225                                 return -LIBBPF_ERRNO__RELOC;
1226                         }
1227                         insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD;
1228                         insns[insn_idx].imm = obj->maps[map_idx].fd;
1229                 } else {
1230                         err = bpf_program__reloc_text(prog, obj,
1231                                                       &prog->reloc_desc[i]);
1232                         if (err)
1233                                 return err;
1234                 }
1235         }
1236
1237         zfree(&prog->reloc_desc);
1238         prog->nr_reloc = 0;
1239         return 0;
1240 }
1241
1242
1243 static int
1244 bpf_object__relocate(struct bpf_object *obj)
1245 {
1246         struct bpf_program *prog;
1247         size_t i;
1248         int err;
1249
1250         for (i = 0; i < obj->nr_programs; i++) {
1251                 prog = &obj->programs[i];
1252
1253                 err = bpf_program__relocate(prog, obj);
1254                 if (err) {
1255                         pr_warning("failed to relocate '%s'\n",
1256                                    prog->section_name);
1257                         return err;
1258                 }
1259         }
1260         return 0;
1261 }
1262
1263 static int bpf_object__collect_reloc(struct bpf_object *obj)
1264 {
1265         int i, err;
1266
1267         if (!obj_elf_valid(obj)) {
1268                 pr_warning("Internal error: elf object is closed\n");
1269                 return -LIBBPF_ERRNO__INTERNAL;
1270         }
1271
1272         for (i = 0; i < obj->efile.nr_reloc; i++) {
1273                 GElf_Shdr *shdr = &obj->efile.reloc[i].shdr;
1274                 Elf_Data *data = obj->efile.reloc[i].data;
1275                 int idx = shdr->sh_info;
1276                 struct bpf_program *prog;
1277
1278                 if (shdr->sh_type != SHT_REL) {
1279                         pr_warning("internal error at %d\n", __LINE__);
1280                         return -LIBBPF_ERRNO__INTERNAL;
1281                 }
1282
1283                 prog = bpf_object__find_prog_by_idx(obj, idx);
1284                 if (!prog) {
1285                         pr_warning("relocation failed: no section(%d)\n", idx);
1286                         return -LIBBPF_ERRNO__RELOC;
1287                 }
1288
1289                 err = bpf_program__collect_reloc(prog,
1290                                                  shdr, data,
1291                                                  obj);
1292                 if (err)
1293                         return err;
1294         }
1295         return 0;
1296 }
1297
1298 static int
1299 load_program(enum bpf_prog_type type, enum bpf_attach_type expected_attach_type,
1300              const char *name, struct bpf_insn *insns, int insns_cnt,
1301              char *license, __u32 kern_version, int *pfd, int prog_ifindex)
1302 {
1303         struct bpf_load_program_attr load_attr;
1304         char *cp, errmsg[STRERR_BUFSIZE];
1305         char *log_buf;
1306         int ret;
1307
1308         memset(&load_attr, 0, sizeof(struct bpf_load_program_attr));
1309         load_attr.prog_type = type;
1310         load_attr.expected_attach_type = expected_attach_type;
1311         load_attr.name = name;
1312         load_attr.insns = insns;
1313         load_attr.insns_cnt = insns_cnt;
1314         load_attr.license = license;
1315         load_attr.kern_version = kern_version;
1316         load_attr.prog_ifindex = prog_ifindex;
1317
1318         if (!load_attr.insns || !load_attr.insns_cnt)
1319                 return -EINVAL;
1320
1321         log_buf = malloc(BPF_LOG_BUF_SIZE);
1322         if (!log_buf)
1323                 pr_warning("Alloc log buffer for bpf loader error, continue without log\n");
1324
1325         ret = bpf_load_program_xattr(&load_attr, log_buf, BPF_LOG_BUF_SIZE);
1326
1327         if (ret >= 0) {
1328                 *pfd = ret;
1329                 ret = 0;
1330                 goto out;
1331         }
1332
1333         ret = -LIBBPF_ERRNO__LOAD;
1334         cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1335         pr_warning("load bpf program failed: %s\n", cp);
1336
1337         if (log_buf && log_buf[0] != '\0') {
1338                 ret = -LIBBPF_ERRNO__VERIFY;
1339                 pr_warning("-- BEGIN DUMP LOG ---\n");
1340                 pr_warning("\n%s\n", log_buf);
1341                 pr_warning("-- END LOG --\n");
1342         } else if (load_attr.insns_cnt >= BPF_MAXINSNS) {
1343                 pr_warning("Program too large (%zu insns), at most %d insns\n",
1344                            load_attr.insns_cnt, BPF_MAXINSNS);
1345                 ret = -LIBBPF_ERRNO__PROG2BIG;
1346         } else {
1347                 /* Wrong program type? */
1348                 if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) {
1349                         int fd;
1350
1351                         load_attr.prog_type = BPF_PROG_TYPE_KPROBE;
1352                         load_attr.expected_attach_type = 0;
1353                         fd = bpf_load_program_xattr(&load_attr, NULL, 0);
1354                         if (fd >= 0) {
1355                                 close(fd);
1356                                 ret = -LIBBPF_ERRNO__PROGTYPE;
1357                                 goto out;
1358                         }
1359                 }
1360
1361                 if (log_buf)
1362                         ret = -LIBBPF_ERRNO__KVER;
1363         }
1364
1365 out:
1366         free(log_buf);
1367         return ret;
1368 }
1369
1370 int
1371 bpf_program__load(struct bpf_program *prog,
1372                   char *license, __u32 kern_version)
1373 {
1374         int err = 0, fd, i;
1375
1376         if (prog->instances.nr < 0 || !prog->instances.fds) {
1377                 if (prog->preprocessor) {
1378                         pr_warning("Internal error: can't load program '%s'\n",
1379                                    prog->section_name);
1380                         return -LIBBPF_ERRNO__INTERNAL;
1381                 }
1382
1383                 prog->instances.fds = malloc(sizeof(int));
1384                 if (!prog->instances.fds) {
1385                         pr_warning("Not enough memory for BPF fds\n");
1386                         return -ENOMEM;
1387                 }
1388                 prog->instances.nr = 1;
1389                 prog->instances.fds[0] = -1;
1390         }
1391
1392         if (!prog->preprocessor) {
1393                 if (prog->instances.nr != 1) {
1394                         pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n",
1395                                    prog->section_name, prog->instances.nr);
1396                 }
1397                 err = load_program(prog->type, prog->expected_attach_type,
1398                                    prog->name, prog->insns, prog->insns_cnt,
1399                                    license, kern_version, &fd,
1400                                    prog->prog_ifindex);
1401                 if (!err)
1402                         prog->instances.fds[0] = fd;
1403                 goto out;
1404         }
1405
1406         for (i = 0; i < prog->instances.nr; i++) {
1407                 struct bpf_prog_prep_result result;
1408                 bpf_program_prep_t preprocessor = prog->preprocessor;
1409
1410                 bzero(&result, sizeof(result));
1411                 err = preprocessor(prog, i, prog->insns,
1412                                    prog->insns_cnt, &result);
1413                 if (err) {
1414                         pr_warning("Preprocessing the %dth instance of program '%s' failed\n",
1415                                    i, prog->section_name);
1416                         goto out;
1417                 }
1418
1419                 if (!result.new_insn_ptr || !result.new_insn_cnt) {
1420                         pr_debug("Skip loading the %dth instance of program '%s'\n",
1421                                  i, prog->section_name);
1422                         prog->instances.fds[i] = -1;
1423                         if (result.pfd)
1424                                 *result.pfd = -1;
1425                         continue;
1426                 }
1427
1428                 err = load_program(prog->type, prog->expected_attach_type,
1429                                    prog->name, result.new_insn_ptr,
1430                                    result.new_insn_cnt,
1431                                    license, kern_version, &fd,
1432                                    prog->prog_ifindex);
1433
1434                 if (err) {
1435                         pr_warning("Loading the %dth instance of program '%s' failed\n",
1436                                         i, prog->section_name);
1437                         goto out;
1438                 }
1439
1440                 if (result.pfd)
1441                         *result.pfd = fd;
1442                 prog->instances.fds[i] = fd;
1443         }
1444 out:
1445         if (err)
1446                 pr_warning("failed to load program '%s'\n",
1447                            prog->section_name);
1448         zfree(&prog->insns);
1449         prog->insns_cnt = 0;
1450         return err;
1451 }
1452
1453 static bool bpf_program__is_function_storage(struct bpf_program *prog,
1454                                              struct bpf_object *obj)
1455 {
1456         return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls;
1457 }
1458
1459 static int
1460 bpf_object__load_progs(struct bpf_object *obj)
1461 {
1462         size_t i;
1463         int err;
1464
1465         for (i = 0; i < obj->nr_programs; i++) {
1466                 if (bpf_program__is_function_storage(&obj->programs[i], obj))
1467                         continue;
1468                 err = bpf_program__load(&obj->programs[i],
1469                                         obj->license,
1470                                         obj->kern_version);
1471                 if (err)
1472                         return err;
1473         }
1474         return 0;
1475 }
1476
1477 static bool bpf_prog_type__needs_kver(enum bpf_prog_type type)
1478 {
1479         switch (type) {
1480         case BPF_PROG_TYPE_SOCKET_FILTER:
1481         case BPF_PROG_TYPE_SCHED_CLS:
1482         case BPF_PROG_TYPE_SCHED_ACT:
1483         case BPF_PROG_TYPE_XDP:
1484         case BPF_PROG_TYPE_CGROUP_SKB:
1485         case BPF_PROG_TYPE_CGROUP_SOCK:
1486         case BPF_PROG_TYPE_LWT_IN:
1487         case BPF_PROG_TYPE_LWT_OUT:
1488         case BPF_PROG_TYPE_LWT_XMIT:
1489         case BPF_PROG_TYPE_LWT_SEG6LOCAL:
1490         case BPF_PROG_TYPE_SOCK_OPS:
1491         case BPF_PROG_TYPE_SK_SKB:
1492         case BPF_PROG_TYPE_CGROUP_DEVICE:
1493         case BPF_PROG_TYPE_SK_MSG:
1494         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
1495         case BPF_PROG_TYPE_LIRC_MODE2:
1496         case BPF_PROG_TYPE_SK_REUSEPORT:
1497         case BPF_PROG_TYPE_FLOW_DISSECTOR:
1498                 return false;
1499         case BPF_PROG_TYPE_UNSPEC:
1500         case BPF_PROG_TYPE_KPROBE:
1501         case BPF_PROG_TYPE_TRACEPOINT:
1502         case BPF_PROG_TYPE_PERF_EVENT:
1503         case BPF_PROG_TYPE_RAW_TRACEPOINT:
1504         default:
1505                 return true;
1506         }
1507 }
1508
1509 static int bpf_object__validate(struct bpf_object *obj, bool needs_kver)
1510 {
1511         if (needs_kver && obj->kern_version == 0) {
1512                 pr_warning("%s doesn't provide kernel version\n",
1513                            obj->path);
1514                 return -LIBBPF_ERRNO__KVERSION;
1515         }
1516         return 0;
1517 }
1518
1519 static struct bpf_object *
1520 __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz,
1521                    bool needs_kver, int flags)
1522 {
1523         struct bpf_object *obj;
1524         int err;
1525
1526         if (elf_version(EV_CURRENT) == EV_NONE) {
1527                 pr_warning("failed to init libelf for %s\n", path);
1528                 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
1529         }
1530
1531         obj = bpf_object__new(path, obj_buf, obj_buf_sz);
1532         if (IS_ERR(obj))
1533                 return obj;
1534
1535         CHECK_ERR(bpf_object__elf_init(obj), err, out);
1536         CHECK_ERR(bpf_object__check_endianness(obj), err, out);
1537         CHECK_ERR(bpf_object__elf_collect(obj, flags), err, out);
1538         CHECK_ERR(bpf_object__collect_reloc(obj), err, out);
1539         CHECK_ERR(bpf_object__validate(obj, needs_kver), err, out);
1540
1541         bpf_object__elf_finish(obj);
1542         return obj;
1543 out:
1544         bpf_object__close(obj);
1545         return ERR_PTR(err);
1546 }
1547
1548 struct bpf_object *__bpf_object__open_xattr(struct bpf_object_open_attr *attr,
1549                                             int flags)
1550 {
1551         /* param validation */
1552         if (!attr->file)
1553                 return NULL;
1554
1555         pr_debug("loading %s\n", attr->file);
1556
1557         return __bpf_object__open(attr->file, NULL, 0,
1558                                   bpf_prog_type__needs_kver(attr->prog_type),
1559                                   flags);
1560 }
1561
1562 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr)
1563 {
1564         return __bpf_object__open_xattr(attr, 0);
1565 }
1566
1567 struct bpf_object *bpf_object__open(const char *path)
1568 {
1569         struct bpf_object_open_attr attr = {
1570                 .file           = path,
1571                 .prog_type      = BPF_PROG_TYPE_UNSPEC,
1572         };
1573
1574         return bpf_object__open_xattr(&attr);
1575 }
1576
1577 struct bpf_object *bpf_object__open_buffer(void *obj_buf,
1578                                            size_t obj_buf_sz,
1579                                            const char *name)
1580 {
1581         char tmp_name[64];
1582
1583         /* param validation */
1584         if (!obj_buf || obj_buf_sz <= 0)
1585                 return NULL;
1586
1587         if (!name) {
1588                 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
1589                          (unsigned long)obj_buf,
1590                          (unsigned long)obj_buf_sz);
1591                 tmp_name[sizeof(tmp_name) - 1] = '\0';
1592                 name = tmp_name;
1593         }
1594         pr_debug("loading object '%s' from buffer\n",
1595                  name);
1596
1597         return __bpf_object__open(name, obj_buf, obj_buf_sz, true, true);
1598 }
1599
1600 int bpf_object__unload(struct bpf_object *obj)
1601 {
1602         size_t i;
1603
1604         if (!obj)
1605                 return -EINVAL;
1606
1607         for (i = 0; i < obj->nr_maps; i++)
1608                 zclose(obj->maps[i].fd);
1609
1610         for (i = 0; i < obj->nr_programs; i++)
1611                 bpf_program__unload(&obj->programs[i]);
1612
1613         return 0;
1614 }
1615
1616 int bpf_object__load(struct bpf_object *obj)
1617 {
1618         int err;
1619
1620         if (!obj)
1621                 return -EINVAL;
1622
1623         if (obj->loaded) {
1624                 pr_warning("object should not be loaded twice\n");
1625                 return -EINVAL;
1626         }
1627
1628         obj->loaded = true;
1629
1630         CHECK_ERR(bpf_object__create_maps(obj), err, out);
1631         CHECK_ERR(bpf_object__relocate(obj), err, out);
1632         CHECK_ERR(bpf_object__load_progs(obj), err, out);
1633
1634         return 0;
1635 out:
1636         bpf_object__unload(obj);
1637         pr_warning("failed to load object '%s'\n", obj->path);
1638         return err;
1639 }
1640
1641 static int check_path(const char *path)
1642 {
1643         char *cp, errmsg[STRERR_BUFSIZE];
1644         struct statfs st_fs;
1645         char *dname, *dir;
1646         int err = 0;
1647
1648         if (path == NULL)
1649                 return -EINVAL;
1650
1651         dname = strdup(path);
1652         if (dname == NULL)
1653                 return -ENOMEM;
1654
1655         dir = dirname(dname);
1656         if (statfs(dir, &st_fs)) {
1657                 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1658                 pr_warning("failed to statfs %s: %s\n", dir, cp);
1659                 err = -errno;
1660         }
1661         free(dname);
1662
1663         if (!err && st_fs.f_type != BPF_FS_MAGIC) {
1664                 pr_warning("specified path %s is not on BPF FS\n", path);
1665                 err = -EINVAL;
1666         }
1667
1668         return err;
1669 }
1670
1671 int bpf_program__pin_instance(struct bpf_program *prog, const char *path,
1672                               int instance)
1673 {
1674         char *cp, errmsg[STRERR_BUFSIZE];
1675         int err;
1676
1677         err = check_path(path);
1678         if (err)
1679                 return err;
1680
1681         if (prog == NULL) {
1682                 pr_warning("invalid program pointer\n");
1683                 return -EINVAL;
1684         }
1685
1686         if (instance < 0 || instance >= prog->instances.nr) {
1687                 pr_warning("invalid prog instance %d of prog %s (max %d)\n",
1688                            instance, prog->section_name, prog->instances.nr);
1689                 return -EINVAL;
1690         }
1691
1692         if (bpf_obj_pin(prog->instances.fds[instance], path)) {
1693                 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1694                 pr_warning("failed to pin program: %s\n", cp);
1695                 return -errno;
1696         }
1697         pr_debug("pinned program '%s'\n", path);
1698
1699         return 0;
1700 }
1701
1702 static int make_dir(const char *path)
1703 {
1704         char *cp, errmsg[STRERR_BUFSIZE];
1705         int err = 0;
1706
1707         if (mkdir(path, 0700) && errno != EEXIST)
1708                 err = -errno;
1709
1710         if (err) {
1711                 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
1712                 pr_warning("failed to mkdir %s: %s\n", path, cp);
1713         }
1714         return err;
1715 }
1716
1717 int bpf_program__pin(struct bpf_program *prog, const char *path)
1718 {
1719         int i, err;
1720
1721         err = check_path(path);
1722         if (err)
1723                 return err;
1724
1725         if (prog == NULL) {
1726                 pr_warning("invalid program pointer\n");
1727                 return -EINVAL;
1728         }
1729
1730         if (prog->instances.nr <= 0) {
1731                 pr_warning("no instances of prog %s to pin\n",
1732                            prog->section_name);
1733                 return -EINVAL;
1734         }
1735
1736         err = make_dir(path);
1737         if (err)
1738                 return err;
1739
1740         for (i = 0; i < prog->instances.nr; i++) {
1741                 char buf[PATH_MAX];
1742                 int len;
1743
1744                 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
1745                 if (len < 0)
1746                         return -EINVAL;
1747                 else if (len >= PATH_MAX)
1748                         return -ENAMETOOLONG;
1749
1750                 err = bpf_program__pin_instance(prog, buf, i);
1751                 if (err)
1752                         return err;
1753         }
1754
1755         return 0;
1756 }
1757
1758 int bpf_map__pin(struct bpf_map *map, const char *path)
1759 {
1760         char *cp, errmsg[STRERR_BUFSIZE];
1761         int err;
1762
1763         err = check_path(path);
1764         if (err)
1765                 return err;
1766
1767         if (map == NULL) {
1768                 pr_warning("invalid map pointer\n");
1769                 return -EINVAL;
1770         }
1771
1772         if (bpf_obj_pin(map->fd, path)) {
1773                 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1774                 pr_warning("failed to pin map: %s\n", cp);
1775                 return -errno;
1776         }
1777
1778         pr_debug("pinned map '%s'\n", path);
1779         return 0;
1780 }
1781
1782 int bpf_object__pin(struct bpf_object *obj, const char *path)
1783 {
1784         struct bpf_program *prog;
1785         struct bpf_map *map;
1786         int err;
1787
1788         if (!obj)
1789                 return -ENOENT;
1790
1791         if (!obj->loaded) {
1792                 pr_warning("object not yet loaded; load it first\n");
1793                 return -ENOENT;
1794         }
1795
1796         err = make_dir(path);
1797         if (err)
1798                 return err;
1799
1800         bpf_map__for_each(map, obj) {
1801                 char buf[PATH_MAX];
1802                 int len;
1803
1804                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
1805                                bpf_map__name(map));
1806                 if (len < 0)
1807                         return -EINVAL;
1808                 else if (len >= PATH_MAX)
1809                         return -ENAMETOOLONG;
1810
1811                 err = bpf_map__pin(map, buf);
1812                 if (err)
1813                         return err;
1814         }
1815
1816         bpf_object__for_each_program(prog, obj) {
1817                 char buf[PATH_MAX];
1818                 int len;
1819
1820                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
1821                                prog->section_name);
1822                 if (len < 0)
1823                         return -EINVAL;
1824                 else if (len >= PATH_MAX)
1825                         return -ENAMETOOLONG;
1826
1827                 err = bpf_program__pin(prog, buf);
1828                 if (err)
1829                         return err;
1830         }
1831
1832         return 0;
1833 }
1834
1835 void bpf_object__close(struct bpf_object *obj)
1836 {
1837         size_t i;
1838
1839         if (!obj)
1840                 return;
1841
1842         if (obj->clear_priv)
1843                 obj->clear_priv(obj, obj->priv);
1844
1845         bpf_object__elf_finish(obj);
1846         bpf_object__unload(obj);
1847         btf__free(obj->btf);
1848
1849         for (i = 0; i < obj->nr_maps; i++) {
1850                 zfree(&obj->maps[i].name);
1851                 if (obj->maps[i].clear_priv)
1852                         obj->maps[i].clear_priv(&obj->maps[i],
1853                                                 obj->maps[i].priv);
1854                 obj->maps[i].priv = NULL;
1855                 obj->maps[i].clear_priv = NULL;
1856         }
1857         zfree(&obj->maps);
1858         obj->nr_maps = 0;
1859
1860         if (obj->programs && obj->nr_programs) {
1861                 for (i = 0; i < obj->nr_programs; i++)
1862                         bpf_program__exit(&obj->programs[i]);
1863         }
1864         zfree(&obj->programs);
1865
1866         list_del(&obj->list);
1867         free(obj);
1868 }
1869
1870 struct bpf_object *
1871 bpf_object__next(struct bpf_object *prev)
1872 {
1873         struct bpf_object *next;
1874
1875         if (!prev)
1876                 next = list_first_entry(&bpf_objects_list,
1877                                         struct bpf_object,
1878                                         list);
1879         else
1880                 next = list_next_entry(prev, list);
1881
1882         /* Empty list is noticed here so don't need checking on entry. */
1883         if (&next->list == &bpf_objects_list)
1884                 return NULL;
1885
1886         return next;
1887 }
1888
1889 const char *bpf_object__name(struct bpf_object *obj)
1890 {
1891         return obj ? obj->path : ERR_PTR(-EINVAL);
1892 }
1893
1894 unsigned int bpf_object__kversion(struct bpf_object *obj)
1895 {
1896         return obj ? obj->kern_version : 0;
1897 }
1898
1899 int bpf_object__btf_fd(const struct bpf_object *obj)
1900 {
1901         return obj->btf ? btf__fd(obj->btf) : -1;
1902 }
1903
1904 int bpf_object__set_priv(struct bpf_object *obj, void *priv,
1905                          bpf_object_clear_priv_t clear_priv)
1906 {
1907         if (obj->priv && obj->clear_priv)
1908                 obj->clear_priv(obj, obj->priv);
1909
1910         obj->priv = priv;
1911         obj->clear_priv = clear_priv;
1912         return 0;
1913 }
1914
1915 void *bpf_object__priv(struct bpf_object *obj)
1916 {
1917         return obj ? obj->priv : ERR_PTR(-EINVAL);
1918 }
1919
1920 static struct bpf_program *
1921 __bpf_program__next(struct bpf_program *prev, struct bpf_object *obj)
1922 {
1923         size_t idx;
1924
1925         if (!obj->programs)
1926                 return NULL;
1927         /* First handler */
1928         if (prev == NULL)
1929                 return &obj->programs[0];
1930
1931         if (prev->obj != obj) {
1932                 pr_warning("error: program handler doesn't match object\n");
1933                 return NULL;
1934         }
1935
1936         idx = (prev - obj->programs) + 1;
1937         if (idx >= obj->nr_programs)
1938                 return NULL;
1939         return &obj->programs[idx];
1940 }
1941
1942 struct bpf_program *
1943 bpf_program__next(struct bpf_program *prev, struct bpf_object *obj)
1944 {
1945         struct bpf_program *prog = prev;
1946
1947         do {
1948                 prog = __bpf_program__next(prog, obj);
1949         } while (prog && bpf_program__is_function_storage(prog, obj));
1950
1951         return prog;
1952 }
1953
1954 int bpf_program__set_priv(struct bpf_program *prog, void *priv,
1955                           bpf_program_clear_priv_t clear_priv)
1956 {
1957         if (prog->priv && prog->clear_priv)
1958                 prog->clear_priv(prog, prog->priv);
1959
1960         prog->priv = priv;
1961         prog->clear_priv = clear_priv;
1962         return 0;
1963 }
1964
1965 void *bpf_program__priv(struct bpf_program *prog)
1966 {
1967         return prog ? prog->priv : ERR_PTR(-EINVAL);
1968 }
1969
1970 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
1971 {
1972         prog->prog_ifindex = ifindex;
1973 }
1974
1975 const char *bpf_program__title(struct bpf_program *prog, bool needs_copy)
1976 {
1977         const char *title;
1978
1979         title = prog->section_name;
1980         if (needs_copy) {
1981                 title = strdup(title);
1982                 if (!title) {
1983                         pr_warning("failed to strdup program title\n");
1984                         return ERR_PTR(-ENOMEM);
1985                 }
1986         }
1987
1988         return title;
1989 }
1990
1991 int bpf_program__fd(struct bpf_program *prog)
1992 {
1993         return bpf_program__nth_fd(prog, 0);
1994 }
1995
1996 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances,
1997                           bpf_program_prep_t prep)
1998 {
1999         int *instances_fds;
2000
2001         if (nr_instances <= 0 || !prep)
2002                 return -EINVAL;
2003
2004         if (prog->instances.nr > 0 || prog->instances.fds) {
2005                 pr_warning("Can't set pre-processor after loading\n");
2006                 return -EINVAL;
2007         }
2008
2009         instances_fds = malloc(sizeof(int) * nr_instances);
2010         if (!instances_fds) {
2011                 pr_warning("alloc memory failed for fds\n");
2012                 return -ENOMEM;
2013         }
2014
2015         /* fill all fd with -1 */
2016         memset(instances_fds, -1, sizeof(int) * nr_instances);
2017
2018         prog->instances.nr = nr_instances;
2019         prog->instances.fds = instances_fds;
2020         prog->preprocessor = prep;
2021         return 0;
2022 }
2023
2024 int bpf_program__nth_fd(struct bpf_program *prog, int n)
2025 {
2026         int fd;
2027
2028         if (!prog)
2029                 return -EINVAL;
2030
2031         if (n >= prog->instances.nr || n < 0) {
2032                 pr_warning("Can't get the %dth fd from program %s: only %d instances\n",
2033                            n, prog->section_name, prog->instances.nr);
2034                 return -EINVAL;
2035         }
2036
2037         fd = prog->instances.fds[n];
2038         if (fd < 0) {
2039                 pr_warning("%dth instance of program '%s' is invalid\n",
2040                            n, prog->section_name);
2041                 return -ENOENT;
2042         }
2043
2044         return fd;
2045 }
2046
2047 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
2048 {
2049         prog->type = type;
2050 }
2051
2052 static bool bpf_program__is_type(struct bpf_program *prog,
2053                                  enum bpf_prog_type type)
2054 {
2055         return prog ? (prog->type == type) : false;
2056 }
2057
2058 #define BPF_PROG_TYPE_FNS(NAME, TYPE)                   \
2059 int bpf_program__set_##NAME(struct bpf_program *prog)   \
2060 {                                                       \
2061         if (!prog)                                      \
2062                 return -EINVAL;                         \
2063         bpf_program__set_type(prog, TYPE);              \
2064         return 0;                                       \
2065 }                                                       \
2066                                                         \
2067 bool bpf_program__is_##NAME(struct bpf_program *prog)   \
2068 {                                                       \
2069         return bpf_program__is_type(prog, TYPE);        \
2070 }                                                       \
2071
2072 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER);
2073 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE);
2074 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS);
2075 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT);
2076 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT);
2077 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT);
2078 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP);
2079 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT);
2080
2081 void bpf_program__set_expected_attach_type(struct bpf_program *prog,
2082                                            enum bpf_attach_type type)
2083 {
2084         prog->expected_attach_type = type;
2085 }
2086
2087 #define BPF_PROG_SEC_IMPL(string, ptype, eatype, is_attachable, atype) \
2088         { string, sizeof(string) - 1, ptype, eatype, is_attachable, atype }
2089
2090 /* Programs that can NOT be attached. */
2091 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0)
2092
2093 /* Programs that can be attached. */
2094 #define BPF_APROG_SEC(string, ptype, atype) \
2095         BPF_PROG_SEC_IMPL(string, ptype, 0, 1, atype)
2096
2097 /* Programs that must specify expected attach type at load time. */
2098 #define BPF_EAPROG_SEC(string, ptype, eatype) \
2099         BPF_PROG_SEC_IMPL(string, ptype, eatype, 1, eatype)
2100
2101 /* Programs that can be attached but attach type can't be identified by section
2102  * name. Kept for backward compatibility.
2103  */
2104 #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype)
2105
2106 static const struct {
2107         const char *sec;
2108         size_t len;
2109         enum bpf_prog_type prog_type;
2110         enum bpf_attach_type expected_attach_type;
2111         int is_attachable;
2112         enum bpf_attach_type attach_type;
2113 } section_names[] = {
2114         BPF_PROG_SEC("socket",                  BPF_PROG_TYPE_SOCKET_FILTER),
2115         BPF_PROG_SEC("kprobe/",                 BPF_PROG_TYPE_KPROBE),
2116         BPF_PROG_SEC("kretprobe/",              BPF_PROG_TYPE_KPROBE),
2117         BPF_PROG_SEC("classifier",              BPF_PROG_TYPE_SCHED_CLS),
2118         BPF_PROG_SEC("action",                  BPF_PROG_TYPE_SCHED_ACT),
2119         BPF_PROG_SEC("tracepoint/",             BPF_PROG_TYPE_TRACEPOINT),
2120         BPF_PROG_SEC("raw_tracepoint/",         BPF_PROG_TYPE_RAW_TRACEPOINT),
2121         BPF_PROG_SEC("xdp",                     BPF_PROG_TYPE_XDP),
2122         BPF_PROG_SEC("perf_event",              BPF_PROG_TYPE_PERF_EVENT),
2123         BPF_PROG_SEC("lwt_in",                  BPF_PROG_TYPE_LWT_IN),
2124         BPF_PROG_SEC("lwt_out",                 BPF_PROG_TYPE_LWT_OUT),
2125         BPF_PROG_SEC("lwt_xmit",                BPF_PROG_TYPE_LWT_XMIT),
2126         BPF_PROG_SEC("lwt_seg6local",           BPF_PROG_TYPE_LWT_SEG6LOCAL),
2127         BPF_APROG_SEC("cgroup_skb/ingress",     BPF_PROG_TYPE_CGROUP_SKB,
2128                                                 BPF_CGROUP_INET_INGRESS),
2129         BPF_APROG_SEC("cgroup_skb/egress",      BPF_PROG_TYPE_CGROUP_SKB,
2130                                                 BPF_CGROUP_INET_EGRESS),
2131         BPF_APROG_COMPAT("cgroup/skb",          BPF_PROG_TYPE_CGROUP_SKB),
2132         BPF_APROG_SEC("cgroup/sock",            BPF_PROG_TYPE_CGROUP_SOCK,
2133                                                 BPF_CGROUP_INET_SOCK_CREATE),
2134         BPF_EAPROG_SEC("cgroup/post_bind4",     BPF_PROG_TYPE_CGROUP_SOCK,
2135                                                 BPF_CGROUP_INET4_POST_BIND),
2136         BPF_EAPROG_SEC("cgroup/post_bind6",     BPF_PROG_TYPE_CGROUP_SOCK,
2137                                                 BPF_CGROUP_INET6_POST_BIND),
2138         BPF_APROG_SEC("cgroup/dev",             BPF_PROG_TYPE_CGROUP_DEVICE,
2139                                                 BPF_CGROUP_DEVICE),
2140         BPF_APROG_SEC("sockops",                BPF_PROG_TYPE_SOCK_OPS,
2141                                                 BPF_CGROUP_SOCK_OPS),
2142         BPF_APROG_SEC("sk_skb/stream_parser",   BPF_PROG_TYPE_SK_SKB,
2143                                                 BPF_SK_SKB_STREAM_PARSER),
2144         BPF_APROG_SEC("sk_skb/stream_verdict",  BPF_PROG_TYPE_SK_SKB,
2145                                                 BPF_SK_SKB_STREAM_VERDICT),
2146         BPF_APROG_COMPAT("sk_skb",              BPF_PROG_TYPE_SK_SKB),
2147         BPF_APROG_SEC("sk_msg",                 BPF_PROG_TYPE_SK_MSG,
2148                                                 BPF_SK_MSG_VERDICT),
2149         BPF_APROG_SEC("lirc_mode2",             BPF_PROG_TYPE_LIRC_MODE2,
2150                                                 BPF_LIRC_MODE2),
2151         BPF_APROG_SEC("flow_dissector",         BPF_PROG_TYPE_FLOW_DISSECTOR,
2152                                                 BPF_FLOW_DISSECTOR),
2153         BPF_EAPROG_SEC("cgroup/bind4",          BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
2154                                                 BPF_CGROUP_INET4_BIND),
2155         BPF_EAPROG_SEC("cgroup/bind6",          BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
2156                                                 BPF_CGROUP_INET6_BIND),
2157         BPF_EAPROG_SEC("cgroup/connect4",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
2158                                                 BPF_CGROUP_INET4_CONNECT),
2159         BPF_EAPROG_SEC("cgroup/connect6",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
2160                                                 BPF_CGROUP_INET6_CONNECT),
2161         BPF_EAPROG_SEC("cgroup/sendmsg4",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
2162                                                 BPF_CGROUP_UDP4_SENDMSG),
2163         BPF_EAPROG_SEC("cgroup/sendmsg6",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
2164                                                 BPF_CGROUP_UDP6_SENDMSG),
2165 };
2166
2167 #undef BPF_PROG_SEC_IMPL
2168 #undef BPF_PROG_SEC
2169 #undef BPF_APROG_SEC
2170 #undef BPF_EAPROG_SEC
2171 #undef BPF_APROG_COMPAT
2172
2173 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
2174                              enum bpf_attach_type *expected_attach_type)
2175 {
2176         int i;
2177
2178         if (!name)
2179                 return -EINVAL;
2180
2181         for (i = 0; i < ARRAY_SIZE(section_names); i++) {
2182                 if (strncmp(name, section_names[i].sec, section_names[i].len))
2183                         continue;
2184                 *prog_type = section_names[i].prog_type;
2185                 *expected_attach_type = section_names[i].expected_attach_type;
2186                 return 0;
2187         }
2188         return -EINVAL;
2189 }
2190
2191 int libbpf_attach_type_by_name(const char *name,
2192                                enum bpf_attach_type *attach_type)
2193 {
2194         int i;
2195
2196         if (!name)
2197                 return -EINVAL;
2198
2199         for (i = 0; i < ARRAY_SIZE(section_names); i++) {
2200                 if (strncmp(name, section_names[i].sec, section_names[i].len))
2201                         continue;
2202                 if (!section_names[i].is_attachable)
2203                         return -EINVAL;
2204                 *attach_type = section_names[i].attach_type;
2205                 return 0;
2206         }
2207         return -EINVAL;
2208 }
2209
2210 static int
2211 bpf_program__identify_section(struct bpf_program *prog,
2212                               enum bpf_prog_type *prog_type,
2213                               enum bpf_attach_type *expected_attach_type)
2214 {
2215         return libbpf_prog_type_by_name(prog->section_name, prog_type,
2216                                         expected_attach_type);
2217 }
2218
2219 int bpf_map__fd(struct bpf_map *map)
2220 {
2221         return map ? map->fd : -EINVAL;
2222 }
2223
2224 const struct bpf_map_def *bpf_map__def(struct bpf_map *map)
2225 {
2226         return map ? &map->def : ERR_PTR(-EINVAL);
2227 }
2228
2229 const char *bpf_map__name(struct bpf_map *map)
2230 {
2231         return map ? map->name : NULL;
2232 }
2233
2234 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map)
2235 {
2236         return map ? map->btf_key_type_id : 0;
2237 }
2238
2239 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map)
2240 {
2241         return map ? map->btf_value_type_id : 0;
2242 }
2243
2244 int bpf_map__set_priv(struct bpf_map *map, void *priv,
2245                      bpf_map_clear_priv_t clear_priv)
2246 {
2247         if (!map)
2248                 return -EINVAL;
2249
2250         if (map->priv) {
2251                 if (map->clear_priv)
2252                         map->clear_priv(map, map->priv);
2253         }
2254
2255         map->priv = priv;
2256         map->clear_priv = clear_priv;
2257         return 0;
2258 }
2259
2260 void *bpf_map__priv(struct bpf_map *map)
2261 {
2262         return map ? map->priv : ERR_PTR(-EINVAL);
2263 }
2264
2265 bool bpf_map__is_offload_neutral(struct bpf_map *map)
2266 {
2267         return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
2268 }
2269
2270 void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex)
2271 {
2272         map->map_ifindex = ifindex;
2273 }
2274
2275 struct bpf_map *
2276 bpf_map__next(struct bpf_map *prev, struct bpf_object *obj)
2277 {
2278         size_t idx;
2279         struct bpf_map *s, *e;
2280
2281         if (!obj || !obj->maps)
2282                 return NULL;
2283
2284         s = obj->maps;
2285         e = obj->maps + obj->nr_maps;
2286
2287         if (prev == NULL)
2288                 return s;
2289
2290         if ((prev < s) || (prev >= e)) {
2291                 pr_warning("error in %s: map handler doesn't belong to object\n",
2292                            __func__);
2293                 return NULL;
2294         }
2295
2296         idx = (prev - obj->maps) + 1;
2297         if (idx >= obj->nr_maps)
2298                 return NULL;
2299         return &obj->maps[idx];
2300 }
2301
2302 struct bpf_map *
2303 bpf_object__find_map_by_name(struct bpf_object *obj, const char *name)
2304 {
2305         struct bpf_map *pos;
2306
2307         bpf_map__for_each(pos, obj) {
2308                 if (pos->name && !strcmp(pos->name, name))
2309                         return pos;
2310         }
2311         return NULL;
2312 }
2313
2314 struct bpf_map *
2315 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset)
2316 {
2317         int i;
2318
2319         for (i = 0; i < obj->nr_maps; i++) {
2320                 if (obj->maps[i].offset == offset)
2321                         return &obj->maps[i];
2322         }
2323         return ERR_PTR(-ENOENT);
2324 }
2325
2326 long libbpf_get_error(const void *ptr)
2327 {
2328         if (IS_ERR(ptr))
2329                 return PTR_ERR(ptr);
2330         return 0;
2331 }
2332
2333 int bpf_prog_load(const char *file, enum bpf_prog_type type,
2334                   struct bpf_object **pobj, int *prog_fd)
2335 {
2336         struct bpf_prog_load_attr attr;
2337
2338         memset(&attr, 0, sizeof(struct bpf_prog_load_attr));
2339         attr.file = file;
2340         attr.prog_type = type;
2341         attr.expected_attach_type = 0;
2342
2343         return bpf_prog_load_xattr(&attr, pobj, prog_fd);
2344 }
2345
2346 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr,
2347                         struct bpf_object **pobj, int *prog_fd)
2348 {
2349         struct bpf_object_open_attr open_attr = {
2350                 .file           = attr->file,
2351                 .prog_type      = attr->prog_type,
2352         };
2353         struct bpf_program *prog, *first_prog = NULL;
2354         enum bpf_attach_type expected_attach_type;
2355         enum bpf_prog_type prog_type;
2356         struct bpf_object *obj;
2357         struct bpf_map *map;
2358         int err;
2359
2360         if (!attr)
2361                 return -EINVAL;
2362         if (!attr->file)
2363                 return -EINVAL;
2364
2365         obj = bpf_object__open_xattr(&open_attr);
2366         if (IS_ERR_OR_NULL(obj))
2367                 return -ENOENT;
2368
2369         bpf_object__for_each_program(prog, obj) {
2370                 /*
2371                  * If type is not specified, try to guess it based on
2372                  * section name.
2373                  */
2374                 prog_type = attr->prog_type;
2375                 prog->prog_ifindex = attr->ifindex;
2376                 expected_attach_type = attr->expected_attach_type;
2377                 if (prog_type == BPF_PROG_TYPE_UNSPEC) {
2378                         err = bpf_program__identify_section(prog, &prog_type,
2379                                                             &expected_attach_type);
2380                         if (err < 0) {
2381                                 pr_warning("failed to guess program type based on section name %s\n",
2382                                            prog->section_name);
2383                                 bpf_object__close(obj);
2384                                 return -EINVAL;
2385                         }
2386                 }
2387
2388                 bpf_program__set_type(prog, prog_type);
2389                 bpf_program__set_expected_attach_type(prog,
2390                                                       expected_attach_type);
2391
2392                 if (!first_prog)
2393                         first_prog = prog;
2394         }
2395
2396         bpf_map__for_each(map, obj) {
2397                 if (!bpf_map__is_offload_neutral(map))
2398                         map->map_ifindex = attr->ifindex;
2399         }
2400
2401         if (!first_prog) {
2402                 pr_warning("object file doesn't contain bpf program\n");
2403                 bpf_object__close(obj);
2404                 return -ENOENT;
2405         }
2406
2407         err = bpf_object__load(obj);
2408         if (err) {
2409                 bpf_object__close(obj);
2410                 return -EINVAL;
2411         }
2412
2413         *pobj = obj;
2414         *prog_fd = bpf_program__fd(first_prog);
2415         return 0;
2416 }
2417
2418 enum bpf_perf_event_ret
2419 bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
2420                            void **copy_mem, size_t *copy_size,
2421                            bpf_perf_event_print_t fn, void *private_data)
2422 {
2423         struct perf_event_mmap_page *header = mmap_mem;
2424         __u64 data_head = ring_buffer_read_head(header);
2425         __u64 data_tail = header->data_tail;
2426         void *base = ((__u8 *)header) + page_size;
2427         int ret = LIBBPF_PERF_EVENT_CONT;
2428         struct perf_event_header *ehdr;
2429         size_t ehdr_size;
2430
2431         while (data_head != data_tail) {
2432                 ehdr = base + (data_tail & (mmap_size - 1));
2433                 ehdr_size = ehdr->size;
2434
2435                 if (((void *)ehdr) + ehdr_size > base + mmap_size) {
2436                         void *copy_start = ehdr;
2437                         size_t len_first = base + mmap_size - copy_start;
2438                         size_t len_secnd = ehdr_size - len_first;
2439
2440                         if (*copy_size < ehdr_size) {
2441                                 free(*copy_mem);
2442                                 *copy_mem = malloc(ehdr_size);
2443                                 if (!*copy_mem) {
2444                                         *copy_size = 0;
2445                                         ret = LIBBPF_PERF_EVENT_ERROR;
2446                                         break;
2447                                 }
2448                                 *copy_size = ehdr_size;
2449                         }
2450
2451                         memcpy(*copy_mem, copy_start, len_first);
2452                         memcpy(*copy_mem + len_first, base, len_secnd);
2453                         ehdr = *copy_mem;
2454                 }
2455
2456                 ret = fn(ehdr, private_data);
2457                 data_tail += ehdr_size;
2458                 if (ret != LIBBPF_PERF_EVENT_CONT)
2459                         break;
2460         }
2461
2462         ring_buffer_write_tail(header, data_tail);
2463         return ret;
2464 }