tools/lib/bpf/libbpf.c

   1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
   2
   3 /*
   4  * Common eBPF ELF object loading operations.
   5  *
   6  * Copyright (C) 2013-2015 Alexei Starovoitov <[email protected]>
   7  * Copyright (C) 2015 Wang Nan <[email protected]>
   8  * Copyright (C) 2015 Huawei Inc.
   9  * Copyright (C) 2017 Nicira, Inc.
  10  * Copyright (C) 2019 Isovalent, Inc.
  11  */
  12
  13 #ifndef _GNU_SOURCE
  14 #define _GNU_SOURCE
  15 #endif
  16 #include <stdlib.h>
  17 #include <stdio.h>
  18 #include <stdarg.h>
  19 #include <libgen.h>
  20 #include <inttypes.h>
  21 #include <limits.h>
  22 #include <string.h>
  23 #include <unistd.h>
  24 #include <endian.h>
  25 #include <fcntl.h>
  26 #include <errno.h>
  27 #include <ctype.h>
  28 #include <asm/unistd.h>
  29 #include <linux/err.h>
  30 #include <linux/kernel.h>
  31 #include <linux/bpf.h>
  32 #include <linux/btf.h>
  33 #include <linux/filter.h>
  34 #include <linux/limits.h>
  35 #include <linux/perf_event.h>
  36 #include <linux/ring_buffer.h>
  37 #include <linux/version.h>
  38 #include <sys/epoll.h>
  39 #include <sys/ioctl.h>
  40 #include <sys/mman.h>
  41 #include <sys/stat.h>
  42 #include <sys/types.h>
  43 #include <sys/vfs.h>
  44 #include <sys/utsname.h>
  45 #include <sys/resource.h>
  46 #include <libelf.h>
  47 #include <gelf.h>
  48 #include <zlib.h>
  49
  50 #include "libbpf.h"
  51 #include "bpf.h"
  52 #include "btf.h"
  53 #include "str_error.h"
  54 #include "libbpf_internal.h"
  55 #include "hashmap.h"
  56 #include "bpf_gen_internal.h"
  57
  58 #ifndef BPF_FS_MAGIC
  59 #define BPF_FS_MAGIC            0xcafe4a11
  60 #endif
  61
  62 #define BPF_INSN_SZ (sizeof(struct bpf_insn))
  63
  64 /* vsprintf() in __base_pr() uses nonliteral format string. It may break
  65  * compilation if user enables corresponding warning. Disable it explicitly.
  66  */
  67 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
  68
  69 #define __printf(a, b)  __attribute__((format(printf, a, b)))
  70
  71 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj);
  72 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog);
  73
  74 static const char * const attach_type_name[] = {
  75         [BPF_CGROUP_INET_INGRESS]       = "cgroup_inet_ingress",
  76         [BPF_CGROUP_INET_EGRESS]        = "cgroup_inet_egress",
  77         [BPF_CGROUP_INET_SOCK_CREATE]   = "cgroup_inet_sock_create",
  78         [BPF_CGROUP_INET_SOCK_RELEASE]  = "cgroup_inet_sock_release",
  79         [BPF_CGROUP_SOCK_OPS]           = "cgroup_sock_ops",
  80         [BPF_CGROUP_DEVICE]             = "cgroup_device",
  81         [BPF_CGROUP_INET4_BIND]         = "cgroup_inet4_bind",
  82         [BPF_CGROUP_INET6_BIND]         = "cgroup_inet6_bind",
  83         [BPF_CGROUP_INET4_CONNECT]      = "cgroup_inet4_connect",
  84         [BPF_CGROUP_INET6_CONNECT]      = "cgroup_inet6_connect",
  85         [BPF_CGROUP_INET4_POST_BIND]    = "cgroup_inet4_post_bind",
  86         [BPF_CGROUP_INET6_POST_BIND]    = "cgroup_inet6_post_bind",
  87         [BPF_CGROUP_INET4_GETPEERNAME]  = "cgroup_inet4_getpeername",
  88         [BPF_CGROUP_INET6_GETPEERNAME]  = "cgroup_inet6_getpeername",
  89         [BPF_CGROUP_INET4_GETSOCKNAME]  = "cgroup_inet4_getsockname",
  90         [BPF_CGROUP_INET6_GETSOCKNAME]  = "cgroup_inet6_getsockname",
  91         [BPF_CGROUP_UDP4_SENDMSG]       = "cgroup_udp4_sendmsg",
  92         [BPF_CGROUP_UDP6_SENDMSG]       = "cgroup_udp6_sendmsg",
  93         [BPF_CGROUP_SYSCTL]             = "cgroup_sysctl",
  94         [BPF_CGROUP_UDP4_RECVMSG]       = "cgroup_udp4_recvmsg",
  95         [BPF_CGROUP_UDP6_RECVMSG]       = "cgroup_udp6_recvmsg",
  96         [BPF_CGROUP_GETSOCKOPT]         = "cgroup_getsockopt",
  97         [BPF_CGROUP_SETSOCKOPT]         = "cgroup_setsockopt",
  98         [BPF_SK_SKB_STREAM_PARSER]      = "sk_skb_stream_parser",
  99         [BPF_SK_SKB_STREAM_VERDICT]     = "sk_skb_stream_verdict",
 100         [BPF_SK_SKB_VERDICT]            = "sk_skb_verdict",
 101         [BPF_SK_MSG_VERDICT]            = "sk_msg_verdict",
 102         [BPF_LIRC_MODE2]                = "lirc_mode2",
 103         [BPF_FLOW_DISSECTOR]            = "flow_dissector",
 104         [BPF_TRACE_RAW_TP]              = "trace_raw_tp",
 105         [BPF_TRACE_FENTRY]              = "trace_fentry",
 106         [BPF_TRACE_FEXIT]               = "trace_fexit",
 107         [BPF_MODIFY_RETURN]             = "modify_return",
 108         [BPF_LSM_MAC]                   = "lsm_mac",
 109         [BPF_LSM_CGROUP]                = "lsm_cgroup",
 110         [BPF_SK_LOOKUP]                 = "sk_lookup",
 111         [BPF_TRACE_ITER]                = "trace_iter",
 112         [BPF_XDP_DEVMAP]                = "xdp_devmap",
 113         [BPF_XDP_CPUMAP]                = "xdp_cpumap",
 114         [BPF_XDP]                       = "xdp",
 115         [BPF_SK_REUSEPORT_SELECT]       = "sk_reuseport_select",
 116         [BPF_SK_REUSEPORT_SELECT_OR_MIGRATE]    = "sk_reuseport_select_or_migrate",
 117         [BPF_PERF_EVENT]                = "perf_event",
 118         [BPF_TRACE_KPROBE_MULTI]        = "trace_kprobe_multi",
 119 };
 120
 121 static const char * const link_type_name[] = {
 122         [BPF_LINK_TYPE_UNSPEC]                  = "unspec",
 123         [BPF_LINK_TYPE_RAW_TRACEPOINT]          = "raw_tracepoint",
 124         [BPF_LINK_TYPE_TRACING]                 = "tracing",
 125         [BPF_LINK_TYPE_CGROUP]                  = "cgroup",
 126         [BPF_LINK_TYPE_ITER]                    = "iter",
 127         [BPF_LINK_TYPE_NETNS]                   = "netns",
 128         [BPF_LINK_TYPE_XDP]                     = "xdp",
 129         [BPF_LINK_TYPE_PERF_EVENT]              = "perf_event",
 130         [BPF_LINK_TYPE_KPROBE_MULTI]            = "kprobe_multi",
 131         [BPF_LINK_TYPE_STRUCT_OPS]              = "struct_ops",
 132 };
 133
 134 static const char * const map_type_name[] = {
 135         [BPF_MAP_TYPE_UNSPEC]                   = "unspec",
 136         [BPF_MAP_TYPE_HASH]                     = "hash",
 137         [BPF_MAP_TYPE_ARRAY]                    = "array",
 138         [BPF_MAP_TYPE_PROG_ARRAY]               = "prog_array",
 139         [BPF_MAP_TYPE_PERF_EVENT_ARRAY]         = "perf_event_array",
 140         [BPF_MAP_TYPE_PERCPU_HASH]              = "percpu_hash",
 141         [BPF_MAP_TYPE_PERCPU_ARRAY]             = "percpu_array",
 142         [BPF_MAP_TYPE_STACK_TRACE]              = "stack_trace",
 143         [BPF_MAP_TYPE_CGROUP_ARRAY]             = "cgroup_array",
 144         [BPF_MAP_TYPE_LRU_HASH]                 = "lru_hash",
 145         [BPF_MAP_TYPE_LRU_PERCPU_HASH]          = "lru_percpu_hash",
 146         [BPF_MAP_TYPE_LPM_TRIE]                 = "lpm_trie",
 147         [BPF_MAP_TYPE_ARRAY_OF_MAPS]            = "array_of_maps",
 148         [BPF_MAP_TYPE_HASH_OF_MAPS]             = "hash_of_maps",
 149         [BPF_MAP_TYPE_DEVMAP]                   = "devmap",
 150         [BPF_MAP_TYPE_DEVMAP_HASH]              = "devmap_hash",
 151         [BPF_MAP_TYPE_SOCKMAP]                  = "sockmap",
 152         [BPF_MAP_TYPE_CPUMAP]                   = "cpumap",
 153         [BPF_MAP_TYPE_XSKMAP]                   = "xskmap",
 154         [BPF_MAP_TYPE_SOCKHASH]                 = "sockhash",
 155         [BPF_MAP_TYPE_CGROUP_STORAGE]           = "cgroup_storage",
 156         [BPF_MAP_TYPE_REUSEPORT_SOCKARRAY]      = "reuseport_sockarray",
 157         [BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE]    = "percpu_cgroup_storage",
 158         [BPF_MAP_TYPE_QUEUE]                    = "queue",
 159         [BPF_MAP_TYPE_STACK]                    = "stack",
 160         [BPF_MAP_TYPE_SK_STORAGE]               = "sk_storage",
 161         [BPF_MAP_TYPE_STRUCT_OPS]               = "struct_ops",
 162         [BPF_MAP_TYPE_RINGBUF]                  = "ringbuf",
 163         [BPF_MAP_TYPE_INODE_STORAGE]            = "inode_storage",
 164         [BPF_MAP_TYPE_TASK_STORAGE]             = "task_storage",
 165         [BPF_MAP_TYPE_BLOOM_FILTER]             = "bloom_filter",
 166         [BPF_MAP_TYPE_USER_RINGBUF]             = "user_ringbuf",
 167 };
 168
 169 static const char * const prog_type_name[] = {
 170         [BPF_PROG_TYPE_UNSPEC]                  = "unspec",
 171         [BPF_PROG_TYPE_SOCKET_FILTER]           = "socket_filter",
 172         [BPF_PROG_TYPE_KPROBE]                  = "kprobe",
 173         [BPF_PROG_TYPE_SCHED_CLS]               = "sched_cls",
 174         [BPF_PROG_TYPE_SCHED_ACT]               = "sched_act",
 175         [BPF_PROG_TYPE_TRACEPOINT]              = "tracepoint",
 176         [BPF_PROG_TYPE_XDP]                     = "xdp",
 177         [BPF_PROG_TYPE_PERF_EVENT]              = "perf_event",
 178         [BPF_PROG_TYPE_CGROUP_SKB]              = "cgroup_skb",
 179         [BPF_PROG_TYPE_CGROUP_SOCK]             = "cgroup_sock",
 180         [BPF_PROG_TYPE_LWT_IN]                  = "lwt_in",
 181         [BPF_PROG_TYPE_LWT_OUT]                 = "lwt_out",
 182         [BPF_PROG_TYPE_LWT_XMIT]                = "lwt_xmit",
 183         [BPF_PROG_TYPE_SOCK_OPS]                = "sock_ops",
 184         [BPF_PROG_TYPE_SK_SKB]                  = "sk_skb",
 185         [BPF_PROG_TYPE_CGROUP_DEVICE]           = "cgroup_device",
 186         [BPF_PROG_TYPE_SK_MSG]                  = "sk_msg",
 187         [BPF_PROG_TYPE_RAW_TRACEPOINT]          = "raw_tracepoint",
 188         [BPF_PROG_TYPE_CGROUP_SOCK_ADDR]        = "cgroup_sock_addr",
 189         [BPF_PROG_TYPE_LWT_SEG6LOCAL]           = "lwt_seg6local",
 190         [BPF_PROG_TYPE_LIRC_MODE2]              = "lirc_mode2",
 191         [BPF_PROG_TYPE_SK_REUSEPORT]            = "sk_reuseport",
 192         [BPF_PROG_TYPE_FLOW_DISSECTOR]          = "flow_dissector",
 193         [BPF_PROG_TYPE_CGROUP_SYSCTL]           = "cgroup_sysctl",
 194         [BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE] = "raw_tracepoint_writable",
 195         [BPF_PROG_TYPE_CGROUP_SOCKOPT]          = "cgroup_sockopt",
 196         [BPF_PROG_TYPE_TRACING]                 = "tracing",
 197         [BPF_PROG_TYPE_STRUCT_OPS]              = "struct_ops",
 198         [BPF_PROG_TYPE_EXT]                     = "ext",
 199         [BPF_PROG_TYPE_LSM]                     = "lsm",
 200         [BPF_PROG_TYPE_SK_LOOKUP]               = "sk_lookup",
 201         [BPF_PROG_TYPE_SYSCALL]                 = "syscall",
 202 };
 203
 204 static int __base_pr(enum libbpf_print_level level, const char *format,
 205                      va_list args)
 206 {
 207         if (level == LIBBPF_DEBUG)
 208                 return 0;
 209
 210         return vfprintf(stderr, format, args);
 211 }
 212
 213 static libbpf_print_fn_t __libbpf_pr = __base_pr;
 214
 215 libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn)
 216 {
 217         libbpf_print_fn_t old_print_fn = __libbpf_pr;
 218
 219         __libbpf_pr = fn;
 220         return old_print_fn;
 221 }
 222
 223 __printf(2, 3)
 224 void libbpf_print(enum libbpf_print_level level, const char *format, ...)
 225 {
 226         va_list args;
 227         int old_errno;
 228
 229         if (!__libbpf_pr)
 230                 return;
 231
 232         old_errno = errno;
 233
 234         va_start(args, format);
 235         __libbpf_pr(level, format, args);
 236         va_end(args);
 237
 238         errno = old_errno;
 239 }
 240
 241 static void pr_perm_msg(int err)
 242 {
 243         struct rlimit limit;
 244         char buf[100];
 245
 246         if (err != -EPERM || geteuid() != 0)
 247                 return;
 248
 249         err = getrlimit(RLIMIT_MEMLOCK, &limit);
 250         if (err)
 251                 return;
 252
 253         if (limit.rlim_cur == RLIM_INFINITY)
 254                 return;
 255
 256         if (limit.rlim_cur < 1024)
 257                 snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur);
 258         else if (limit.rlim_cur < 1024*1024)
 259                 snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024);
 260         else
 261                 snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024));
 262
 263         pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n",
 264                 buf);
 265 }
 266
 267 #define STRERR_BUFSIZE  128
 268
 269 /* Copied from tools/perf/util/util.h */
 270 #ifndef zfree
 271 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
 272 #endif
 273
 274 #ifndef zclose
 275 # define zclose(fd) ({                  \
 276         int ___err = 0;                 \
 277         if ((fd) >= 0)                  \
 278                 ___err = close((fd));   \
 279         fd = -1;                        \
 280         ___err; })
 281 #endif
 282
 283 static inline __u64 ptr_to_u64(const void *ptr)
 284 {
 285         return (__u64) (unsigned long) ptr;
 286 }
 287
 288 int libbpf_set_strict_mode(enum libbpf_strict_mode mode)
 289 {
 290         /* as of v1.0 libbpf_set_strict_mode() is a no-op */
 291         return 0;
 292 }
 293
 294 __u32 libbpf_major_version(void)
 295 {
 296         return LIBBPF_MAJOR_VERSION;
 297 }
 298
 299 __u32 libbpf_minor_version(void)
 300 {
 301         return LIBBPF_MINOR_VERSION;
 302 }
 303
 304 const char *libbpf_version_string(void)
 305 {
 306 #define __S(X) #X
 307 #define _S(X) __S(X)
 308         return  "v" _S(LIBBPF_MAJOR_VERSION) "." _S(LIBBPF_MINOR_VERSION);
 309 #undef _S
 310 #undef __S
 311 }
 312
 313 enum reloc_type {
 314         RELO_LD64,
 315         RELO_CALL,
 316         RELO_DATA,
 317         RELO_EXTERN_VAR,
 318         RELO_EXTERN_FUNC,
 319         RELO_SUBPROG_ADDR,
 320         RELO_CORE,
 321 };
 322
 323 struct reloc_desc {
 324         enum reloc_type type;
 325         int insn_idx;
 326         union {
 327                 const struct bpf_core_relo *core_relo; /* used when type == RELO_CORE */
 328                 struct {
 329                         int map_idx;
 330                         int sym_off;
 331                 };
 332         };
 333 };
 334
 335 /* stored as sec_def->cookie for all libbpf-supported SEC()s */
 336 enum sec_def_flags {
 337         SEC_NONE = 0,
 338         /* expected_attach_type is optional, if kernel doesn't support that */
 339         SEC_EXP_ATTACH_OPT = 1,
 340         /* legacy, only used by libbpf_get_type_names() and
 341          * libbpf_attach_type_by_name(), not used by libbpf itself at all.
 342          * This used to be associated with cgroup (and few other) BPF programs
 343          * that were attachable through BPF_PROG_ATTACH command. Pretty
 344          * meaningless nowadays, though.
 345          */
 346         SEC_ATTACHABLE = 2,
 347         SEC_ATTACHABLE_OPT = SEC_ATTACHABLE | SEC_EXP_ATTACH_OPT,
 348         /* attachment target is specified through BTF ID in either kernel or
 349          * other BPF program's BTF object */
 350         SEC_ATTACH_BTF = 4,
 351         /* BPF program type allows sleeping/blocking in kernel */
 352         SEC_SLEEPABLE = 8,
 353         /* BPF program support non-linear XDP buffer */
 354         SEC_XDP_FRAGS = 16,
 355 };
 356
 357 struct bpf_sec_def {
 358         char *sec;
 359         enum bpf_prog_type prog_type;
 360         enum bpf_attach_type expected_attach_type;
 361         long cookie;
 362         int handler_id;
 363
 364         libbpf_prog_setup_fn_t prog_setup_fn;
 365         libbpf_prog_prepare_load_fn_t prog_prepare_load_fn;
 366         libbpf_prog_attach_fn_t prog_attach_fn;
 367 };
 368
 369 /*
 370  * bpf_prog should be a better name but it has been used in
 371  * linux/filter.h.
 372  */
 373 struct bpf_program {
 374         char *name;
 375         char *sec_name;
 376         size_t sec_idx;
 377         const struct bpf_sec_def *sec_def;
 378         /* this program's instruction offset (in number of instructions)
 379          * within its containing ELF section
 380          */
 381         size_t sec_insn_off;
 382         /* number of original instructions in ELF section belonging to this
 383          * program, not taking into account subprogram instructions possible
 384          * appended later during relocation
 385          */
 386         size_t sec_insn_cnt;
 387         /* Offset (in number of instructions) of the start of instruction
 388          * belonging to this BPF program  within its containing main BPF
 389          * program. For the entry-point (main) BPF program, this is always
 390          * zero. For a sub-program, this gets reset before each of main BPF
 391          * programs are processed and relocated and is used to determined
 392          * whether sub-program was already appended to the main program, and
 393          * if yes, at which instruction offset.
 394          */
 395         size_t sub_insn_off;
 396
 397         /* instructions that belong to BPF program; insns[0] is located at
 398          * sec_insn_off instruction within its ELF section in ELF file, so
 399          * when mapping ELF file instruction index to the local instruction,
 400          * one needs to subtract sec_insn_off; and vice versa.
 401          */
 402         struct bpf_insn *insns;
 403         /* actual number of instruction in this BPF program's image; for
 404          * entry-point BPF programs this includes the size of main program
 405          * itself plus all the used sub-programs, appended at the end
 406          */
 407         size_t insns_cnt;
 408
 409         struct reloc_desc *reloc_desc;
 410         int nr_reloc;
 411
 412         /* BPF verifier log settings */
 413         char *log_buf;
 414         size_t log_size;
 415         __u32 log_level;
 416
 417         struct bpf_object *obj;
 418
 419         int fd;
 420         bool autoload;
 421         bool autoattach;
 422         bool mark_btf_static;
 423         enum bpf_prog_type type;
 424         enum bpf_attach_type expected_attach_type;
 425
 426         int prog_ifindex;
 427         __u32 attach_btf_obj_fd;
 428         __u32 attach_btf_id;
 429         __u32 attach_prog_fd;
 430
 431         void *func_info;
 432         __u32 func_info_rec_size;
 433         __u32 func_info_cnt;
 434
 435         void *line_info;
 436         __u32 line_info_rec_size;
 437         __u32 line_info_cnt;
 438         __u32 prog_flags;
 439 };
 440
 441 struct bpf_struct_ops {
 442         const char *tname;
 443         const struct btf_type *type;
 444         struct bpf_program **progs;
 445         __u32 *kern_func_off;
 446         /* e.g. struct tcp_congestion_ops in bpf_prog's btf format */
 447         void *data;
 448         /* e.g. struct bpf_struct_ops_tcp_congestion_ops in
 449          *      btf_vmlinux's format.
 450          * struct bpf_struct_ops_tcp_congestion_ops {
 451          *      [... some other kernel fields ...]
 452          *      struct tcp_congestion_ops data;
 453          * }
 454          * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)
 455          * bpf_map__init_kern_struct_ops() will populate the "kern_vdata"
 456          * from "data".
 457          */
 458         void *kern_vdata;
 459         __u32 type_id;
 460 };
 461
 462 #define DATA_SEC ".data"
 463 #define BSS_SEC ".bss"
 464 #define RODATA_SEC ".rodata"
 465 #define KCONFIG_SEC ".kconfig"
 466 #define KSYMS_SEC ".ksyms"
 467 #define STRUCT_OPS_SEC ".struct_ops"
 468
 469 enum libbpf_map_type {
 470         LIBBPF_MAP_UNSPEC,
 471         LIBBPF_MAP_DATA,
 472         LIBBPF_MAP_BSS,
 473         LIBBPF_MAP_RODATA,
 474         LIBBPF_MAP_KCONFIG,
 475 };
 476
 477 struct bpf_map_def {
 478         unsigned int type;
 479         unsigned int key_size;
 480         unsigned int value_size;
 481         unsigned int max_entries;
 482         unsigned int map_flags;
 483 };
 484
 485 struct bpf_map {
 486         struct bpf_object *obj;
 487         char *name;
 488         /* real_name is defined for special internal maps (.rodata*,
 489          * .data*, .bss, .kconfig) and preserves their original ELF section
 490          * name. This is important to be be able to find corresponding BTF
 491          * DATASEC information.
 492          */
 493         char *real_name;
 494         int fd;
 495         int sec_idx;
 496         size_t sec_offset;
 497         int map_ifindex;
 498         int inner_map_fd;
 499         struct bpf_map_def def;
 500         __u32 numa_node;
 501         __u32 btf_var_idx;
 502         __u32 btf_key_type_id;
 503         __u32 btf_value_type_id;
 504         __u32 btf_vmlinux_value_type_id;
 505         enum libbpf_map_type libbpf_type;
 506         void *mmaped;
 507         struct bpf_struct_ops *st_ops;
 508         struct bpf_map *inner_map;
 509         void **init_slots;
 510         int init_slots_sz;
 511         char *pin_path;
 512         bool pinned;
 513         bool reused;
 514         bool autocreate;
 515         __u64 map_extra;
 516 };
 517
 518 enum extern_type {
 519         EXT_UNKNOWN,
 520         EXT_KCFG,
 521         EXT_KSYM,
 522 };
 523
 524 enum kcfg_type {
 525         KCFG_UNKNOWN,
 526         KCFG_CHAR,
 527         KCFG_BOOL,
 528         KCFG_INT,
 529         KCFG_TRISTATE,
 530         KCFG_CHAR_ARR,
 531 };
 532
 533 struct extern_desc {
 534         enum extern_type type;
 535         int sym_idx;
 536         int btf_id;
 537         int sec_btf_id;
 538         const char *name;
 539         bool is_set;
 540         bool is_weak;
 541         union {
 542                 struct {
 543                         enum kcfg_type type;
 544                         int sz;
 545                         int align;
 546                         int data_off;
 547                         bool is_signed;
 548                 } kcfg;
 549                 struct {
 550                         unsigned long long addr;
 551
 552                         /* target btf_id of the corresponding kernel var. */
 553                         int kernel_btf_obj_fd;
 554                         int kernel_btf_id;
 555
 556                         /* local btf_id of the ksym extern's type. */
 557                         __u32 type_id;
 558                         /* BTF fd index to be patched in for insn->off, this is
 559                          * 0 for vmlinux BTF, index in obj->fd_array for module
 560                          * BTF
 561                          */
 562                         __s16 btf_fd_idx;
 563                 } ksym;
 564         };
 565 };
 566
 567 struct module_btf {
 568         struct btf *btf;
 569         char *name;
 570         __u32 id;
 571         int fd;
 572         int fd_array_idx;
 573 };
 574
 575 enum sec_type {
 576         SEC_UNUSED = 0,
 577         SEC_RELO,
 578         SEC_BSS,
 579         SEC_DATA,
 580         SEC_RODATA,
 581 };
 582
 583 struct elf_sec_desc {
 584         enum sec_type sec_type;
 585         Elf64_Shdr *shdr;
 586         Elf_Data *data;
 587 };
 588
 589 struct elf_state {
 590         int fd;
 591         const void *obj_buf;
 592         size_t obj_buf_sz;
 593         Elf *elf;
 594         Elf64_Ehdr *ehdr;
 595         Elf_Data *symbols;
 596         Elf_Data *st_ops_data;
 597         size_t shstrndx; /* section index for section name strings */
 598         size_t strtabidx;
 599         struct elf_sec_desc *secs;
 600         int sec_cnt;
 601         int btf_maps_shndx;
 602         __u32 btf_maps_sec_btf_id;
 603         int text_shndx;
 604         int symbols_shndx;
 605         int st_ops_shndx;
 606 };
 607
 608 struct usdt_manager;
 609
 610 struct bpf_object {
 611         char name[BPF_OBJ_NAME_LEN];
 612         char license[64];
 613         __u32 kern_version;
 614
 615         struct bpf_program *programs;
 616         size_t nr_programs;
 617         struct bpf_map *maps;
 618         size_t nr_maps;
 619         size_t maps_cap;
 620
 621         char *kconfig;
 622         struct extern_desc *externs;
 623         int nr_extern;
 624         int kconfig_map_idx;
 625
 626         bool loaded;
 627         bool has_subcalls;
 628         bool has_rodata;
 629
 630         struct bpf_gen *gen_loader;
 631
 632         /* Information when doing ELF related work. Only valid if efile.elf is not NULL */
 633         struct elf_state efile;
 634
 635         struct btf *btf;
 636         struct btf_ext *btf_ext;
 637
 638         /* Parse and load BTF vmlinux if any of the programs in the object need
 639          * it at load time.
 640          */
 641         struct btf *btf_vmlinux;
 642         /* Path to the custom BTF to be used for BPF CO-RE relocations as an
 643          * override for vmlinux BTF.
 644          */
 645         char *btf_custom_path;
 646         /* vmlinux BTF override for CO-RE relocations */
 647         struct btf *btf_vmlinux_override;
 648         /* Lazily initialized kernel module BTFs */
 649         struct module_btf *btf_modules;
 650         bool btf_modules_loaded;
 651         size_t btf_module_cnt;
 652         size_t btf_module_cap;
 653
 654         /* optional log settings passed to BPF_BTF_LOAD and BPF_PROG_LOAD commands */
 655         char *log_buf;
 656         size_t log_size;
 657         __u32 log_level;
 658
 659         int *fd_array;
 660         size_t fd_array_cap;
 661         size_t fd_array_cnt;
 662
 663         struct usdt_manager *usdt_man;
 664
 665         char path[];
 666 };
 667
 668 static const char *elf_sym_str(const struct bpf_object *obj, size_t off);
 669 static const char *elf_sec_str(const struct bpf_object *obj, size_t off);
 670 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx);
 671 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name);
 672 static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn);
 673 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn);
 674 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn);
 675 static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx);
 676 static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx);
 677
 678 void bpf_program__unload(struct bpf_program *prog)
 679 {
 680         if (!prog)
 681                 return;
 682
 683         zclose(prog->fd);
 684
 685         zfree(&prog->func_info);
 686         zfree(&prog->line_info);
 687 }
 688
 689 static void bpf_program__exit(struct bpf_program *prog)
 690 {
 691         if (!prog)
 692                 return;
 693
 694         bpf_program__unload(prog);
 695         zfree(&prog->name);
 696         zfree(&prog->sec_name);
 697         zfree(&prog->insns);
 698         zfree(&prog->reloc_desc);
 699
 700         prog->nr_reloc = 0;
 701         prog->insns_cnt = 0;
 702         prog->sec_idx = -1;
 703 }
 704
 705 static bool insn_is_subprog_call(const struct bpf_insn *insn)
 706 {
 707         return BPF_CLASS(insn->code) == BPF_JMP &&
 708                BPF_OP(insn->code) == BPF_CALL &&
 709                BPF_SRC(insn->code) == BPF_K &&
 710                insn->src_reg == BPF_PSEUDO_CALL &&
 711                insn->dst_reg == 0 &&
 712                insn->off == 0;
 713 }
 714
 715 static bool is_call_insn(const struct bpf_insn *insn)
 716 {
 717         return insn->code == (BPF_JMP | BPF_CALL);
 718 }
 719
 720 static bool insn_is_pseudo_func(struct bpf_insn *insn)
 721 {
 722         return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
 723 }
 724
 725 static int
 726 bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog,
 727                       const char *name, size_t sec_idx, const char *sec_name,
 728                       size_t sec_off, void *insn_data, size_t insn_data_sz)
 729 {
 730         if (insn_data_sz == 0 || insn_data_sz % BPF_INSN_SZ || sec_off % BPF_INSN_SZ) {
 731                 pr_warn("sec '%s': corrupted program '%s', offset %zu, size %zu\n",
 732                         sec_name, name, sec_off, insn_data_sz);
 733                 return -EINVAL;
 734         }
 735
 736         memset(prog, 0, sizeof(*prog));
 737         prog->obj = obj;
 738
 739         prog->sec_idx = sec_idx;
 740         prog->sec_insn_off = sec_off / BPF_INSN_SZ;
 741         prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ;
 742         /* insns_cnt can later be increased by appending used subprograms */
 743         prog->insns_cnt = prog->sec_insn_cnt;
 744
 745         prog->type = BPF_PROG_TYPE_UNSPEC;
 746         prog->fd = -1;
 747
 748         /* libbpf's convention for SEC("?abc...") is that it's just like
 749          * SEC("abc...") but the corresponding bpf_program starts out with
 750          * autoload set to false.
 751          */
 752         if (sec_name[0] == '?') {
 753                 prog->autoload = false;
 754                 /* from now on forget there was ? in section name */
 755                 sec_name++;
 756         } else {
 757                 prog->autoload = true;
 758         }
 759
 760         prog->autoattach = true;
 761
 762         /* inherit object's log_level */
 763         prog->log_level = obj->log_level;
 764
 765         prog->sec_name = strdup(sec_name);
 766         if (!prog->sec_name)
 767                 goto errout;
 768
 769         prog->name = strdup(name);
 770         if (!prog->name)
 771                 goto errout;
 772
 773         prog->insns = malloc(insn_data_sz);
 774         if (!prog->insns)
 775                 goto errout;
 776         memcpy(prog->insns, insn_data, insn_data_sz);
 777
 778         return 0;
 779 errout:
 780         pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name);
 781         bpf_program__exit(prog);
 782         return -ENOMEM;
 783 }
 784
 785 static int
 786 bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data,
 787                          const char *sec_name, int sec_idx)
 788 {
 789         Elf_Data *symbols = obj->efile.symbols;
 790         struct bpf_program *prog, *progs;
 791         void *data = sec_data->d_buf;
 792         size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms;
 793         int nr_progs, err, i;
 794         const char *name;
 795         Elf64_Sym *sym;
 796
 797         progs = obj->programs;
 798         nr_progs = obj->nr_programs;
 799         nr_syms = symbols->d_size / sizeof(Elf64_Sym);
 800         sec_off = 0;
 801
 802         for (i = 0; i < nr_syms; i++) {
 803                 sym = elf_sym_by_idx(obj, i);
 804
 805                 if (sym->st_shndx != sec_idx)
 806                         continue;
 807                 if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC)
 808                         continue;
 809
 810                 prog_sz = sym->st_size;
 811                 sec_off = sym->st_value;
 812
 813                 name = elf_sym_str(obj, sym->st_name);
 814                 if (!name) {
 815                         pr_warn("sec '%s': failed to get symbol name for offset %zu\n",
 816                                 sec_name, sec_off);
 817                         return -LIBBPF_ERRNO__FORMAT;
 818                 }
 819
 820                 if (sec_off + prog_sz > sec_sz) {
 821                         pr_warn("sec '%s': program at offset %zu crosses section boundary\n",
 822                                 sec_name, sec_off);
 823                         return -LIBBPF_ERRNO__FORMAT;
 824                 }
 825
 826                 if (sec_idx != obj->efile.text_shndx && ELF64_ST_BIND(sym->st_info) == STB_LOCAL) {
 827                         pr_warn("sec '%s': program '%s' is static and not supported\n", sec_name, name);
 828                         return -ENOTSUP;
 829                 }
 830
 831                 pr_debug("sec '%s': found program '%s' at insn offset %zu (%zu bytes), code size %zu insns (%zu bytes)\n",
 832                          sec_name, name, sec_off / BPF_INSN_SZ, sec_off, prog_sz / BPF_INSN_SZ, prog_sz);
 833
 834                 progs = libbpf_reallocarray(progs, nr_progs + 1, sizeof(*progs));
 835                 if (!progs) {
 836                         /*
 837                          * In this case the original obj->programs
 838                          * is still valid, so don't need special treat for
 839                          * bpf_close_object().
 840                          */
 841                         pr_warn("sec '%s': failed to alloc memory for new program '%s'\n",
 842                                 sec_name, name);
 843                         return -ENOMEM;
 844                 }
 845                 obj->programs = progs;
 846
 847                 prog = &progs[nr_progs];
 848
 849                 err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name,
 850                                             sec_off, data + sec_off, prog_sz);
 851                 if (err)
 852                         return err;
 853
 854                 /* if function is a global/weak symbol, but has restricted
 855                  * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF FUNC
 856                  * as static to enable more permissive BPF verification mode
 857                  * with more outside context available to BPF verifier
 858                  */
 859                 if (ELF64_ST_BIND(sym->st_info) != STB_LOCAL
 860                     && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN
 861                         || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL))
 862                         prog->mark_btf_static = true;
 863
 864                 nr_progs++;
 865                 obj->nr_programs = nr_progs;
 866         }
 867
 868         return 0;
 869 }
 870
 871 __u32 get_kernel_version(void)
 872 {
 873         /* On Ubuntu LINUX_VERSION_CODE doesn't correspond to info.release,
 874          * but Ubuntu provides /proc/version_signature file, as described at
 875          * https://ubuntu.com/kernel, with an example contents below, which we
 876          * can use to get a proper LINUX_VERSION_CODE.
 877          *
 878          *   Ubuntu 5.4.0-12.15-generic 5.4.8
 879          *
 880          * In the above, 5.4.8 is what kernel is actually expecting, while
 881          * uname() call will return 5.4.0 in info.release.
 882          */
 883         const char *ubuntu_kver_file = "/proc/version_signature";
 884         __u32 major, minor, patch;
 885         struct utsname info;
 886
 887         if (faccessat(AT_FDCWD, ubuntu_kver_file, R_OK, AT_EACCESS) == 0) {
 888                 FILE *f;
 889
 890                 f = fopen(ubuntu_kver_file, "r");
 891                 if (f) {
 892                         if (fscanf(f, "%*s %*s %d.%d.%d\n", &major, &minor, &patch) == 3) {
 893                                 fclose(f);
 894                                 return KERNEL_VERSION(major, minor, patch);
 895                         }
 896                         fclose(f);
 897                 }
 898                 /* something went wrong, fall back to uname() approach */
 899         }
 900
 901         uname(&info);
 902         if (sscanf(info.release, "%u.%u.%u", &major, &minor, &patch) != 3)
 903                 return 0;
 904         return KERNEL_VERSION(major, minor, patch);
 905 }
 906
 907 static const struct btf_member *
 908 find_member_by_offset(const struct btf_type *t, __u32 bit_offset)
 909 {
 910         struct btf_member *m;
 911         int i;
 912
 913         for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
 914                 if (btf_member_bit_offset(t, i) == bit_offset)
 915                         return m;
 916         }
 917
 918         return NULL;
 919 }
 920
 921 static const struct btf_member *
 922 find_member_by_name(const struct btf *btf, const struct btf_type *t,
 923                     const char *name)
 924 {
 925         struct btf_member *m;
 926         int i;
 927
 928         for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
 929                 if (!strcmp(btf__name_by_offset(btf, m->name_off), name))
 930                         return m;
 931         }
 932
 933         return NULL;
 934 }
 935
 936 #define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_"
 937 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
 938                                    const char *name, __u32 kind);
 939
 940 static int
 941 find_struct_ops_kern_types(const struct btf *btf, const char *tname,
 942                            const struct btf_type **type, __u32 *type_id,
 943                            const struct btf_type **vtype, __u32 *vtype_id,
 944                            const struct btf_member **data_member)
 945 {
 946         const struct btf_type *kern_type, *kern_vtype;
 947         const struct btf_member *kern_data_member;
 948         __s32 kern_vtype_id, kern_type_id;
 949         __u32 i;
 950
 951         kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT);
 952         if (kern_type_id < 0) {
 953                 pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n",
 954                         tname);
 955                 return kern_type_id;
 956         }
 957         kern_type = btf__type_by_id(btf, kern_type_id);
 958
 959         /* Find the corresponding "map_value" type that will be used
 960          * in map_update(BPF_MAP_TYPE_STRUCT_OPS).  For example,
 961          * find "struct bpf_struct_ops_tcp_congestion_ops" from the
 962          * btf_vmlinux.
 963          */
 964         kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX,
 965                                                 tname, BTF_KIND_STRUCT);
 966         if (kern_vtype_id < 0) {
 967                 pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n",
 968                         STRUCT_OPS_VALUE_PREFIX, tname);
 969                 return kern_vtype_id;
 970         }
 971         kern_vtype = btf__type_by_id(btf, kern_vtype_id);
 972
 973         /* Find "struct tcp_congestion_ops" from
 974          * struct bpf_struct_ops_tcp_congestion_ops {
 975          *      [ ... ]
 976          *      struct tcp_congestion_ops data;
 977          * }
 978          */
 979         kern_data_member = btf_members(kern_vtype);
 980         for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) {
 981                 if (kern_data_member->type == kern_type_id)
 982                         break;
 983         }
 984         if (i == btf_vlen(kern_vtype)) {
 985                 pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n",
 986                         tname, STRUCT_OPS_VALUE_PREFIX, tname);
 987                 return -EINVAL;
 988         }
 989
 990         *type = kern_type;
 991         *type_id = kern_type_id;
 992         *vtype = kern_vtype;
 993         *vtype_id = kern_vtype_id;
 994         *data_member = kern_data_member;
 995
 996         return 0;
 997 }
 998
 999 static bool bpf_map__is_struct_ops(const struct bpf_map *map)
1000 {
1001         return map->def.type == BPF_MAP_TYPE_STRUCT_OPS;
1002 }
1003
1004 /* Init the map's fields that depend on kern_btf */
1005 static int bpf_map__init_kern_struct_ops(struct bpf_map *map,
1006                                          const struct btf *btf,
1007                                          const struct btf *kern_btf)
1008 {
1009         const struct btf_member *member, *kern_member, *kern_data_member;
1010         const struct btf_type *type, *kern_type, *kern_vtype;
1011         __u32 i, kern_type_id, kern_vtype_id, kern_data_off;
1012         struct bpf_struct_ops *st_ops;
1013         void *data, *kern_data;
1014         const char *tname;
1015         int err;
1016
1017         st_ops = map->st_ops;
1018         type = st_ops->type;
1019         tname = st_ops->tname;
1020         err = find_struct_ops_kern_types(kern_btf, tname,
1021                                          &kern_type, &kern_type_id,
1022                                          &kern_vtype, &kern_vtype_id,
1023                                          &kern_data_member);
1024         if (err)
1025                 return err;
1026
1027         pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n",
1028                  map->name, st_ops->type_id, kern_type_id, kern_vtype_id);
1029
1030         map->def.value_size = kern_vtype->size;
1031         map->btf_vmlinux_value_type_id = kern_vtype_id;
1032
1033         st_ops->kern_vdata = calloc(1, kern_vtype->size);
1034         if (!st_ops->kern_vdata)
1035                 return -ENOMEM;
1036
1037         data = st_ops->data;
1038         kern_data_off = kern_data_member->offset / 8;
1039         kern_data = st_ops->kern_vdata + kern_data_off;
1040
1041         member = btf_members(type);
1042         for (i = 0; i < btf_vlen(type); i++, member++) {
1043                 const struct btf_type *mtype, *kern_mtype;
1044                 __u32 mtype_id, kern_mtype_id;
1045                 void *mdata, *kern_mdata;
1046                 __s64 msize, kern_msize;
1047                 __u32 moff, kern_moff;
1048                 __u32 kern_member_idx;
1049                 const char *mname;
1050
1051                 mname = btf__name_by_offset(btf, member->name_off);
1052                 kern_member = find_member_by_name(kern_btf, kern_type, mname);
1053                 if (!kern_member) {
1054                         pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n",
1055                                 map->name, mname);
1056                         return -ENOTSUP;
1057                 }
1058
1059                 kern_member_idx = kern_member - btf_members(kern_type);
1060                 if (btf_member_bitfield_size(type, i) ||
1061                     btf_member_bitfield_size(kern_type, kern_member_idx)) {
1062                         pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n",
1063                                 map->name, mname);
1064                         return -ENOTSUP;
1065                 }
1066
1067                 moff = member->offset / 8;
1068                 kern_moff = kern_member->offset / 8;
1069
1070                 mdata = data + moff;
1071                 kern_mdata = kern_data + kern_moff;
1072
1073                 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);
1074                 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type,
1075                                                     &kern_mtype_id);
1076                 if (BTF_INFO_KIND(mtype->info) !=
1077                     BTF_INFO_KIND(kern_mtype->info)) {
1078                         pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n",
1079                                 map->name, mname, BTF_INFO_KIND(mtype->info),
1080                                 BTF_INFO_KIND(kern_mtype->info));
1081                         return -ENOTSUP;
1082                 }
1083
1084                 if (btf_is_ptr(mtype)) {
1085                         struct bpf_program *prog;
1086
1087                         prog = st_ops->progs[i];
1088                         if (!prog)
1089                                 continue;
1090
1091                         kern_mtype = skip_mods_and_typedefs(kern_btf,
1092                                                             kern_mtype->type,
1093                                                             &kern_mtype_id);
1094
1095                         /* mtype->type must be a func_proto which was
1096                          * guaranteed in bpf_object__collect_st_ops_relos(),
1097                          * so only check kern_mtype for func_proto here.
1098                          */
1099                         if (!btf_is_func_proto(kern_mtype)) {
1100                                 pr_warn("struct_ops init_kern %s: kernel member %s is not a func ptr\n",
1101                                         map->name, mname);
1102                                 return -ENOTSUP;
1103                         }
1104
1105                         prog->attach_btf_id = kern_type_id;
1106                         prog->expected_attach_type = kern_member_idx;
1107
1108                         st_ops->kern_func_off[i] = kern_data_off + kern_moff;
1109
1110                         pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n",
1111                                  map->name, mname, prog->name, moff,
1112                                  kern_moff);
1113
1114                         continue;
1115                 }
1116
1117                 msize = btf__resolve_size(btf, mtype_id);
1118                 kern_msize = btf__resolve_size(kern_btf, kern_mtype_id);
1119                 if (msize < 0 || kern_msize < 0 || msize != kern_msize) {
1120                         pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n",
1121                                 map->name, mname, (ssize_t)msize,
1122                                 (ssize_t)kern_msize);
1123                         return -ENOTSUP;
1124                 }
1125
1126                 pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n",
1127                          map->name, mname, (unsigned int)msize,
1128                          moff, kern_moff);
1129                 memcpy(kern_mdata, mdata, msize);
1130         }
1131
1132         return 0;
1133 }
1134
1135 static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj)
1136 {
1137         struct bpf_map *map;
1138         size_t i;
1139         int err;
1140
1141         for (i = 0; i < obj->nr_maps; i++) {
1142                 map = &obj->maps[i];
1143
1144                 if (!bpf_map__is_struct_ops(map))
1145                         continue;
1146
1147                 err = bpf_map__init_kern_struct_ops(map, obj->btf,
1148                                                     obj->btf_vmlinux);
1149                 if (err)
1150                         return err;
1151         }
1152
1153         return 0;
1154 }
1155
1156 static int bpf_object__init_struct_ops_maps(struct bpf_object *obj)
1157 {
1158         const struct btf_type *type, *datasec;
1159         const struct btf_var_secinfo *vsi;
1160         struct bpf_struct_ops *st_ops;
1161         const char *tname, *var_name;
1162         __s32 type_id, datasec_id;
1163         const struct btf *btf;
1164         struct bpf_map *map;
1165         __u32 i;
1166
1167         if (obj->efile.st_ops_shndx == -1)
1168                 return 0;
1169
1170         btf = obj->btf;
1171         datasec_id = btf__find_by_name_kind(btf, STRUCT_OPS_SEC,
1172                                             BTF_KIND_DATASEC);
1173         if (datasec_id < 0) {
1174                 pr_warn("struct_ops init: DATASEC %s not found\n",
1175                         STRUCT_OPS_SEC);
1176                 return -EINVAL;
1177         }
1178
1179         datasec = btf__type_by_id(btf, datasec_id);
1180         vsi = btf_var_secinfos(datasec);
1181         for (i = 0; i < btf_vlen(datasec); i++, vsi++) {
1182                 type = btf__type_by_id(obj->btf, vsi->type);
1183                 var_name = btf__name_by_offset(obj->btf, type->name_off);
1184
1185                 type_id = btf__resolve_type(obj->btf, vsi->type);
1186                 if (type_id < 0) {
1187                         pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n",
1188                                 vsi->type, STRUCT_OPS_SEC);
1189                         return -EINVAL;
1190                 }
1191
1192                 type = btf__type_by_id(obj->btf, type_id);
1193                 tname = btf__name_by_offset(obj->btf, type->name_off);
1194                 if (!tname[0]) {
1195                         pr_warn("struct_ops init: anonymous type is not supported\n");
1196                         return -ENOTSUP;
1197                 }
1198                 if (!btf_is_struct(type)) {
1199                         pr_warn("struct_ops init: %s is not a struct\n", tname);
1200                         return -EINVAL;
1201                 }
1202
1203                 map = bpf_object__add_map(obj);
1204                 if (IS_ERR(map))
1205                         return PTR_ERR(map);
1206
1207                 map->sec_idx = obj->efile.st_ops_shndx;
1208                 map->sec_offset = vsi->offset;
1209                 map->name = strdup(var_name);
1210                 if (!map->name)
1211                         return -ENOMEM;
1212
1213                 map->def.type = BPF_MAP_TYPE_STRUCT_OPS;
1214                 map->def.key_size = sizeof(int);
1215                 map->def.value_size = type->size;
1216                 map->def.max_entries = 1;
1217
1218                 map->st_ops = calloc(1, sizeof(*map->st_ops));
1219                 if (!map->st_ops)
1220                         return -ENOMEM;
1221                 st_ops = map->st_ops;
1222                 st_ops->data = malloc(type->size);
1223                 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs));
1224                 st_ops->kern_func_off = malloc(btf_vlen(type) *
1225                                                sizeof(*st_ops->kern_func_off));
1226                 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off)
1227                         return -ENOMEM;
1228
1229                 if (vsi->offset + type->size > obj->efile.st_ops_data->d_size) {
1230                         pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n",
1231                                 var_name, STRUCT_OPS_SEC);
1232                         return -EINVAL;
1233                 }
1234
1235                 memcpy(st_ops->data,
1236                        obj->efile.st_ops_data->d_buf + vsi->offset,
1237                        type->size);
1238                 st_ops->tname = tname;
1239                 st_ops->type = type;
1240                 st_ops->type_id = type_id;
1241
1242                 pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n",
1243                          tname, type_id, var_name, vsi->offset);
1244         }
1245
1246         return 0;
1247 }
1248
1249 static struct bpf_object *bpf_object__new(const char *path,
1250                                           const void *obj_buf,
1251                                           size_t obj_buf_sz,
1252                                           const char *obj_name)
1253 {
1254         struct bpf_object *obj;
1255         char *end;
1256
1257         obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
1258         if (!obj) {
1259                 pr_warn("alloc memory failed for %s\n", path);
1260                 return ERR_PTR(-ENOMEM);
1261         }
1262
1263         strcpy(obj->path, path);
1264         if (obj_name) {
1265                 libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name));
1266         } else {
1267                 /* Using basename() GNU version which doesn't modify arg. */
1268                 libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name));
1269                 end = strchr(obj->name, '.');
1270                 if (end)
1271                         *end = 0;
1272         }
1273
1274         obj->efile.fd = -1;
1275         /*
1276          * Caller of this function should also call
1277          * bpf_object__elf_finish() after data collection to return
1278          * obj_buf to user. If not, we should duplicate the buffer to
1279          * avoid user freeing them before elf finish.
1280          */
1281         obj->efile.obj_buf = obj_buf;
1282         obj->efile.obj_buf_sz = obj_buf_sz;
1283         obj->efile.btf_maps_shndx = -1;
1284         obj->efile.st_ops_shndx = -1;
1285         obj->kconfig_map_idx = -1;
1286
1287         obj->kern_version = get_kernel_version();
1288         obj->loaded = false;
1289
1290         return obj;
1291 }
1292
1293 static void bpf_object__elf_finish(struct bpf_object *obj)
1294 {
1295         if (!obj->efile.elf)
1296                 return;
1297
1298         elf_end(obj->efile.elf);
1299         obj->efile.elf = NULL;
1300         obj->efile.symbols = NULL;
1301         obj->efile.st_ops_data = NULL;
1302
1303         zfree(&obj->efile.secs);
1304         obj->efile.sec_cnt = 0;
1305         zclose(obj->efile.fd);
1306         obj->efile.obj_buf = NULL;
1307         obj->efile.obj_buf_sz = 0;
1308 }
1309
1310 static int bpf_object__elf_init(struct bpf_object *obj)
1311 {
1312         Elf64_Ehdr *ehdr;
1313         int err = 0;
1314         Elf *elf;
1315
1316         if (obj->efile.elf) {
1317                 pr_warn("elf: init internal error\n");
1318                 return -LIBBPF_ERRNO__LIBELF;
1319         }
1320
1321         if (obj->efile.obj_buf_sz > 0) {
1322                 /* obj_buf should have been validated by bpf_object__open_mem(). */
1323                 elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz);
1324         } else {
1325                 obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC);
1326                 if (obj->efile.fd < 0) {
1327                         char errmsg[STRERR_BUFSIZE], *cp;
1328
1329                         err = -errno;
1330                         cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
1331                         pr_warn("elf: failed to open %s: %s\n", obj->path, cp);
1332                         return err;
1333                 }
1334
1335                 elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL);
1336         }
1337
1338         if (!elf) {
1339                 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1));
1340                 err = -LIBBPF_ERRNO__LIBELF;
1341                 goto errout;
1342         }
1343
1344         obj->efile.elf = elf;
1345
1346         if (elf_kind(elf) != ELF_K_ELF) {
1347                 err = -LIBBPF_ERRNO__FORMAT;
1348                 pr_warn("elf: '%s' is not a proper ELF object\n", obj->path);
1349                 goto errout;
1350         }
1351
1352         if (gelf_getclass(elf) != ELFCLASS64) {
1353                 err = -LIBBPF_ERRNO__FORMAT;
1354                 pr_warn("elf: '%s' is not a 64-bit ELF object\n", obj->path);
1355                 goto errout;
1356         }
1357
1358         obj->efile.ehdr = ehdr = elf64_getehdr(elf);
1359         if (!obj->efile.ehdr) {
1360                 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1));
1361                 err = -LIBBPF_ERRNO__FORMAT;
1362                 goto errout;
1363         }
1364
1365         if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) {
1366                 pr_warn("elf: failed to get section names section index for %s: %s\n",
1367                         obj->path, elf_errmsg(-1));
1368                 err = -LIBBPF_ERRNO__FORMAT;
1369                 goto errout;
1370         }
1371
1372         /* Elf is corrupted/truncated, avoid calling elf_strptr. */
1373         if (!elf_rawdata(elf_getscn(elf, obj->efile.shstrndx), NULL)) {
1374                 pr_warn("elf: failed to get section names strings from %s: %s\n",
1375                         obj->path, elf_errmsg(-1));
1376                 err = -LIBBPF_ERRNO__FORMAT;
1377                 goto errout;
1378         }
1379
1380         /* Old LLVM set e_machine to EM_NONE */
1381         if (ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF)) {
1382                 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path);
1383                 err = -LIBBPF_ERRNO__FORMAT;
1384                 goto errout;
1385         }
1386
1387         return 0;
1388 errout:
1389         bpf_object__elf_finish(obj);
1390         return err;
1391 }
1392
1393 static int bpf_object__check_endianness(struct bpf_object *obj)
1394 {
1395 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1396         if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
1397                 return 0;
1398 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
1399         if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
1400                 return 0;
1401 #else
1402 # error "Unrecognized __BYTE_ORDER__"
1403 #endif
1404         pr_warn("elf: endianness mismatch in %s.\n", obj->path);
1405         return -LIBBPF_ERRNO__ENDIAN;
1406 }
1407
1408 static int
1409 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size)
1410 {
1411         /* libbpf_strlcpy() only copies first N - 1 bytes, so size + 1 won't
1412          * go over allowed ELF data section buffer
1413          */
1414         libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license)));
1415         pr_debug("license of %s is %s\n", obj->path, obj->license);
1416         return 0;
1417 }
1418
1419 static int
1420 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size)
1421 {
1422         __u32 kver;
1423
1424         if (size != sizeof(kver)) {
1425                 pr_warn("invalid kver section in %s\n", obj->path);
1426                 return -LIBBPF_ERRNO__FORMAT;
1427         }
1428         memcpy(&kver, data, sizeof(kver));
1429         obj->kern_version = kver;
1430         pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
1431         return 0;
1432 }
1433
1434 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type)
1435 {
1436         if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
1437             type == BPF_MAP_TYPE_HASH_OF_MAPS)
1438                 return true;
1439         return false;
1440 }
1441
1442 static int find_elf_sec_sz(const struct bpf_object *obj, const char *name, __u32 *size)
1443 {
1444         Elf_Data *data;
1445         Elf_Scn *scn;
1446
1447         if (!name)
1448                 return -EINVAL;
1449
1450         scn = elf_sec_by_name(obj, name);
1451         data = elf_sec_data(obj, scn);
1452         if (data) {
1453                 *size = data->d_size;
1454                 return 0; /* found it */
1455         }
1456
1457         return -ENOENT;
1458 }
1459
1460 static int find_elf_var_offset(const struct bpf_object *obj, const char *name, __u32 *off)
1461 {
1462         Elf_Data *symbols = obj->efile.symbols;
1463         const char *sname;
1464         size_t si;
1465
1466         if (!name || !off)
1467                 return -EINVAL;
1468
1469         for (si = 0; si < symbols->d_size / sizeof(Elf64_Sym); si++) {
1470                 Elf64_Sym *sym = elf_sym_by_idx(obj, si);
1471
1472                 if (ELF64_ST_TYPE(sym->st_info) != STT_OBJECT)
1473                         continue;
1474
1475                 if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL &&
1476                     ELF64_ST_BIND(sym->st_info) != STB_WEAK)
1477                         continue;
1478
1479                 sname = elf_sym_str(obj, sym->st_name);
1480                 if (!sname) {
1481                         pr_warn("failed to get sym name string for var %s\n", name);
1482                         return -EIO;
1483                 }
1484                 if (strcmp(name, sname) == 0) {
1485                         *off = sym->st_value;
1486                         return 0;
1487                 }
1488         }
1489
1490         return -ENOENT;
1491 }
1492
1493 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj)
1494 {
1495         struct bpf_map *map;
1496         int err;
1497
1498         err = libbpf_ensure_mem((void **)&obj->maps, &obj->maps_cap,
1499                                 sizeof(*obj->maps), obj->nr_maps + 1);
1500         if (err)
1501                 return ERR_PTR(err);
1502
1503         map = &obj->maps[obj->nr_maps++];
1504         map->obj = obj;
1505         map->fd = -1;
1506         map->inner_map_fd = -1;
1507         map->autocreate = true;
1508
1509         return map;
1510 }
1511
1512 static size_t bpf_map_mmap_sz(const struct bpf_map *map)
1513 {
1514         long page_sz = sysconf(_SC_PAGE_SIZE);
1515         size_t map_sz;
1516
1517         map_sz = (size_t)roundup(map->def.value_size, 8) * map->def.max_entries;
1518         map_sz = roundup(map_sz, page_sz);
1519         return map_sz;
1520 }
1521
1522 static char *internal_map_name(struct bpf_object *obj, const char *real_name)
1523 {
1524         char map_name[BPF_OBJ_NAME_LEN], *p;
1525         int pfx_len, sfx_len = max((size_t)7, strlen(real_name));
1526
1527         /* This is one of the more confusing parts of libbpf for various
1528          * reasons, some of which are historical. The original idea for naming
1529          * internal names was to include as much of BPF object name prefix as
1530          * possible, so that it can be distinguished from similar internal
1531          * maps of a different BPF object.
1532          * As an example, let's say we have bpf_object named 'my_object_name'
1533          * and internal map corresponding to '.rodata' ELF section. The final
1534          * map name advertised to user and to the kernel will be
1535          * 'my_objec.rodata', taking first 8 characters of object name and
1536          * entire 7 characters of '.rodata'.
1537          * Somewhat confusingly, if internal map ELF section name is shorter
1538          * than 7 characters, e.g., '.bss', we still reserve 7 characters
1539          * for the suffix, even though we only have 4 actual characters, and
1540          * resulting map will be called 'my_objec.bss', not even using all 15
1541          * characters allowed by the kernel. Oh well, at least the truncated
1542          * object name is somewhat consistent in this case. But if the map
1543          * name is '.kconfig', we'll still have entirety of '.kconfig' added
1544          * (8 chars) and thus will be left with only first 7 characters of the
1545          * object name ('my_obje'). Happy guessing, user, that the final map
1546          * name will be "my_obje.kconfig".
1547          * Now, with libbpf starting to support arbitrarily named .rodata.*
1548          * and .data.* data sections, it's possible that ELF section name is
1549          * longer than allowed 15 chars, so we now need to be careful to take
1550          * only up to 15 first characters of ELF name, taking no BPF object
1551          * name characters at all. So '.rodata.abracadabra' will result in
1552          * '.rodata.abracad' kernel and user-visible name.
1553          * We need to keep this convoluted logic intact for .data, .bss and
1554          * .rodata maps, but for new custom .data.custom and .rodata.custom
1555          * maps we use their ELF names as is, not prepending bpf_object name
1556          * in front. We still need to truncate them to 15 characters for the
1557          * kernel. Full name can be recovered for such maps by using DATASEC
1558          * BTF type associated with such map's value type, though.
1559          */
1560         if (sfx_len >= BPF_OBJ_NAME_LEN)
1561                 sfx_len = BPF_OBJ_NAME_LEN - 1;
1562
1563         /* if there are two or more dots in map name, it's a custom dot map */
1564         if (strchr(real_name + 1, '.') != NULL)
1565                 pfx_len = 0;
1566         else
1567                 pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name));
1568
1569         snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,
1570                  sfx_len, real_name);
1571
1572         /* sanitise map name to characters allowed by kernel */
1573         for (p = map_name; *p && p < map_name + sizeof(map_name); p++)
1574                 if (!isalnum(*p) && *p != '_' && *p != '.')
1575                         *p = '_';
1576
1577         return strdup(map_name);
1578 }
1579
1580 static int
1581 bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map);
1582
1583 static int
1584 bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
1585                               const char *real_name, int sec_idx, void *data, size_t data_sz)
1586 {
1587         struct bpf_map_def *def;
1588         struct bpf_map *map;
1589         int err;
1590
1591         map = bpf_object__add_map(obj);
1592         if (IS_ERR(map))
1593                 return PTR_ERR(map);
1594
1595         map->libbpf_type = type;
1596         map->sec_idx = sec_idx;
1597         map->sec_offset = 0;
1598         map->real_name = strdup(real_name);
1599         map->name = internal_map_name(obj, real_name);
1600         if (!map->real_name || !map->name) {
1601                 zfree(&map->real_name);
1602                 zfree(&map->name);
1603                 return -ENOMEM;
1604         }
1605
1606         def = &map->def;
1607         def->type = BPF_MAP_TYPE_ARRAY;
1608         def->key_size = sizeof(int);
1609         def->value_size = data_sz;
1610         def->max_entries = 1;
1611         def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG
1612                          ? BPF_F_RDONLY_PROG : 0;
1613         def->map_flags |= BPF_F_MMAPABLE;
1614
1615         pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n",
1616                  map->name, map->sec_idx, map->sec_offset, def->map_flags);
1617
1618         map->mmaped = mmap(NULL, bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE,
1619                            MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1620         if (map->mmaped == MAP_FAILED) {
1621                 err = -errno;
1622                 map->mmaped = NULL;
1623                 pr_warn("failed to alloc map '%s' content buffer: %d\n",
1624                         map->name, err);
1625                 zfree(&map->real_name);
1626                 zfree(&map->name);
1627                 return err;
1628         }
1629
1630         /* failures are fine because of maps like .rodata.str1.1 */
1631         (void) bpf_map_find_btf_info(obj, map);
1632
1633         if (data)
1634                 memcpy(map->mmaped, data, data_sz);
1635
1636         pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
1637         return 0;
1638 }
1639
1640 static int bpf_object__init_global_data_maps(struct bpf_object *obj)
1641 {
1642         struct elf_sec_desc *sec_desc;
1643         const char *sec_name;
1644         int err = 0, sec_idx;
1645
1646         /*
1647          * Populate obj->maps with libbpf internal maps.
1648          */
1649         for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) {
1650                 sec_desc = &obj->efile.secs[sec_idx];
1651
1652                 /* Skip recognized sections with size 0. */
1653                 if (!sec_desc->data || sec_desc->data->d_size == 0)
1654                         continue;
1655
1656                 switch (sec_desc->sec_type) {
1657                 case SEC_DATA:
1658                         sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
1659                         err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA,
1660                                                             sec_name, sec_idx,
1661                                                             sec_desc->data->d_buf,
1662                                                             sec_desc->data->d_size);
1663                         break;
1664                 case SEC_RODATA:
1665                         obj->has_rodata = true;
1666                         sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
1667                         err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA,
1668                                                             sec_name, sec_idx,
1669                                                             sec_desc->data->d_buf,
1670                                                             sec_desc->data->d_size);
1671                         break;
1672                 case SEC_BSS:
1673                         sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
1674                         err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS,
1675                                                             sec_name, sec_idx,
1676                                                             NULL,
1677                                                             sec_desc->data->d_size);
1678                         break;
1679                 default:
1680                         /* skip */
1681                         break;
1682                 }
1683                 if (err)
1684                         return err;
1685         }
1686         return 0;
1687 }
1688
1689
1690 static struct extern_desc *find_extern_by_name(const struct bpf_object *obj,
1691                                                const void *name)
1692 {
1693         int i;
1694
1695         for (i = 0; i < obj->nr_extern; i++) {
1696                 if (strcmp(obj->externs[i].name, name) == 0)
1697                         return &obj->externs[i];
1698         }
1699         return NULL;
1700 }
1701
1702 static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val,
1703                               char value)
1704 {
1705         switch (ext->kcfg.type) {
1706         case KCFG_BOOL:
1707                 if (value == 'm') {
1708                         pr_warn("extern (kcfg) '%s': value '%c' implies tristate or char type\n",
1709                                 ext->name, value);
1710                         return -EINVAL;
1711                 }
1712                 *(bool *)ext_val = value == 'y' ? true : false;
1713                 break;
1714         case KCFG_TRISTATE:
1715                 if (value == 'y')
1716                         *(enum libbpf_tristate *)ext_val = TRI_YES;
1717                 else if (value == 'm')
1718                         *(enum libbpf_tristate *)ext_val = TRI_MODULE;
1719                 else /* value == 'n' */
1720                         *(enum libbpf_tristate *)ext_val = TRI_NO;
1721                 break;
1722         case KCFG_CHAR:
1723                 *(char *)ext_val = value;
1724                 break;
1725         case KCFG_UNKNOWN:
1726         case KCFG_INT:
1727         case KCFG_CHAR_ARR:
1728         default:
1729                 pr_warn("extern (kcfg) '%s': value '%c' implies bool, tristate, or char type\n",
1730                         ext->name, value);
1731                 return -EINVAL;
1732         }
1733         ext->is_set = true;
1734         return 0;
1735 }
1736
1737 static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val,
1738                               const char *value)
1739 {
1740         size_t len;
1741
1742         if (ext->kcfg.type != KCFG_CHAR_ARR) {
1743                 pr_warn("extern (kcfg) '%s': value '%s' implies char array type\n",
1744                         ext->name, value);
1745                 return -EINVAL;
1746         }
1747
1748         len = strlen(value);
1749         if (value[len - 1] != '"') {
1750                 pr_warn("extern (kcfg) '%s': invalid string config '%s'\n",
1751                         ext->name, value);
1752                 return -EINVAL;
1753         }
1754
1755         /* strip quotes */
1756         len -= 2;
1757         if (len >= ext->kcfg.sz) {
1758                 pr_warn("extern (kcfg) '%s': long string '%s' of (%zu bytes) truncated to %d bytes\n",
1759                         ext->name, value, len, ext->kcfg.sz - 1);
1760                 len = ext->kcfg.sz - 1;
1761         }
1762         memcpy(ext_val, value + 1, len);
1763         ext_val[len] = '\0';
1764         ext->is_set = true;
1765         return 0;
1766 }
1767
1768 static int parse_u64(const char *value, __u64 *res)
1769 {
1770         char *value_end;
1771         int err;
1772
1773         errno = 0;
1774         *res = strtoull(value, &value_end, 0);
1775         if (errno) {
1776                 err = -errno;
1777                 pr_warn("failed to parse '%s' as integer: %d\n", value, err);
1778                 return err;
1779         }
1780         if (*value_end) {
1781                 pr_warn("failed to parse '%s' as integer completely\n", value);
1782                 return -EINVAL;
1783         }
1784         return 0;
1785 }
1786
1787 static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v)
1788 {
1789         int bit_sz = ext->kcfg.sz * 8;
1790
1791         if (ext->kcfg.sz == 8)
1792                 return true;
1793
1794         /* Validate that value stored in u64 fits in integer of `ext->sz`
1795          * bytes size without any loss of information. If the target integer
1796          * is signed, we rely on the following limits of integer type of
1797          * Y bits and subsequent transformation:
1798          *
1799          *     -2^(Y-1) <= X           <= 2^(Y-1) - 1
1800          *            0 <= X + 2^(Y-1) <= 2^Y - 1
1801          *            0 <= X + 2^(Y-1) <  2^Y
1802          *
1803          *  For unsigned target integer, check that all the (64 - Y) bits are
1804          *  zero.
1805          */
1806         if (ext->kcfg.is_signed)
1807                 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);
1808         else
1809                 return (v >> bit_sz) == 0;
1810 }
1811
1812 static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val,
1813                               __u64 value)
1814 {
1815         if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR &&
1816             ext->kcfg.type != KCFG_BOOL) {
1817                 pr_warn("extern (kcfg) '%s': value '%llu' implies integer, char, or boolean type\n",
1818                         ext->name, (unsigned long long)value);
1819                 return -EINVAL;
1820         }
1821         if (ext->kcfg.type == KCFG_BOOL && value > 1) {
1822                 pr_warn("extern (kcfg) '%s': value '%llu' isn't boolean compatible\n",
1823                         ext->name, (unsigned long long)value);
1824                 return -EINVAL;
1825
1826         }
1827         if (!is_kcfg_value_in_range(ext, value)) {
1828                 pr_warn("extern (kcfg) '%s': value '%llu' doesn't fit in %d bytes\n",
1829                         ext->name, (unsigned long long)value, ext->kcfg.sz);
1830                 return -ERANGE;
1831         }
1832         switch (ext->kcfg.sz) {
1833                 case 1: *(__u8 *)ext_val = value; break;
1834                 case 2: *(__u16 *)ext_val = value; break;
1835                 case 4: *(__u32 *)ext_val = value; break;
1836                 case 8: *(__u64 *)ext_val = value; break;
1837                 default:
1838                         return -EINVAL;
1839         }
1840         ext->is_set = true;
1841         return 0;
1842 }
1843
1844 static int bpf_object__process_kconfig_line(struct bpf_object *obj,
1845                                             char *buf, void *data)
1846 {
1847         struct extern_desc *ext;
1848         char *sep, *value;
1849         int len, err = 0;
1850         void *ext_val;
1851         __u64 num;
1852
1853         if (!str_has_pfx(buf, "CONFIG_"))
1854                 return 0;
1855
1856         sep = strchr(buf, '=');
1857         if (!sep) {
1858                 pr_warn("failed to parse '%s': no separator\n", buf);
1859                 return -EINVAL;
1860         }
1861
1862         /* Trim ending '\n' */
1863         len = strlen(buf);
1864         if (buf[len - 1] == '\n')
1865                 buf[len - 1] = '\0';
1866         /* Split on '=' and ensure that a value is present. */
1867         *sep = '\0';
1868         if (!sep[1]) {
1869                 *sep = '=';
1870                 pr_warn("failed to parse '%s': no value\n", buf);
1871                 return -EINVAL;
1872         }
1873
1874         ext = find_extern_by_name(obj, buf);
1875         if (!ext || ext->is_set)
1876                 return 0;
1877
1878         ext_val = data + ext->kcfg.data_off;
1879         value = sep + 1;
1880
1881         switch (*value) {
1882         case 'y': case 'n': case 'm':
1883                 err = set_kcfg_value_tri(ext, ext_val, *value);
1884                 break;
1885         case '"':
1886                 err = set_kcfg_value_str(ext, ext_val, value);
1887                 break;
1888         default:
1889                 /* assume integer */
1890                 err = parse_u64(value, &num);
1891                 if (err) {
1892                         pr_warn("extern (kcfg) '%s': value '%s' isn't a valid integer\n", ext->name, value);
1893                         return err;
1894                 }
1895                 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) {
1896                         pr_warn("extern (kcfg) '%s': value '%s' implies integer type\n", ext->name, value);
1897                         return -EINVAL;
1898                 }
1899                 err = set_kcfg_value_num(ext, ext_val, num);
1900                 break;
1901         }
1902         if (err)
1903                 return err;
1904         pr_debug("extern (kcfg) '%s': set to %s\n", ext->name, value);
1905         return 0;
1906 }
1907
1908 static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data)
1909 {
1910         char buf[PATH_MAX];
1911         struct utsname uts;
1912         int len, err = 0;
1913         gzFile file;
1914
1915         uname(&uts);
1916         len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release);
1917         if (len < 0)
1918                 return -EINVAL;
1919         else if (len >= PATH_MAX)
1920                 return -ENAMETOOLONG;
1921
1922         /* gzopen also accepts uncompressed files. */
1923         file = gzopen(buf, "r");
1924         if (!file)
1925                 file = gzopen("/proc/config.gz", "r");
1926
1927         if (!file) {
1928                 pr_warn("failed to open system Kconfig\n");
1929                 return -ENOENT;
1930         }
1931
1932         while (gzgets(file, buf, sizeof(buf))) {
1933                 err = bpf_object__process_kconfig_line(obj, buf, data);
1934                 if (err) {
1935                         pr_warn("error parsing system Kconfig line '%s': %d\n",
1936                                 buf, err);
1937                         goto out;
1938                 }
1939         }
1940
1941 out:
1942         gzclose(file);
1943         return err;
1944 }
1945
1946 static int bpf_object__read_kconfig_mem(struct bpf_object *obj,
1947                                         const char *config, void *data)
1948 {
1949         char buf[PATH_MAX];
1950         int err = 0;
1951         FILE *file;
1952
1953         file = fmemopen((void *)config, strlen(config), "r");
1954         if (!file) {
1955                 err = -errno;
1956                 pr_warn("failed to open in-memory Kconfig: %d\n", err);
1957                 return err;
1958         }
1959
1960         while (fgets(buf, sizeof(buf), file)) {
1961                 err = bpf_object__process_kconfig_line(obj, buf, data);
1962                 if (err) {
1963                         pr_warn("error parsing in-memory Kconfig line '%s': %d\n",
1964                                 buf, err);
1965                         break;
1966                 }
1967         }
1968
1969         fclose(file);
1970         return err;
1971 }
1972
1973 static int bpf_object__init_kconfig_map(struct bpf_object *obj)
1974 {
1975         struct extern_desc *last_ext = NULL, *ext;
1976         size_t map_sz;
1977         int i, err;
1978
1979         for (i = 0; i < obj->nr_extern; i++) {
1980                 ext = &obj->externs[i];
1981                 if (ext->type == EXT_KCFG)
1982                         last_ext = ext;
1983         }
1984
1985         if (!last_ext)
1986                 return 0;
1987
1988         map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz;
1989         err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG,
1990                                             ".kconfig", obj->efile.symbols_shndx,
1991                                             NULL, map_sz);
1992         if (err)
1993                 return err;
1994
1995         obj->kconfig_map_idx = obj->nr_maps - 1;
1996
1997         return 0;
1998 }
1999
2000 const struct btf_type *
2001 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id)
2002 {
2003         const struct btf_type *t = btf__type_by_id(btf, id);
2004
2005         if (res_id)
2006                 *res_id = id;
2007
2008         while (btf_is_mod(t) || btf_is_typedef(t)) {
2009                 if (res_id)
2010                         *res_id = t->type;
2011                 t = btf__type_by_id(btf, t->type);
2012         }
2013
2014         return t;
2015 }
2016
2017 static const struct btf_type *
2018 resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id)
2019 {
2020         const struct btf_type *t;
2021
2022         t = skip_mods_and_typedefs(btf, id, NULL);
2023         if (!btf_is_ptr(t))
2024                 return NULL;
2025
2026         t = skip_mods_and_typedefs(btf, t->type, res_id);
2027
2028         return btf_is_func_proto(t) ? t : NULL;
2029 }
2030
2031 static const char *__btf_kind_str(__u16 kind)
2032 {
2033         switch (kind) {
2034         case BTF_KIND_UNKN: return "void";
2035         case BTF_KIND_INT: return "int";
2036         case BTF_KIND_PTR: return "ptr";
2037         case BTF_KIND_ARRAY: return "array";
2038         case BTF_KIND_STRUCT: return "struct";
2039         case BTF_KIND_UNION: return "union";
2040         case BTF_KIND_ENUM: return "enum";
2041         case BTF_KIND_FWD: return "fwd";
2042         case BTF_KIND_TYPEDEF: return "typedef";
2043         case BTF_KIND_VOLATILE: return "volatile";
2044         case BTF_KIND_CONST: return "const";
2045         case BTF_KIND_RESTRICT: return "restrict";
2046         case BTF_KIND_FUNC: return "func";
2047         case BTF_KIND_FUNC_PROTO: return "func_proto";
2048         case BTF_KIND_VAR: return "var";
2049         case BTF_KIND_DATASEC: return "datasec";
2050         case BTF_KIND_FLOAT: return "float";
2051         case BTF_KIND_DECL_TAG: return "decl_tag";
2052         case BTF_KIND_TYPE_TAG: return "type_tag";
2053         case BTF_KIND_ENUM64: return "enum64";
2054         default: return "unknown";
2055         }
2056 }
2057
2058 const char *btf_kind_str(const struct btf_type *t)
2059 {
2060         return __btf_kind_str(btf_kind(t));
2061 }
2062
2063 /*
2064  * Fetch integer attribute of BTF map definition. Such attributes are
2065  * represented using a pointer to an array, in which dimensionality of array
2066  * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY];
2067  * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF
2068  * type definition, while using only sizeof(void *) space in ELF data section.
2069  */
2070 static bool get_map_field_int(const char *map_name, const struct btf *btf,
2071                               const struct btf_member *m, __u32 *res)
2072 {
2073         const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
2074         const char *name = btf__name_by_offset(btf, m->name_off);
2075         const struct btf_array *arr_info;
2076         const struct btf_type *arr_t;
2077
2078         if (!btf_is_ptr(t)) {
2079                 pr_warn("map '%s': attr '%s': expected PTR, got %s.\n",
2080                         map_name, name, btf_kind_str(t));
2081                 return false;
2082         }
2083
2084         arr_t = btf__type_by_id(btf, t->type);
2085         if (!arr_t) {
2086                 pr_warn("map '%s': attr '%s': type [%u] not found.\n",
2087                         map_name, name, t->type);
2088                 return false;
2089         }
2090         if (!btf_is_array(arr_t)) {
2091                 pr_warn("map '%s': attr '%s': expected ARRAY, got %s.\n",
2092                         map_name, name, btf_kind_str(arr_t));
2093                 return false;
2094         }
2095         arr_info = btf_array(arr_t);
2096         *res = arr_info->nelems;
2097         return true;
2098 }
2099
2100 static int pathname_concat(char *buf, size_t buf_sz, const char *path, const char *name)
2101 {
2102         int len;
2103
2104         len = snprintf(buf, buf_sz, "%s/%s", path, name);
2105         if (len < 0)
2106                 return -EINVAL;
2107         if (len >= buf_sz)
2108                 return -ENAMETOOLONG;
2109
2110         return 0;
2111 }
2112
2113 static int build_map_pin_path(struct bpf_map *map, const char *path)
2114 {
2115         char buf[PATH_MAX];
2116         int err;
2117
2118         if (!path)
2119                 path = "/sys/fs/bpf";
2120
2121         err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map));
2122         if (err)
2123                 return err;
2124
2125         return bpf_map__set_pin_path(map, buf);
2126 }
2127
2128 /* should match definition in bpf_helpers.h */
2129 enum libbpf_pin_type {
2130         LIBBPF_PIN_NONE,
2131         /* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */
2132         LIBBPF_PIN_BY_NAME,
2133 };
2134
2135 int parse_btf_map_def(const char *map_name, struct btf *btf,
2136                       const struct btf_type *def_t, bool strict,
2137                       struct btf_map_def *map_def, struct btf_map_def *inner_def)
2138 {
2139         const struct btf_type *t;
2140         const struct btf_member *m;
2141         bool is_inner = inner_def == NULL;
2142         int vlen, i;
2143
2144         vlen = btf_vlen(def_t);
2145         m = btf_members(def_t);
2146         for (i = 0; i < vlen; i++, m++) {
2147                 const char *name = btf__name_by_offset(btf, m->name_off);
2148
2149                 if (!name) {
2150                         pr_warn("map '%s': invalid field #%d.\n", map_name, i);
2151                         return -EINVAL;
2152                 }
2153                 if (strcmp(name, "type") == 0) {
2154                         if (!get_map_field_int(map_name, btf, m, &map_def->map_type))
2155                                 return -EINVAL;
2156                         map_def->parts |= MAP_DEF_MAP_TYPE;
2157                 } else if (strcmp(name, "max_entries") == 0) {
2158                         if (!get_map_field_int(map_name, btf, m, &map_def->max_entries))
2159                                 return -EINVAL;
2160                         map_def->parts |= MAP_DEF_MAX_ENTRIES;
2161                 } else if (strcmp(name, "map_flags") == 0) {
2162                         if (!get_map_field_int(map_name, btf, m, &map_def->map_flags))
2163                                 return -EINVAL;
2164                         map_def->parts |= MAP_DEF_MAP_FLAGS;
2165                 } else if (strcmp(name, "numa_node") == 0) {
2166                         if (!get_map_field_int(map_name, btf, m, &map_def->numa_node))
2167                                 return -EINVAL;
2168                         map_def->parts |= MAP_DEF_NUMA_NODE;
2169                 } else if (strcmp(name, "key_size") == 0) {
2170                         __u32 sz;
2171
2172                         if (!get_map_field_int(map_name, btf, m, &sz))
2173                                 return -EINVAL;
2174                         if (map_def->key_size && map_def->key_size != sz) {
2175                                 pr_warn("map '%s': conflicting key size %u != %u.\n",
2176                                         map_name, map_def->key_size, sz);
2177                                 return -EINVAL;
2178                         }
2179                         map_def->key_size = sz;
2180                         map_def->parts |= MAP_DEF_KEY_SIZE;
2181                 } else if (strcmp(name, "key") == 0) {
2182                         __s64 sz;
2183
2184                         t = btf__type_by_id(btf, m->type);
2185                         if (!t) {
2186                                 pr_warn("map '%s': key type [%d] not found.\n",
2187                                         map_name, m->type);
2188                                 return -EINVAL;
2189                         }
2190                         if (!btf_is_ptr(t)) {
2191                                 pr_warn("map '%s': key spec is not PTR: %s.\n",
2192                                         map_name, btf_kind_str(t));
2193                                 return -EINVAL;
2194                         }
2195                         sz = btf__resolve_size(btf, t->type);
2196                         if (sz < 0) {
2197                                 pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n",
2198                                         map_name, t->type, (ssize_t)sz);
2199                                 return sz;
2200                         }
2201                         if (map_def->key_size && map_def->key_size != sz) {
2202                                 pr_warn("map '%s': conflicting key size %u != %zd.\n",
2203                                         map_name, map_def->key_size, (ssize_t)sz);
2204                                 return -EINVAL;
2205                         }
2206                         map_def->key_size = sz;
2207                         map_def->key_type_id = t->type;
2208                         map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE;
2209                 } else if (strcmp(name, "value_size") == 0) {
2210                         __u32 sz;
2211
2212                         if (!get_map_field_int(map_name, btf, m, &sz))
2213                                 return -EINVAL;
2214                         if (map_def->value_size && map_def->value_size != sz) {
2215                                 pr_warn("map '%s': conflicting value size %u != %u.\n",
2216                                         map_name, map_def->value_size, sz);
2217                                 return -EINVAL;
2218                         }
2219                         map_def->value_size = sz;
2220                         map_def->parts |= MAP_DEF_VALUE_SIZE;
2221                 } else if (strcmp(name, "value") == 0) {
2222                         __s64 sz;
2223
2224                         t = btf__type_by_id(btf, m->type);
2225                         if (!t) {
2226                                 pr_warn("map '%s': value type [%d] not found.\n",
2227                                         map_name, m->type);
2228                                 return -EINVAL;
2229                         }
2230                         if (!btf_is_ptr(t)) {
2231                                 pr_warn("map '%s': value spec is not PTR: %s.\n",
2232                                         map_name, btf_kind_str(t));
2233                                 return -EINVAL;
2234                         }
2235                         sz = btf__resolve_size(btf, t->type);
2236                         if (sz < 0) {
2237                                 pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n",
2238                                         map_name, t->type, (ssize_t)sz);
2239                                 return sz;
2240                         }
2241                         if (map_def->value_size && map_def->value_size != sz) {
2242                                 pr_warn("map '%s': conflicting value size %u != %zd.\n",
2243                                         map_name, map_def->value_size, (ssize_t)sz);
2244                                 return -EINVAL;
2245                         }
2246                         map_def->value_size = sz;
2247                         map_def->value_type_id = t->type;
2248                         map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE;
2249                 }
2250                 else if (strcmp(name, "values") == 0) {
2251                         bool is_map_in_map = bpf_map_type__is_map_in_map(map_def->map_type);
2252                         bool is_prog_array = map_def->map_type == BPF_MAP_TYPE_PROG_ARRAY;
2253                         const char *desc = is_map_in_map ? "map-in-map inner" : "prog-array value";
2254                         char inner_map_name[128];
2255                         int err;
2256
2257                         if (is_inner) {
2258                                 pr_warn("map '%s': multi-level inner maps not supported.\n",
2259                                         map_name);
2260                                 return -ENOTSUP;
2261                         }
2262                         if (i != vlen - 1) {
2263                                 pr_warn("map '%s': '%s' member should be last.\n",
2264                                         map_name, name);
2265                                 return -EINVAL;
2266                         }
2267                         if (!is_map_in_map && !is_prog_array) {
2268                                 pr_warn("map '%s': should be map-in-map or prog-array.\n",
2269                                         map_name);
2270                                 return -ENOTSUP;
2271                         }
2272                         if (map_def->value_size && map_def->value_size != 4) {
2273                                 pr_warn("map '%s': conflicting value size %u != 4.\n",
2274                                         map_name, map_def->value_size);
2275                                 return -EINVAL;
2276                         }
2277                         map_def->value_size = 4;
2278                         t = btf__type_by_id(btf, m->type);
2279                         if (!t) {
2280                                 pr_warn("map '%s': %s type [%d] not found.\n",
2281                                         map_name, desc, m->type);
2282                                 return -EINVAL;
2283                         }
2284                         if (!btf_is_array(t) || btf_array(t)->nelems) {
2285                                 pr_warn("map '%s': %s spec is not a zero-sized array.\n",
2286                                         map_name, desc);
2287                                 return -EINVAL;
2288                         }
2289                         t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL);
2290                         if (!btf_is_ptr(t)) {
2291                                 pr_warn("map '%s': %s def is of unexpected kind %s.\n",
2292                                         map_name, desc, btf_kind_str(t));
2293                                 return -EINVAL;
2294                         }
2295                         t = skip_mods_and_typedefs(btf, t->type, NULL);
2296                         if (is_prog_array) {
2297                                 if (!btf_is_func_proto(t)) {
2298                                         pr_warn("map '%s': prog-array value def is of unexpected kind %s.\n",
2299                                                 map_name, btf_kind_str(t));
2300                                         return -EINVAL;
2301                                 }
2302                                 continue;
2303                         }
2304                         if (!btf_is_struct(t)) {
2305                                 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
2306                                         map_name, btf_kind_str(t));
2307                                 return -EINVAL;
2308                         }
2309
2310                         snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", map_name);
2311                         err = parse_btf_map_def(inner_map_name, btf, t, strict, inner_def, NULL);
2312                         if (err)
2313                                 return err;
2314
2315                         map_def->parts |= MAP_DEF_INNER_MAP;
2316                 } else if (strcmp(name, "pinning") == 0) {
2317                         __u32 val;
2318
2319                         if (is_inner) {
2320                                 pr_warn("map '%s': inner def can't be pinned.\n", map_name);
2321                                 return -EINVAL;
2322                         }
2323                         if (!get_map_field_int(map_name, btf, m, &val))
2324                                 return -EINVAL;
2325                         if (val != LIBBPF_PIN_NONE && val != LIBBPF_PIN_BY_NAME) {
2326                                 pr_warn("map '%s': invalid pinning value %u.\n",
2327                                         map_name, val);
2328                                 return -EINVAL;
2329                         }
2330                         map_def->pinning = val;
2331                         map_def->parts |= MAP_DEF_PINNING;
2332                 } else if (strcmp(name, "map_extra") == 0) {
2333                         __u32 map_extra;
2334
2335                         if (!get_map_field_int(map_name, btf, m, &map_extra))
2336                                 return -EINVAL;
2337                         map_def->map_extra = map_extra;
2338                         map_def->parts |= MAP_DEF_MAP_EXTRA;
2339                 } else {
2340                         if (strict) {
2341                                 pr_warn("map '%s': unknown field '%s'.\n", map_name, name);
2342                                 return -ENOTSUP;
2343                         }
2344                         pr_debug("map '%s': ignoring unknown field '%s'.\n", map_name, name);
2345                 }
2346         }
2347
2348         if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) {
2349                 pr_warn("map '%s': map type isn't specified.\n", map_name);
2350                 return -EINVAL;
2351         }
2352
2353         return 0;
2354 }
2355
2356 static size_t adjust_ringbuf_sz(size_t sz)
2357 {
2358         __u32 page_sz = sysconf(_SC_PAGE_SIZE);
2359         __u32 mul;
2360
2361         /* if user forgot to set any size, make sure they see error */
2362         if (sz == 0)
2363                 return 0;
2364         /* Kernel expects BPF_MAP_TYPE_RINGBUF's max_entries to be
2365          * a power-of-2 multiple of kernel's page size. If user diligently
2366          * satisified these conditions, pass the size through.
2367          */
2368         if ((sz % page_sz) == 0 && is_pow_of_2(sz / page_sz))
2369                 return sz;
2370
2371         /* Otherwise find closest (page_sz * power_of_2) product bigger than
2372          * user-set size to satisfy both user size request and kernel
2373          * requirements and substitute correct max_entries for map creation.
2374          */
2375         for (mul = 1; mul <= UINT_MAX / page_sz; mul <<= 1) {
2376                 if (mul * page_sz > sz)
2377                         return mul * page_sz;
2378         }
2379
2380         /* if it's impossible to satisfy the conditions (i.e., user size is
2381          * very close to UINT_MAX but is not a power-of-2 multiple of
2382          * page_size) then just return original size and let kernel reject it
2383          */
2384         return sz;
2385 }
2386
2387 static bool map_is_ringbuf(const struct bpf_map *map)
2388 {
2389         return map->def.type == BPF_MAP_TYPE_RINGBUF ||
2390                map->def.type == BPF_MAP_TYPE_USER_RINGBUF;
2391 }
2392
2393 static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def)
2394 {
2395         map->def.type = def->map_type;
2396         map->def.key_size = def->key_size;
2397         map->def.value_size = def->value_size;
2398         map->def.max_entries = def->max_entries;
2399         map->def.map_flags = def->map_flags;
2400         map->map_extra = def->map_extra;
2401
2402         map->numa_node = def->numa_node;
2403         map->btf_key_type_id = def->key_type_id;
2404         map->btf_value_type_id = def->value_type_id;
2405
2406         /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
2407         if (map_is_ringbuf(map))
2408                 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
2409
2410         if (def->parts & MAP_DEF_MAP_TYPE)
2411                 pr_debug("map '%s': found type = %u.\n", map->name, def->map_type);
2412
2413         if (def->parts & MAP_DEF_KEY_TYPE)
2414                 pr_debug("map '%s': found key [%u], sz = %u.\n",
2415                          map->name, def->key_type_id, def->key_size);
2416         else if (def->parts & MAP_DEF_KEY_SIZE)
2417                 pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size);
2418
2419         if (def->parts & MAP_DEF_VALUE_TYPE)
2420                 pr_debug("map '%s': found value [%u], sz = %u.\n",
2421                          map->name, def->value_type_id, def->value_size);
2422         else if (def->parts & MAP_DEF_VALUE_SIZE)
2423                 pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size);
2424
2425         if (def->parts & MAP_DEF_MAX_ENTRIES)
2426                 pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries);
2427         if (def->parts & MAP_DEF_MAP_FLAGS)
2428                 pr_debug("map '%s': found map_flags = 0x%x.\n", map->name, def->map_flags);
2429         if (def->parts & MAP_DEF_MAP_EXTRA)
2430                 pr_debug("map '%s': found map_extra = 0x%llx.\n", map->name,
2431                          (unsigned long long)def->map_extra);
2432         if (def->parts & MAP_DEF_PINNING)
2433                 pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning);
2434         if (def->parts & MAP_DEF_NUMA_NODE)
2435                 pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node);
2436
2437         if (def->parts & MAP_DEF_INNER_MAP)
2438                 pr_debug("map '%s': found inner map definition.\n", map->name);
2439 }
2440
2441 static const char *btf_var_linkage_str(__u32 linkage)
2442 {
2443         switch (linkage) {
2444         case BTF_VAR_STATIC: return "static";
2445         case BTF_VAR_GLOBAL_ALLOCATED: return "global";
2446         case BTF_VAR_GLOBAL_EXTERN: return "extern";
2447         default: return "unknown";
2448         }
2449 }
2450
2451 static int bpf_object__init_user_btf_map(struct bpf_object *obj,
2452                                          const struct btf_type *sec,
2453                                          int var_idx, int sec_idx,
2454                                          const Elf_Data *data, bool strict,
2455                                          const char *pin_root_path)
2456 {
2457         struct btf_map_def map_def = {}, inner_def = {};
2458         const struct btf_type *var, *def;
2459         const struct btf_var_secinfo *vi;
2460         const struct btf_var *var_extra;
2461         const char *map_name;
2462         struct bpf_map *map;
2463         int err;
2464
2465         vi = btf_var_secinfos(sec) + var_idx;
2466         var = btf__type_by_id(obj->btf, vi->type);
2467         var_extra = btf_var(var);
2468         map_name = btf__name_by_offset(obj->btf, var->name_off);
2469
2470         if (map_name == NULL || map_name[0] == '\0') {
2471                 pr_warn("map #%d: empty name.\n", var_idx);
2472                 return -EINVAL;
2473         }
2474         if ((__u64)vi->offset + vi->size > data->d_size) {
2475                 pr_warn("map '%s' BTF data is corrupted.\n", map_name);
2476                 return -EINVAL;
2477         }
2478         if (!btf_is_var(var)) {
2479                 pr_warn("map '%s': unexpected var kind %s.\n",
2480                         map_name, btf_kind_str(var));
2481                 return -EINVAL;
2482         }
2483         if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) {
2484                 pr_warn("map '%s': unsupported map linkage %s.\n",
2485                         map_name, btf_var_linkage_str(var_extra->linkage));
2486                 return -EOPNOTSUPP;
2487         }
2488
2489         def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
2490         if (!btf_is_struct(def)) {
2491                 pr_warn("map '%s': unexpected def kind %s.\n",
2492                         map_name, btf_kind_str(var));
2493                 return -EINVAL;
2494         }
2495         if (def->size > vi->size) {
2496                 pr_warn("map '%s': invalid def size.\n", map_name);
2497                 return -EINVAL;
2498         }
2499
2500         map = bpf_object__add_map(obj);
2501         if (IS_ERR(map))
2502                 return PTR_ERR(map);
2503         map->name = strdup(map_name);
2504         if (!map->name) {
2505                 pr_warn("map '%s': failed to alloc map name.\n", map_name);
2506                 return -ENOMEM;
2507         }
2508         map->libbpf_type = LIBBPF_MAP_UNSPEC;
2509         map->def.type = BPF_MAP_TYPE_UNSPEC;
2510         map->sec_idx = sec_idx;
2511         map->sec_offset = vi->offset;
2512         map->btf_var_idx = var_idx;
2513         pr_debug("map '%s': at sec_idx %d, offset %zu.\n",
2514                  map_name, map->sec_idx, map->sec_offset);
2515
2516         err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def);
2517         if (err)
2518                 return err;
2519
2520         fill_map_from_def(map, &map_def);
2521
2522         if (map_def.pinning == LIBBPF_PIN_BY_NAME) {
2523                 err = build_map_pin_path(map, pin_root_path);
2524                 if (err) {
2525                         pr_warn("map '%s': couldn't build pin path.\n", map->name);
2526                         return err;
2527                 }
2528         }
2529
2530         if (map_def.parts & MAP_DEF_INNER_MAP) {
2531                 map->inner_map = calloc(1, sizeof(*map->inner_map));
2532                 if (!map->inner_map)
2533                         return -ENOMEM;
2534                 map->inner_map->fd = -1;
2535                 map->inner_map->sec_idx = sec_idx;
2536                 map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1);
2537                 if (!map->inner_map->name)
2538                         return -ENOMEM;
2539                 sprintf(map->inner_map->name, "%s.inner", map_name);
2540
2541                 fill_map_from_def(map->inner_map, &inner_def);
2542         }
2543
2544         err = bpf_map_find_btf_info(obj, map);
2545         if (err)
2546                 return err;
2547
2548         return 0;
2549 }
2550
2551 static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict,
2552                                           const char *pin_root_path)
2553 {
2554         const struct btf_type *sec = NULL;
2555         int nr_types, i, vlen, err;
2556         const struct btf_type *t;
2557         const char *name;
2558         Elf_Data *data;
2559         Elf_Scn *scn;
2560
2561         if (obj->efile.btf_maps_shndx < 0)
2562                 return 0;
2563
2564         scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx);
2565         data = elf_sec_data(obj, scn);
2566         if (!scn || !data) {
2567                 pr_warn("elf: failed to get %s map definitions for %s\n",
2568                         MAPS_ELF_SEC, obj->path);
2569                 return -EINVAL;
2570         }
2571
2572         nr_types = btf__type_cnt(obj->btf);
2573         for (i = 1; i < nr_types; i++) {
2574                 t = btf__type_by_id(obj->btf, i);
2575                 if (!btf_is_datasec(t))
2576                         continue;
2577                 name = btf__name_by_offset(obj->btf, t->name_off);
2578                 if (strcmp(name, MAPS_ELF_SEC) == 0) {
2579                         sec = t;
2580                         obj->efile.btf_maps_sec_btf_id = i;
2581                         break;
2582                 }
2583         }
2584
2585         if (!sec) {
2586                 pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC);
2587                 return -ENOENT;
2588         }
2589
2590         vlen = btf_vlen(sec);
2591         for (i = 0; i < vlen; i++) {
2592                 err = bpf_object__init_user_btf_map(obj, sec, i,
2593                                                     obj->efile.btf_maps_shndx,
2594                                                     data, strict,
2595                                                     pin_root_path);
2596                 if (err)
2597                         return err;
2598         }
2599
2600         return 0;
2601 }
2602
2603 static int bpf_object__init_maps(struct bpf_object *obj,
2604                                  const struct bpf_object_open_opts *opts)
2605 {
2606         const char *pin_root_path;
2607         bool strict;
2608         int err = 0;
2609
2610         strict = !OPTS_GET(opts, relaxed_maps, false);
2611         pin_root_path = OPTS_GET(opts, pin_root_path, NULL);
2612
2613         err = err ?: bpf_object__init_user_btf_maps(obj, strict, pin_root_path);
2614         err = err ?: bpf_object__init_global_data_maps(obj);
2615         err = err ?: bpf_object__init_kconfig_map(obj);
2616         err = err ?: bpf_object__init_struct_ops_maps(obj);
2617
2618         return err;
2619 }
2620
2621 static bool section_have_execinstr(struct bpf_object *obj, int idx)
2622 {
2623         Elf64_Shdr *sh;
2624
2625         sh = elf_sec_hdr(obj, elf_sec_by_idx(obj, idx));
2626         if (!sh)
2627                 return false;
2628
2629         return sh->sh_flags & SHF_EXECINSTR;
2630 }
2631
2632 static bool btf_needs_sanitization(struct bpf_object *obj)
2633 {
2634         bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);
2635         bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);
2636         bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);
2637         bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
2638         bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);
2639         bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);
2640         bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);
2641
2642         return !has_func || !has_datasec || !has_func_global || !has_float ||
2643                !has_decl_tag || !has_type_tag || !has_enum64;
2644 }
2645
2646 static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)
2647 {
2648         bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);
2649         bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);
2650         bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);
2651         bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
2652         bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);
2653         bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);
2654         bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);
2655         int enum64_placeholder_id = 0;
2656         struct btf_type *t;
2657         int i, j, vlen;
2658
2659         for (i = 1; i < btf__type_cnt(btf); i++) {
2660                 t = (struct btf_type *)btf__type_by_id(btf, i);
2661
2662                 if ((!has_datasec && btf_is_var(t)) || (!has_decl_tag && btf_is_decl_tag(t))) {
2663                         /* replace VAR/DECL_TAG with INT */
2664                         t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
2665                         /*
2666                          * using size = 1 is the safest choice, 4 will be too
2667                          * big and cause kernel BTF validation failure if
2668                          * original variable took less than 4 bytes
2669                          */
2670                         t->size = 1;
2671                         *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8);
2672                 } else if (!has_datasec && btf_is_datasec(t)) {
2673                         /* replace DATASEC with STRUCT */
2674                         const struct btf_var_secinfo *v = btf_var_secinfos(t);
2675                         struct btf_member *m = btf_members(t);
2676                         struct btf_type *vt;
2677                         char *name;
2678
2679                         name = (char *)btf__name_by_offset(btf, t->name_off);
2680                         while (*name) {
2681                                 if (*name == '.')
2682                                         *name = '_';
2683                                 name++;
2684                         }
2685
2686                         vlen = btf_vlen(t);
2687                         t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);
2688                         for (j = 0; j < vlen; j++, v++, m++) {
2689                                 /* order of field assignments is important */
2690                                 m->offset = v->offset * 8;
2691                                 m->type = v->type;
2692                                 /* preserve variable name as member name */
2693                                 vt = (void *)btf__type_by_id(btf, v->type);
2694                                 m->name_off = vt->name_off;
2695                         }
2696                 } else if (!has_func && btf_is_func_proto(t)) {
2697                         /* replace FUNC_PROTO with ENUM */
2698                         vlen = btf_vlen(t);
2699                         t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);
2700                         t->size = sizeof(__u32); /* kernel enforced */
2701                 } else if (!has_func && btf_is_func(t)) {
2702                         /* replace FUNC with TYPEDEF */
2703                         t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);
2704                 } else if (!has_func_global && btf_is_func(t)) {
2705                         /* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */
2706                         t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0);
2707                 } else if (!has_float && btf_is_float(t)) {
2708                         /* replace FLOAT with an equally-sized empty STRUCT;
2709                          * since C compilers do not accept e.g. "float" as a
2710                          * valid struct name, make it anonymous
2711                          */
2712                         t->name_off = 0;
2713                         t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0);
2714                 } else if (!has_type_tag && btf_is_type_tag(t)) {
2715                         /* replace TYPE_TAG with a CONST */
2716                         t->name_off = 0;
2717                         t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0);
2718                 } else if (!has_enum64 && btf_is_enum(t)) {
2719                         /* clear the kflag */
2720                         t->info = btf_type_info(btf_kind(t), btf_vlen(t), false);
2721                 } else if (!has_enum64 && btf_is_enum64(t)) {
2722                         /* replace ENUM64 with a union */
2723                         struct btf_member *m;
2724
2725                         if (enum64_placeholder_id == 0) {
2726                                 enum64_placeholder_id = btf__add_int(btf, "enum64_placeholder", 1, 0);
2727                                 if (enum64_placeholder_id < 0)
2728                                         return enum64_placeholder_id;
2729
2730                                 t = (struct btf_type *)btf__type_by_id(btf, i);
2731                         }
2732
2733                         m = btf_members(t);
2734                         vlen = btf_vlen(t);
2735                         t->info = BTF_INFO_ENC(BTF_KIND_UNION, 0, vlen);
2736                         for (j = 0; j < vlen; j++, m++) {
2737                                 m->type = enum64_placeholder_id;
2738                                 m->offset = 0;
2739                         }
2740                 }
2741         }
2742
2743         return 0;
2744 }
2745
2746 static bool libbpf_needs_btf(const struct bpf_object *obj)
2747 {
2748         return obj->efile.btf_maps_shndx >= 0 ||
2749                obj->efile.st_ops_shndx >= 0 ||
2750                obj->nr_extern > 0;
2751 }
2752
2753 static bool kernel_needs_btf(const struct bpf_object *obj)
2754 {
2755         return obj->efile.st_ops_shndx >= 0;
2756 }
2757
2758 static int bpf_object__init_btf(struct bpf_object *obj,
2759                                 Elf_Data *btf_data,
2760                                 Elf_Data *btf_ext_data)
2761 {
2762         int err = -ENOENT;
2763
2764         if (btf_data) {
2765                 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
2766                 err = libbpf_get_error(obj->btf);
2767                 if (err) {
2768                         obj->btf = NULL;
2769                         pr_warn("Error loading ELF section %s: %d.\n", BTF_ELF_SEC, err);
2770                         goto out;
2771                 }
2772                 /* enforce 8-byte pointers for BPF-targeted BTFs */
2773                 btf__set_pointer_size(obj->btf, 8);
2774         }
2775         if (btf_ext_data) {
2776                 struct btf_ext_info *ext_segs[3];
2777                 int seg_num, sec_num;
2778
2779                 if (!obj->btf) {
2780                         pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",
2781                                  BTF_EXT_ELF_SEC, BTF_ELF_SEC);
2782                         goto out;
2783                 }
2784                 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size);
2785                 err = libbpf_get_error(obj->btf_ext);
2786                 if (err) {
2787                         pr_warn("Error loading ELF section %s: %d. Ignored and continue.\n",
2788                                 BTF_EXT_ELF_SEC, err);
2789                         obj->btf_ext = NULL;
2790                         goto out;
2791                 }
2792
2793                 /* setup .BTF.ext to ELF section mapping */
2794                 ext_segs[0] = &obj->btf_ext->func_info;
2795                 ext_segs[1] = &obj->btf_ext->line_info;
2796                 ext_segs[2] = &obj->btf_ext->core_relo_info;
2797                 for (seg_num = 0; seg_num < ARRAY_SIZE(ext_segs); seg_num++) {
2798                         struct btf_ext_info *seg = ext_segs[seg_num];
2799                         const struct btf_ext_info_sec *sec;
2800                         const char *sec_name;
2801                         Elf_Scn *scn;
2802
2803                         if (seg->sec_cnt == 0)
2804                                 continue;
2805
2806                         seg->sec_idxs = calloc(seg->sec_cnt, sizeof(*seg->sec_idxs));
2807                         if (!seg->sec_idxs) {
2808                                 err = -ENOMEM;
2809                                 goto out;
2810                         }
2811
2812                         sec_num = 0;
2813                         for_each_btf_ext_sec(seg, sec) {
2814                                 /* preventively increment index to avoid doing
2815                                  * this before every continue below
2816                                  */
2817                                 sec_num++;
2818
2819                                 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
2820                                 if (str_is_empty(sec_name))
2821                                         continue;
2822                                 scn = elf_sec_by_name(obj, sec_name);
2823                                 if (!scn)
2824                                         continue;
2825
2826                                 seg->sec_idxs[sec_num - 1] = elf_ndxscn(scn);
2827                         }
2828                 }
2829         }
2830 out:
2831         if (err && libbpf_needs_btf(obj)) {
2832                 pr_warn("BTF is required, but is missing or corrupted.\n");
2833                 return err;
2834         }
2835         return 0;
2836 }
2837
2838 static int compare_vsi_off(const void *_a, const void *_b)
2839 {
2840         const struct btf_var_secinfo *a = _a;
2841         const struct btf_var_secinfo *b = _b;
2842
2843         return a->offset - b->offset;
2844 }
2845
2846 static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf,
2847                              struct btf_type *t)
2848 {
2849         __u32 size = 0, off = 0, i, vars = btf_vlen(t);
2850         const char *name = btf__name_by_offset(btf, t->name_off);
2851         const struct btf_type *t_var;
2852         struct btf_var_secinfo *vsi;
2853         const struct btf_var *var;
2854         int ret;
2855
2856         if (!name) {
2857                 pr_debug("No name found in string section for DATASEC kind.\n");
2858                 return -ENOENT;
2859         }
2860
2861         /* .extern datasec size and var offsets were set correctly during
2862          * extern collection step, so just skip straight to sorting variables
2863          */
2864         if (t->size)
2865                 goto sort_vars;
2866
2867         ret = find_elf_sec_sz(obj, name, &size);
2868         if (ret || !size) {
2869                 pr_debug("Invalid size for section %s: %u bytes\n", name, size);
2870                 return -ENOENT;
2871         }
2872
2873         t->size = size;
2874
2875         for (i = 0, vsi = btf_var_secinfos(t); i < vars; i++, vsi++) {
2876                 t_var = btf__type_by_id(btf, vsi->type);
2877                 if (!t_var || !btf_is_var(t_var)) {
2878                         pr_debug("Non-VAR type seen in section %s\n", name);
2879                         return -EINVAL;
2880                 }
2881
2882                 var = btf_var(t_var);
2883                 if (var->linkage == BTF_VAR_STATIC)
2884                         continue;
2885
2886                 name = btf__name_by_offset(btf, t_var->name_off);
2887                 if (!name) {
2888                         pr_debug("No name found in string section for VAR kind\n");
2889                         return -ENOENT;
2890                 }
2891
2892                 ret = find_elf_var_offset(obj, name, &off);
2893                 if (ret) {
2894                         pr_debug("No offset found in symbol table for VAR %s\n",
2895                                  name);
2896                         return -ENOENT;
2897                 }
2898
2899                 vsi->offset = off;
2900         }
2901
2902 sort_vars:
2903         qsort(btf_var_secinfos(t), vars, sizeof(*vsi), compare_vsi_off);
2904         return 0;
2905 }
2906
2907 static int btf_finalize_data(struct bpf_object *obj, struct btf *btf)
2908 {
2909         int err = 0;
2910         __u32 i, n = btf__type_cnt(btf);
2911
2912         for (i = 1; i < n; i++) {
2913                 struct btf_type *t = btf_type_by_id(btf, i);
2914
2915                 /* Loader needs to fix up some of the things compiler
2916                  * couldn't get its hands on while emitting BTF. This
2917                  * is section size and global variable offset. We use
2918                  * the info from the ELF itself for this purpose.
2919                  */
2920                 if (btf_is_datasec(t)) {
2921                         err = btf_fixup_datasec(obj, btf, t);
2922                         if (err)
2923                                 break;
2924                 }
2925         }
2926
2927         return libbpf_err(err);
2928 }
2929
2930 static int bpf_object__finalize_btf(struct bpf_object *obj)
2931 {
2932         int err;
2933
2934         if (!obj->btf)
2935                 return 0;
2936
2937         err = btf_finalize_data(obj, obj->btf);
2938         if (err) {
2939                 pr_warn("Error finalizing %s: %d.\n", BTF_ELF_SEC, err);
2940                 return err;
2941         }
2942
2943         return 0;
2944 }
2945
2946 static bool prog_needs_vmlinux_btf(struct bpf_program *prog)
2947 {
2948         if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
2949             prog->type == BPF_PROG_TYPE_LSM)
2950                 return true;
2951
2952         /* BPF_PROG_TYPE_TRACING programs which do not attach to other programs
2953          * also need vmlinux BTF
2954          */
2955         if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
2956                 return true;
2957
2958         return false;
2959 }
2960
2961 static bool obj_needs_vmlinux_btf(const struct bpf_object *obj)
2962 {
2963         struct bpf_program *prog;
2964         int i;
2965
2966         /* CO-RE relocations need kernel BTF, only when btf_custom_path
2967          * is not specified
2968          */
2969         if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path)
2970                 return true;
2971
2972         /* Support for typed ksyms needs kernel BTF */
2973         for (i = 0; i < obj->nr_extern; i++) {
2974                 const struct extern_desc *ext;
2975
2976                 ext = &obj->externs[i];
2977                 if (ext->type == EXT_KSYM && ext->ksym.type_id)
2978                         return true;
2979         }
2980
2981         bpf_object__for_each_program(prog, obj) {
2982                 if (!prog->autoload)
2983                         continue;
2984                 if (prog_needs_vmlinux_btf(prog))
2985                         return true;
2986         }
2987
2988         return false;
2989 }
2990
2991 static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force)
2992 {
2993         int err;
2994
2995         /* btf_vmlinux could be loaded earlier */
2996         if (obj->btf_vmlinux || obj->gen_loader)
2997                 return 0;
2998
2999         if (!force && !obj_needs_vmlinux_btf(obj))
3000                 return 0;
3001
3002         obj->btf_vmlinux = btf__load_vmlinux_btf();
3003         err = libbpf_get_error(obj->btf_vmlinux);
3004         if (err) {
3005                 pr_warn("Error loading vmlinux BTF: %d\n", err);
3006                 obj->btf_vmlinux = NULL;
3007                 return err;
3008         }
3009         return 0;
3010 }
3011
3012 static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj)
3013 {
3014         struct btf *kern_btf = obj->btf;
3015         bool btf_mandatory, sanitize;
3016         int i, err = 0;
3017
3018         if (!obj->btf)
3019                 return 0;
3020
3021         if (!kernel_supports(obj, FEAT_BTF)) {
3022                 if (kernel_needs_btf(obj)) {
3023                         err = -EOPNOTSUPP;
3024                         goto report;
3025                 }
3026                 pr_debug("Kernel doesn't support BTF, skipping uploading it.\n");
3027                 return 0;
3028         }
3029
3030         /* Even though some subprogs are global/weak, user might prefer more
3031          * permissive BPF verification process that BPF verifier performs for
3032          * static functions, taking into account more context from the caller
3033          * functions. In such case, they need to mark such subprogs with
3034          * __attribute__((visibility("hidden"))) and libbpf will adjust
3035          * corresponding FUNC BTF type to be marked as static and trigger more
3036          * involved BPF verification process.
3037          */
3038         for (i = 0; i < obj->nr_programs; i++) {
3039                 struct bpf_program *prog = &obj->programs[i];
3040                 struct btf_type *t;
3041                 const char *name;
3042                 int j, n;
3043
3044                 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog))
3045                         continue;
3046
3047                 n = btf__type_cnt(obj->btf);
3048                 for (j = 1; j < n; j++) {
3049                         t = btf_type_by_id(obj->btf, j);
3050                         if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL)
3051                                 continue;
3052
3053                         name = btf__str_by_offset(obj->btf, t->name_off);
3054                         if (strcmp(name, prog->name) != 0)
3055                                 continue;
3056
3057                         t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0);
3058                         break;
3059                 }
3060         }
3061
3062         sanitize = btf_needs_sanitization(obj);
3063         if (sanitize) {
3064                 const void *raw_data;
3065                 __u32 sz;
3066
3067                 /* clone BTF to sanitize a copy and leave the original intact */
3068                 raw_data = btf__raw_data(obj->btf, &sz);
3069                 kern_btf = btf__new(raw_data, sz);
3070                 err = libbpf_get_error(kern_btf);
3071                 if (err)
3072                         return err;
3073
3074                 /* enforce 8-byte pointers for BPF-targeted BTFs */
3075                 btf__set_pointer_size(obj->btf, 8);
3076                 err = bpf_object__sanitize_btf(obj, kern_btf);
3077                 if (err)
3078                         return err;
3079         }
3080
3081         if (obj->gen_loader) {
3082                 __u32 raw_size = 0;
3083                 const void *raw_data = btf__raw_data(kern_btf, &raw_size);
3084
3085                 if (!raw_data)
3086                         return -ENOMEM;
3087                 bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size);
3088                 /* Pretend to have valid FD to pass various fd >= 0 checks.
3089                  * This fd == 0 will not be used with any syscall and will be reset to -1 eventually.
3090                  */
3091                 btf__set_fd(kern_btf, 0);
3092         } else {
3093                 /* currently BPF_BTF_LOAD only supports log_level 1 */
3094                 err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size,
3095                                            obj->log_level ? 1 : 0);
3096         }
3097         if (sanitize) {
3098                 if (!err) {
3099                         /* move fd to libbpf's BTF */
3100                         btf__set_fd(obj->btf, btf__fd(kern_btf));
3101                         btf__set_fd(kern_btf, -1);
3102                 }
3103                 btf__free(kern_btf);
3104         }
3105 report:
3106         if (err) {
3107                 btf_mandatory = kernel_needs_btf(obj);
3108                 pr_warn("Error loading .BTF into kernel: %d. %s\n", err,
3109                         btf_mandatory ? "BTF is mandatory, can't proceed."
3110                                       : "BTF is optional, ignoring.");
3111                 if (!btf_mandatory)
3112                         err = 0;
3113         }
3114         return err;
3115 }
3116
3117 static const char *elf_sym_str(const struct bpf_object *obj, size_t off)
3118 {
3119         const char *name;
3120
3121         name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off);
3122         if (!name) {
3123                 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
3124                         off, obj->path, elf_errmsg(-1));
3125                 return NULL;
3126         }
3127
3128         return name;
3129 }
3130
3131 static const char *elf_sec_str(const struct bpf_object *obj, size_t off)
3132 {
3133         const char *name;
3134
3135         name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off);
3136         if (!name) {
3137                 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
3138                         off, obj->path, elf_errmsg(-1));
3139                 return NULL;
3140         }
3141
3142         return name;
3143 }
3144
3145 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx)
3146 {
3147         Elf_Scn *scn;
3148
3149         scn = elf_getscn(obj->efile.elf, idx);
3150         if (!scn) {
3151                 pr_warn("elf: failed to get section(%zu) from %s: %s\n",
3152                         idx, obj->path, elf_errmsg(-1));
3153                 return NULL;
3154         }
3155         return scn;
3156 }
3157
3158 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name)
3159 {
3160         Elf_Scn *scn = NULL;
3161         Elf *elf = obj->efile.elf;
3162         const char *sec_name;
3163
3164         while ((scn = elf_nextscn(elf, scn)) != NULL) {
3165                 sec_name = elf_sec_name(obj, scn);
3166                 if (!sec_name)
3167                         return NULL;
3168
3169                 if (strcmp(sec_name, name) != 0)
3170                         continue;
3171
3172                 return scn;
3173         }
3174         return NULL;
3175 }
3176
3177 static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn)
3178 {
3179         Elf64_Shdr *shdr;
3180
3181         if (!scn)
3182                 return NULL;
3183
3184         shdr = elf64_getshdr(scn);
3185         if (!shdr) {
3186                 pr_warn("elf: failed to get section(%zu) header from %s: %s\n",
3187                         elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3188                 return NULL;
3189         }
3190
3191         return shdr;
3192 }
3193
3194 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn)
3195 {
3196         const char *name;
3197         Elf64_Shdr *sh;
3198
3199         if (!scn)
3200                 return NULL;
3201
3202         sh = elf_sec_hdr(obj, scn);
3203         if (!sh)
3204                 return NULL;
3205
3206         name = elf_sec_str(obj, sh->sh_name);
3207         if (!name) {
3208                 pr_warn("elf: failed to get section(%zu) name from %s: %s\n",
3209                         elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3210                 return NULL;
3211         }
3212
3213         return name;
3214 }
3215
3216 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn)
3217 {
3218         Elf_Data *data;
3219
3220         if (!scn)
3221                 return NULL;
3222
3223         data = elf_getdata(scn, 0);
3224         if (!data) {
3225                 pr_warn("elf: failed to get section(%zu) %s data from %s: %s\n",
3226                         elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "<?>",
3227                         obj->path, elf_errmsg(-1));
3228                 return NULL;
3229         }
3230
3231         return data;
3232 }
3233
3234 static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx)
3235 {
3236         if (idx >= obj->efile.symbols->d_size / sizeof(Elf64_Sym))
3237                 return NULL;
3238
3239         return (Elf64_Sym *)obj->efile.symbols->d_buf + idx;
3240 }
3241
3242 static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx)
3243 {
3244         if (idx >= data->d_size / sizeof(Elf64_Rel))
3245                 return NULL;
3246
3247         return (Elf64_Rel *)data->d_buf + idx;
3248 }
3249
3250 static bool is_sec_name_dwarf(const char *name)
3251 {
3252         /* approximation, but the actual list is too long */
3253         return str_has_pfx(name, ".debug_");
3254 }
3255
3256 static bool ignore_elf_section(Elf64_Shdr *hdr, const char *name)
3257 {
3258         /* no special handling of .strtab */
3259         if (hdr->sh_type == SHT_STRTAB)
3260                 return true;
3261
3262         /* ignore .llvm_addrsig section as well */
3263         if (hdr->sh_type == SHT_LLVM_ADDRSIG)
3264                 return true;
3265
3266         /* no subprograms will lead to an empty .text section, ignore it */
3267         if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 &&
3268             strcmp(name, ".text") == 0)
3269                 return true;
3270
3271         /* DWARF sections */
3272         if (is_sec_name_dwarf(name))
3273                 return true;
3274
3275         if (str_has_pfx(name, ".rel")) {
3276                 name += sizeof(".rel") - 1;
3277                 /* DWARF section relocations */
3278                 if (is_sec_name_dwarf(name))
3279                         return true;
3280
3281                 /* .BTF and .BTF.ext don't need relocations */
3282                 if (strcmp(name, BTF_ELF_SEC) == 0 ||
3283                     strcmp(name, BTF_EXT_ELF_SEC) == 0)
3284                         return true;
3285         }
3286
3287         return false;
3288 }
3289
3290 static int cmp_progs(const void *_a, const void *_b)
3291 {
3292         const struct bpf_program *a = _a;
3293         const struct bpf_program *b = _b;
3294
3295         if (a->sec_idx != b->sec_idx)
3296                 return a->sec_idx < b->sec_idx ? -1 : 1;
3297
3298         /* sec_insn_off can't be the same within the section */
3299         return a->sec_insn_off < b->sec_insn_off ? -1 : 1;
3300 }
3301
3302 static int bpf_object__elf_collect(struct bpf_object *obj)
3303 {
3304         struct elf_sec_desc *sec_desc;
3305         Elf *elf = obj->efile.elf;
3306         Elf_Data *btf_ext_data = NULL;
3307         Elf_Data *btf_data = NULL;
3308         int idx = 0, err = 0;
3309         const char *name;
3310         Elf_Data *data;
3311         Elf_Scn *scn;
3312         Elf64_Shdr *sh;
3313
3314         /* ELF section indices are 0-based, but sec #0 is special "invalid"
3315          * section. e_shnum does include sec #0, so e_shnum is the necessary
3316          * size of an array to keep all the sections.
3317          */
3318         obj->efile.sec_cnt = obj->efile.ehdr->e_shnum;
3319         obj->efile.secs = calloc(obj->efile.sec_cnt, sizeof(*obj->efile.secs));
3320         if (!obj->efile.secs)
3321                 return -ENOMEM;
3322
3323         /* a bunch of ELF parsing functionality depends on processing symbols,
3324          * so do the first pass and find the symbol table
3325          */
3326         scn = NULL;
3327         while ((scn = elf_nextscn(elf, scn)) != NULL) {
3328                 sh = elf_sec_hdr(obj, scn);
3329                 if (!sh)
3330                         return -LIBBPF_ERRNO__FORMAT;
3331
3332                 if (sh->sh_type == SHT_SYMTAB) {
3333                         if (obj->efile.symbols) {
3334                                 pr_warn("elf: multiple symbol tables in %s\n", obj->path);
3335                                 return -LIBBPF_ERRNO__FORMAT;
3336                         }
3337
3338                         data = elf_sec_data(obj, scn);
3339                         if (!data)
3340                                 return -LIBBPF_ERRNO__FORMAT;
3341
3342                         idx = elf_ndxscn(scn);
3343
3344                         obj->efile.symbols = data;
3345                         obj->efile.symbols_shndx = idx;
3346                         obj->efile.strtabidx = sh->sh_link;
3347                 }
3348         }
3349
3350         if (!obj->efile.symbols) {
3351                 pr_warn("elf: couldn't find symbol table in %s, stripped object file?\n",
3352                         obj->path);
3353                 return -ENOENT;
3354         }
3355
3356         scn = NULL;
3357         while ((scn = elf_nextscn(elf, scn)) != NULL) {
3358                 idx = elf_ndxscn(scn);
3359                 sec_desc = &obj->efile.secs[idx];
3360
3361                 sh = elf_sec_hdr(obj, scn);
3362                 if (!sh)
3363                         return -LIBBPF_ERRNO__FORMAT;
3364
3365                 name = elf_sec_str(obj, sh->sh_name);
3366                 if (!name)
3367                         return -LIBBPF_ERRNO__FORMAT;
3368
3369                 if (ignore_elf_section(sh, name))
3370                         continue;
3371
3372                 data = elf_sec_data(obj, scn);
3373                 if (!data)
3374                         return -LIBBPF_ERRNO__FORMAT;
3375
3376                 pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
3377                          idx, name, (unsigned long)data->d_size,
3378                          (int)sh->sh_link, (unsigned long)sh->sh_flags,
3379                          (int)sh->sh_type);
3380
3381                 if (strcmp(name, "license") == 0) {
3382                         err = bpf_object__init_license(obj, data->d_buf, data->d_size);
3383                         if (err)
3384                                 return err;
3385                 } else if (strcmp(name, "version") == 0) {
3386                         err = bpf_object__init_kversion(obj, data->d_buf, data->d_size);
3387                         if (err)
3388                                 return err;
3389                 } else if (strcmp(name, "maps") == 0) {
3390                         pr_warn("elf: legacy map definitions in 'maps' section are not supported by libbpf v1.0+\n");
3391                         return -ENOTSUP;
3392                 } else if (strcmp(name, MAPS_ELF_SEC) == 0) {
3393                         obj->efile.btf_maps_shndx = idx;
3394                 } else if (strcmp(name, BTF_ELF_SEC) == 0) {
3395                         if (sh->sh_type != SHT_PROGBITS)
3396                                 return -LIBBPF_ERRNO__FORMAT;
3397                         btf_data = data;
3398                 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) {
3399                         if (sh->sh_type != SHT_PROGBITS)
3400                                 return -LIBBPF_ERRNO__FORMAT;
3401                         btf_ext_data = data;
3402                 } else if (sh->sh_type == SHT_SYMTAB) {
3403                         /* already processed during the first pass above */
3404                 } else if (sh->sh_type == SHT_PROGBITS && data->d_size > 0) {
3405                         if (sh->sh_flags & SHF_EXECINSTR) {
3406                                 if (strcmp(name, ".text") == 0)
3407                                         obj->efile.text_shndx = idx;
3408                                 err = bpf_object__add_programs(obj, data, name, idx);
3409                                 if (err)
3410                                         return err;
3411                         } else if (strcmp(name, DATA_SEC) == 0 ||
3412                                    str_has_pfx(name, DATA_SEC ".")) {
3413                                 sec_desc->sec_type = SEC_DATA;
3414                                 sec_desc->shdr = sh;
3415                                 sec_desc->data = data;
3416                         } else if (strcmp(name, RODATA_SEC) == 0 ||
3417                                    str_has_pfx(name, RODATA_SEC ".")) {
3418                                 sec_desc->sec_type = SEC_RODATA;
3419                                 sec_desc->shdr = sh;
3420                                 sec_desc->data = data;
3421                         } else if (strcmp(name, STRUCT_OPS_SEC) == 0) {
3422                                 obj->efile.st_ops_data = data;
3423                                 obj->efile.st_ops_shndx = idx;
3424                         } else {
3425                                 pr_info("elf: skipping unrecognized data section(%d) %s\n",
3426                                         idx, name);
3427                         }
3428                 } else if (sh->sh_type == SHT_REL) {
3429                         int targ_sec_idx = sh->sh_info; /* points to other section */
3430
3431                         if (sh->sh_entsize != sizeof(Elf64_Rel) ||
3432                             targ_sec_idx >= obj->efile.sec_cnt)
3433                                 return -LIBBPF_ERRNO__FORMAT;
3434
3435                         /* Only do relo for section with exec instructions */
3436                         if (!section_have_execinstr(obj, targ_sec_idx) &&
3437                             strcmp(name, ".rel" STRUCT_OPS_SEC) &&
3438                             strcmp(name, ".rel" MAPS_ELF_SEC)) {
3439                                 pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n",
3440                                         idx, name, targ_sec_idx,
3441                                         elf_sec_name(obj, elf_sec_by_idx(obj, targ_sec_idx)) ?: "<?>");
3442                                 continue;
3443                         }
3444
3445                         sec_desc->sec_type = SEC_RELO;
3446                         sec_desc->shdr = sh;
3447                         sec_desc->data = data;
3448                 } else if (sh->sh_type == SHT_NOBITS && strcmp(name, BSS_SEC) == 0) {
3449                         sec_desc->sec_type = SEC_BSS;
3450                         sec_desc->shdr = sh;
3451                         sec_desc->data = data;
3452                 } else {
3453                         pr_info("elf: skipping section(%d) %s (size %zu)\n", idx, name,
3454                                 (size_t)sh->sh_size);
3455                 }
3456         }
3457
3458         if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {
3459                 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path);
3460                 return -LIBBPF_ERRNO__FORMAT;
3461         }
3462
3463         /* sort BPF programs by section name and in-section instruction offset
3464          * for faster search */
3465         if (obj->nr_programs)
3466                 qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs);
3467
3468         return bpf_object__init_btf(obj, btf_data, btf_ext_data);
3469 }
3470
3471 static bool sym_is_extern(const Elf64_Sym *sym)
3472 {
3473         int bind = ELF64_ST_BIND(sym->st_info);
3474         /* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */
3475         return sym->st_shndx == SHN_UNDEF &&
3476                (bind == STB_GLOBAL || bind == STB_WEAK) &&
3477                ELF64_ST_TYPE(sym->st_info) == STT_NOTYPE;
3478 }
3479
3480 static bool sym_is_subprog(const Elf64_Sym *sym, int text_shndx)
3481 {
3482         int bind = ELF64_ST_BIND(sym->st_info);
3483         int type = ELF64_ST_TYPE(sym->st_info);
3484
3485         /* in .text section */
3486         if (sym->st_shndx != text_shndx)
3487                 return false;
3488
3489         /* local function */
3490         if (bind == STB_LOCAL && type == STT_SECTION)
3491                 return true;
3492
3493         /* global function */
3494         return bind == STB_GLOBAL && type == STT_FUNC;
3495 }
3496
3497 static int find_extern_btf_id(const struct btf *btf, const char *ext_name)
3498 {
3499         const struct btf_type *t;
3500         const char *tname;
3501         int i, n;
3502
3503         if (!btf)
3504                 return -ESRCH;
3505
3506         n = btf__type_cnt(btf);
3507         for (i = 1; i < n; i++) {
3508                 t = btf__type_by_id(btf, i);
3509
3510                 if (!btf_is_var(t) && !btf_is_func(t))
3511                         continue;
3512
3513                 tname = btf__name_by_offset(btf, t->name_off);
3514                 if (strcmp(tname, ext_name))
3515                         continue;
3516
3517                 if (btf_is_var(t) &&
3518                     btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
3519                         return -EINVAL;
3520
3521                 if (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_EXTERN)
3522                         return -EINVAL;
3523
3524                 return i;
3525         }
3526
3527         return -ENOENT;
3528 }
3529
3530 static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) {
3531         const struct btf_var_secinfo *vs;
3532         const struct btf_type *t;
3533         int i, j, n;
3534
3535         if (!btf)
3536                 return -ESRCH;
3537
3538         n = btf__type_cnt(btf);
3539         for (i = 1; i < n; i++) {
3540                 t = btf__type_by_id(btf, i);
3541
3542                 if (!btf_is_datasec(t))
3543                         continue;
3544
3545                 vs = btf_var_secinfos(t);
3546                 for (j = 0; j < btf_vlen(t); j++, vs++) {
3547                         if (vs->type == ext_btf_id)
3548                                 return i;
3549                 }
3550         }
3551
3552         return -ENOENT;
3553 }
3554
3555 static enum kcfg_type find_kcfg_type(const struct btf *btf, int id,
3556                                      bool *is_signed)
3557 {
3558         const struct btf_type *t;
3559         const char *name;
3560
3561         t = skip_mods_and_typedefs(btf, id, NULL);
3562         name = btf__name_by_offset(btf, t->name_off);
3563
3564         if (is_signed)
3565                 *is_signed = false;
3566         switch (btf_kind(t)) {
3567         case BTF_KIND_INT: {
3568                 int enc = btf_int_encoding(t);
3569
3570                 if (enc & BTF_INT_BOOL)
3571                         return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN;
3572                 if (is_signed)
3573                         *is_signed = enc & BTF_INT_SIGNED;
3574                 if (t->size == 1)
3575                         return KCFG_CHAR;
3576                 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1)))
3577                         return KCFG_UNKNOWN;
3578                 return KCFG_INT;
3579         }
3580         case BTF_KIND_ENUM:
3581                 if (t->size != 4)
3582                         return KCFG_UNKNOWN;
3583                 if (strcmp(name, "libbpf_tristate"))
3584                         return KCFG_UNKNOWN;
3585                 return KCFG_TRISTATE;
3586         case BTF_KIND_ENUM64:
3587                 if (strcmp(name, "libbpf_tristate"))
3588                         return KCFG_UNKNOWN;
3589                 return KCFG_TRISTATE;
3590         case BTF_KIND_ARRAY:
3591                 if (btf_array(t)->nelems == 0)
3592                         return KCFG_UNKNOWN;
3593                 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR)
3594                         return KCFG_UNKNOWN;
3595                 return KCFG_CHAR_ARR;
3596         default:
3597                 return KCFG_UNKNOWN;
3598         }
3599 }
3600
3601 static int cmp_externs(const void *_a, const void *_b)
3602 {
3603         const struct extern_desc *a = _a;
3604         const struct extern_desc *b = _b;
3605
3606         if (a->type != b->type)
3607                 return a->type < b->type ? -1 : 1;
3608
3609         if (a->type == EXT_KCFG) {
3610                 /* descending order by alignment requirements */
3611                 if (a->kcfg.align != b->kcfg.align)
3612                         return a->kcfg.align > b->kcfg.align ? -1 : 1;
3613                 /* ascending order by size, within same alignment class */
3614                 if (a->kcfg.sz != b->kcfg.sz)
3615                         return a->kcfg.sz < b->kcfg.sz ? -1 : 1;
3616         }
3617
3618         /* resolve ties by name */
3619         return strcmp(a->name, b->name);
3620 }
3621
3622 static int find_int_btf_id(const struct btf *btf)
3623 {
3624         const struct btf_type *t;
3625         int i, n;
3626
3627         n = btf__type_cnt(btf);
3628         for (i = 1; i < n; i++) {
3629                 t = btf__type_by_id(btf, i);
3630
3631                 if (btf_is_int(t) && btf_int_bits(t) == 32)
3632                         return i;
3633         }
3634
3635         return 0;
3636 }
3637
3638 static int add_dummy_ksym_var(struct btf *btf)
3639 {
3640         int i, int_btf_id, sec_btf_id, dummy_var_btf_id;
3641         const struct btf_var_secinfo *vs;
3642         const struct btf_type *sec;
3643
3644         if (!btf)
3645                 return 0;
3646
3647         sec_btf_id = btf__find_by_name_kind(btf, KSYMS_SEC,
3648                                             BTF_KIND_DATASEC);
3649         if (sec_btf_id < 0)
3650                 return 0;
3651
3652         sec = btf__type_by_id(btf, sec_btf_id);
3653         vs = btf_var_secinfos(sec);
3654         for (i = 0; i < btf_vlen(sec); i++, vs++) {
3655                 const struct btf_type *vt;
3656
3657                 vt = btf__type_by_id(btf, vs->type);
3658                 if (btf_is_func(vt))
3659                         break;
3660         }
3661
3662         /* No func in ksyms sec.  No need to add dummy var. */
3663         if (i == btf_vlen(sec))
3664                 return 0;
3665
3666         int_btf_id = find_int_btf_id(btf);
3667         dummy_var_btf_id = btf__add_var(btf,
3668                                         "dummy_ksym",
3669                                         BTF_VAR_GLOBAL_ALLOCATED,
3670                                         int_btf_id);
3671         if (dummy_var_btf_id < 0)
3672                 pr_warn("cannot create a dummy_ksym var\n");
3673
3674         return dummy_var_btf_id;
3675 }
3676
3677 static int bpf_object__collect_externs(struct bpf_object *obj)
3678 {
3679         struct btf_type *sec, *kcfg_sec = NULL, *ksym_sec = NULL;
3680         const struct btf_type *t;
3681         struct extern_desc *ext;
3682         int i, n, off, dummy_var_btf_id;
3683         const char *ext_name, *sec_name;
3684         Elf_Scn *scn;
3685         Elf64_Shdr *sh;
3686
3687         if (!obj->efile.symbols)
3688                 return 0;
3689
3690         scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx);
3691         sh = elf_sec_hdr(obj, scn);
3692         if (!sh || sh->sh_entsize != sizeof(Elf64_Sym))
3693                 return -LIBBPF_ERRNO__FORMAT;
3694
3695         dummy_var_btf_id = add_dummy_ksym_var(obj->btf);
3696         if (dummy_var_btf_id < 0)
3697                 return dummy_var_btf_id;
3698
3699         n = sh->sh_size / sh->sh_entsize;
3700         pr_debug("looking for externs among %d symbols...\n", n);
3701
3702         for (i = 0; i < n; i++) {
3703                 Elf64_Sym *sym = elf_sym_by_idx(obj, i);
3704
3705                 if (!sym)
3706                         return -LIBBPF_ERRNO__FORMAT;
3707                 if (!sym_is_extern(sym))
3708                         continue;
3709                 ext_name = elf_sym_str(obj, sym->st_name);
3710                 if (!ext_name || !ext_name[0])
3711                         continue;
3712
3713                 ext = obj->externs;
3714                 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext));
3715                 if (!ext)
3716                         return -ENOMEM;
3717                 obj->externs = ext;
3718                 ext = &ext[obj->nr_extern];
3719                 memset(ext, 0, sizeof(*ext));
3720                 obj->nr_extern++;
3721
3722                 ext->btf_id = find_extern_btf_id(obj->btf, ext_name);
3723                 if (ext->btf_id <= 0) {
3724                         pr_warn("failed to find BTF for extern '%s': %d\n",
3725                                 ext_name, ext->btf_id);
3726                         return ext->btf_id;
3727                 }
3728                 t = btf__type_by_id(obj->btf, ext->btf_id);
3729                 ext->name = btf__name_by_offset(obj->btf, t->name_off);
3730                 ext->sym_idx = i;
3731                 ext->is_weak = ELF64_ST_BIND(sym->st_info) == STB_WEAK;
3732
3733                 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id);
3734                 if (ext->sec_btf_id <= 0) {
3735                         pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n",
3736                                 ext_name, ext->btf_id, ext->sec_btf_id);
3737                         return ext->sec_btf_id;
3738                 }
3739                 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id);
3740                 sec_name = btf__name_by_offset(obj->btf, sec->name_off);
3741
3742                 if (strcmp(sec_name, KCONFIG_SEC) == 0) {
3743                         if (btf_is_func(t)) {
3744                                 pr_warn("extern function %s is unsupported under %s section\n",
3745                                         ext->name, KCONFIG_SEC);
3746                                 return -ENOTSUP;
3747                         }
3748                         kcfg_sec = sec;
3749                         ext->type = EXT_KCFG;
3750                         ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);
3751                         if (ext->kcfg.sz <= 0) {
3752                                 pr_warn("failed to resolve size of extern (kcfg) '%s': %d\n",
3753                                         ext_name, ext->kcfg.sz);
3754                                 return ext->kcfg.sz;
3755                         }
3756                         ext->kcfg.align = btf__align_of(obj->btf, t->type);
3757                         if (ext->kcfg.align <= 0) {
3758                                 pr_warn("failed to determine alignment of extern (kcfg) '%s': %d\n",
3759                                         ext_name, ext->kcfg.align);
3760                                 return -EINVAL;
3761                         }
3762                         ext->kcfg.type = find_kcfg_type(obj->btf, t->type,
3763                                                         &ext->kcfg.is_signed);
3764                         if (ext->kcfg.type == KCFG_UNKNOWN) {
3765                                 pr_warn("extern (kcfg) '%s': type is unsupported\n", ext_name);
3766                                 return -ENOTSUP;
3767                         }
3768                 } else if (strcmp(sec_name, KSYMS_SEC) == 0) {
3769                         ksym_sec = sec;
3770                         ext->type = EXT_KSYM;
3771                         skip_mods_and_typedefs(obj->btf, t->type,
3772                                                &ext->ksym.type_id);
3773                 } else {
3774                         pr_warn("unrecognized extern section '%s'\n", sec_name);
3775                         return -ENOTSUP;
3776                 }
3777         }
3778         pr_debug("collected %d externs total\n", obj->nr_extern);
3779
3780         if (!obj->nr_extern)
3781                 return 0;
3782
3783         /* sort externs by type, for kcfg ones also by (align, size, name) */
3784         qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs);
3785
3786         /* for .ksyms section, we need to turn all externs into allocated
3787          * variables in BTF to pass kernel verification; we do this by
3788          * pretending that each extern is a 8-byte variable
3789          */
3790         if (ksym_sec) {
3791                 /* find existing 4-byte integer type in BTF to use for fake
3792                  * extern variables in DATASEC
3793                  */
3794                 int int_btf_id = find_int_btf_id(obj->btf);
3795                 /* For extern function, a dummy_var added earlier
3796                  * will be used to replace the vs->type and
3797                  * its name string will be used to refill
3798                  * the missing param's name.
3799                  */
3800                 const struct btf_type *dummy_var;
3801
3802                 dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id);
3803                 for (i = 0; i < obj->nr_extern; i++) {
3804                         ext = &obj->externs[i];
3805                         if (ext->type != EXT_KSYM)
3806                                 continue;
3807                         pr_debug("extern (ksym) #%d: symbol %d, name %s\n",
3808                                  i, ext->sym_idx, ext->name);
3809                 }
3810
3811                 sec = ksym_sec;
3812                 n = btf_vlen(sec);
3813                 for (i = 0, off = 0; i < n; i++, off += sizeof(int)) {
3814                         struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
3815                         struct btf_type *vt;
3816
3817                         vt = (void *)btf__type_by_id(obj->btf, vs->type);
3818                         ext_name = btf__name_by_offset(obj->btf, vt->name_off);
3819                         ext = find_extern_by_name(obj, ext_name);
3820                         if (!ext) {
3821                                 pr_warn("failed to find extern definition for BTF %s '%s'\n",
3822                                         btf_kind_str(vt), ext_name);
3823                                 return -ESRCH;
3824                         }
3825                         if (btf_is_func(vt)) {
3826                                 const struct btf_type *func_proto;
3827                                 struct btf_param *param;
3828                                 int j;
3829
3830                                 func_proto = btf__type_by_id(obj->btf,
3831                                                              vt->type);
3832                                 param = btf_params(func_proto);
3833                                 /* Reuse the dummy_var string if the
3834                                  * func proto does not have param name.
3835                                  */
3836                                 for (j = 0; j < btf_vlen(func_proto); j++)
3837                                         if (param[j].type && !param[j].name_off)
3838                                                 param[j].name_off =
3839                                                         dummy_var->name_off;
3840                                 vs->type = dummy_var_btf_id;
3841                                 vt->info &= ~0xffff;
3842                                 vt->info |= BTF_FUNC_GLOBAL;
3843                         } else {
3844                                 btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
3845                                 vt->type = int_btf_id;
3846                         }
3847                         vs->offset = off;
3848                         vs->size = sizeof(int);
3849                 }
3850                 sec->size = off;
3851         }
3852
3853         if (kcfg_sec) {
3854                 sec = kcfg_sec;
3855                 /* for kcfg externs calculate their offsets within a .kconfig map */
3856                 off = 0;
3857                 for (i = 0; i < obj->nr_extern; i++) {
3858                         ext = &obj->externs[i];
3859                         if (ext->type != EXT_KCFG)
3860                                 continue;
3861
3862                         ext->kcfg.data_off = roundup(off, ext->kcfg.align);
3863                         off = ext->kcfg.data_off + ext->kcfg.sz;
3864                         pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n",
3865                                  i, ext->sym_idx, ext->kcfg.data_off, ext->name);
3866                 }
3867                 sec->size = off;
3868                 n = btf_vlen(sec);
3869                 for (i = 0; i < n; i++) {
3870                         struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
3871
3872                         t = btf__type_by_id(obj->btf, vs->type);
3873                         ext_name = btf__name_by_offset(obj->btf, t->name_off);
3874                         ext = find_extern_by_name(obj, ext_name);
3875                         if (!ext) {
3876                                 pr_warn("failed to find extern definition for BTF var '%s'\n",
3877                                         ext_name);
3878                                 return -ESRCH;
3879                         }
3880                         btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
3881                         vs->offset = ext->kcfg.data_off;
3882                 }
3883         }
3884         return 0;
3885 }
3886
3887 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog)
3888 {
3889         return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1;
3890 }
3891
3892 struct bpf_program *
3893 bpf_object__find_program_by_name(const struct bpf_object *obj,
3894                                  const char *name)
3895 {
3896         struct bpf_program *prog;
3897
3898         bpf_object__for_each_program(prog, obj) {
3899                 if (prog_is_subprog(obj, prog))
3900                         continue;
3901                 if (!strcmp(prog->name, name))
3902                         return prog;
3903         }
3904         return errno = ENOENT, NULL;
3905 }
3906
3907 static bool bpf_object__shndx_is_data(const struct bpf_object *obj,
3908                                       int shndx)
3909 {
3910         switch (obj->efile.secs[shndx].sec_type) {
3911         case SEC_BSS:
3912         case SEC_DATA:
3913         case SEC_RODATA:
3914                 return true;
3915         default:
3916                 return false;
3917         }
3918 }
3919
3920 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj,
3921                                       int shndx)
3922 {
3923         return shndx == obj->efile.btf_maps_shndx;
3924 }
3925
3926 static enum libbpf_map_type
3927 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx)
3928 {
3929         if (shndx == obj->efile.symbols_shndx)
3930                 return LIBBPF_MAP_KCONFIG;
3931
3932         switch (obj->efile.secs[shndx].sec_type) {
3933         case SEC_BSS:
3934                 return LIBBPF_MAP_BSS;
3935         case SEC_DATA:
3936                 return LIBBPF_MAP_DATA;
3937         case SEC_RODATA:
3938                 return LIBBPF_MAP_RODATA;
3939         default:
3940                 return LIBBPF_MAP_UNSPEC;
3941         }
3942 }
3943
3944 static int bpf_program__record_reloc(struct bpf_program *prog,
3945                                      struct reloc_desc *reloc_desc,
3946                                      __u32 insn_idx, const char *sym_name,
3947                                      const Elf64_Sym *sym, const Elf64_Rel *rel)
3948 {
3949         struct bpf_insn *insn = &prog->insns[insn_idx];
3950         size_t map_idx, nr_maps = prog->obj->nr_maps;
3951         struct bpf_object *obj = prog->obj;
3952         __u32 shdr_idx = sym->st_shndx;
3953         enum libbpf_map_type type;
3954         const char *sym_sec_name;
3955         struct bpf_map *map;
3956
3957         if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) {
3958                 pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n",
3959                         prog->name, sym_name, insn_idx, insn->code);
3960                 return -LIBBPF_ERRNO__RELOC;
3961         }
3962
3963         if (sym_is_extern(sym)) {
3964                 int sym_idx = ELF64_R_SYM(rel->r_info);
3965                 int i, n = obj->nr_extern;
3966                 struct extern_desc *ext;
3967
3968                 for (i = 0; i < n; i++) {
3969                         ext = &obj->externs[i];
3970                         if (ext->sym_idx == sym_idx)
3971                                 break;
3972                 }
3973                 if (i >= n) {
3974                         pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n",
3975                                 prog->name, sym_name, sym_idx);
3976                         return -LIBBPF_ERRNO__RELOC;
3977                 }
3978                 pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n",
3979                          prog->name, i, ext->name, ext->sym_idx, insn_idx);
3980                 if (insn->code == (BPF_JMP | BPF_CALL))
3981                         reloc_desc->type = RELO_EXTERN_FUNC;
3982                 else
3983                         reloc_desc->type = RELO_EXTERN_VAR;
3984                 reloc_desc->insn_idx = insn_idx;
3985                 reloc_desc->sym_off = i; /* sym_off stores extern index */
3986                 return 0;
3987         }
3988
3989         /* sub-program call relocation */
3990         if (is_call_insn(insn)) {
3991                 if (insn->src_reg != BPF_PSEUDO_CALL) {
3992                         pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
3993                         return -LIBBPF_ERRNO__RELOC;
3994                 }
3995                 /* text_shndx can be 0, if no default "main" program exists */
3996                 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {
3997                         sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
3998                         pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n",
3999                                 prog->name, sym_name, sym_sec_name);
4000                         return -LIBBPF_ERRNO__RELOC;
4001                 }
4002                 if (sym->st_value % BPF_INSN_SZ) {
4003                         pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n",
4004                                 prog->name, sym_name, (size_t)sym->st_value);
4005                         return -LIBBPF_ERRNO__RELOC;
4006                 }
4007                 reloc_desc->type = RELO_CALL;
4008                 reloc_desc->insn_idx = insn_idx;
4009                 reloc_desc->sym_off = sym->st_value;
4010                 return 0;
4011         }
4012
4013         if (!shdr_idx || shdr_idx >= SHN_LORESERVE) {
4014                 pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n",
4015                         prog->name, sym_name, shdr_idx);
4016                 return -LIBBPF_ERRNO__RELOC;
4017         }
4018
4019         /* loading subprog addresses */
4020         if (sym_is_subprog(sym, obj->efile.text_shndx)) {
4021                 /* global_func: sym->st_value = offset in the section, insn->imm = 0.
4022                  * local_func: sym->st_value = 0, insn->imm = offset in the section.
4023                  */
4024                 if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) {
4025                         pr_warn("prog '%s': bad subprog addr relo against '%s' at offset %zu+%d\n",
4026                                 prog->name, sym_name, (size_t)sym->st_value, insn->imm);
4027                         return -LIBBPF_ERRNO__RELOC;
4028                 }
4029
4030                 reloc_desc->type = RELO_SUBPROG_ADDR;
4031                 reloc_desc->insn_idx = insn_idx;
4032                 reloc_desc->sym_off = sym->st_value;
4033                 return 0;
4034         }
4035
4036         type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);
4037         sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
4038
4039         /* generic map reference relocation */
4040         if (type == LIBBPF_MAP_UNSPEC) {
4041                 if (!bpf_object__shndx_is_maps(obj, shdr_idx)) {
4042                         pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n",
4043                                 prog->name, sym_name, sym_sec_name);
4044                         return -LIBBPF_ERRNO__RELOC;
4045                 }
4046                 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
4047                         map = &obj->maps[map_idx];
4048                         if (map->libbpf_type != type ||
4049                             map->sec_idx != sym->st_shndx ||
4050                             map->sec_offset != sym->st_value)
4051                                 continue;
4052                         pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n",
4053                                  prog->name, map_idx, map->name, map->sec_idx,
4054                                  map->sec_offset, insn_idx);
4055                         break;
4056                 }
4057                 if (map_idx >= nr_maps) {
4058                         pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n",
4059                                 prog->name, sym_sec_name, (size_t)sym->st_value);
4060                         return -LIBBPF_ERRNO__RELOC;
4061                 }
4062                 reloc_desc->type = RELO_LD64;
4063                 reloc_desc->insn_idx = insn_idx;
4064                 reloc_desc->map_idx = map_idx;
4065                 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */
4066                 return 0;
4067         }
4068
4069         /* global data map relocation */
4070         if (!bpf_object__shndx_is_data(obj, shdr_idx)) {
4071                 pr_warn("prog '%s': bad data relo against section '%s'\n",
4072                         prog->name, sym_sec_name);
4073                 return -LIBBPF_ERRNO__RELOC;
4074         }
4075         for (map_idx = 0; map_idx < nr_maps; map_idx++) {
4076                 map = &obj->maps[map_idx];
4077                 if (map->libbpf_type != type || map->sec_idx != sym->st_shndx)
4078                         continue;
4079                 pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n",
4080                          prog->name, map_idx, map->name, map->sec_idx,
4081                          map->sec_offset, insn_idx);
4082                 break;
4083         }
4084         if (map_idx >= nr_maps) {
4085                 pr_warn("prog '%s': data relo failed to find map for section '%s'\n",
4086                         prog->name, sym_sec_name);
4087                 return -LIBBPF_ERRNO__RELOC;
4088         }
4089
4090         reloc_desc->type = RELO_DATA;
4091         reloc_desc->insn_idx = insn_idx;
4092         reloc_desc->map_idx = map_idx;
4093         reloc_desc->sym_off = sym->st_value;
4094         return 0;
4095 }
4096
4097 static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx)
4098 {
4099         return insn_idx >= prog->sec_insn_off &&
4100                insn_idx < prog->sec_insn_off + prog->sec_insn_cnt;
4101 }
4102
4103 static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj,
4104                                                  size_t sec_idx, size_t insn_idx)
4105 {
4106         int l = 0, r = obj->nr_programs - 1, m;
4107         struct bpf_program *prog;
4108
4109         while (l < r) {
4110                 m = l + (r - l + 1) / 2;
4111                 prog = &obj->programs[m];
4112
4113                 if (prog->sec_idx < sec_idx ||
4114                     (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx))
4115                         l = m;
4116                 else
4117                         r = m - 1;
4118         }
4119         /* matching program could be at index l, but it still might be the
4120          * wrong one, so we need to double check conditions for the last time
4121          */
4122         prog = &obj->programs[l];
4123         if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx))
4124                 return prog;
4125         return NULL;
4126 }
4127
4128 static int
4129 bpf_object__collect_prog_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Data *data)
4130 {
4131         const char *relo_sec_name, *sec_name;
4132         size_t sec_idx = shdr->sh_info, sym_idx;
4133         struct bpf_program *prog;
4134         struct reloc_desc *relos;
4135         int err, i, nrels;
4136         const char *sym_name;
4137         __u32 insn_idx;
4138         Elf_Scn *scn;
4139         Elf_Data *scn_data;
4140         Elf64_Sym *sym;
4141         Elf64_Rel *rel;
4142
4143         if (sec_idx >= obj->efile.sec_cnt)
4144                 return -EINVAL;
4145
4146         scn = elf_sec_by_idx(obj, sec_idx);
4147         scn_data = elf_sec_data(obj, scn);
4148
4149         relo_sec_name = elf_sec_str(obj, shdr->sh_name);
4150         sec_name = elf_sec_name(obj, scn);
4151         if (!relo_sec_name || !sec_name)
4152                 return -EINVAL;
4153
4154         pr_debug("sec '%s': collecting relocation for section(%zu) '%s'\n",
4155                  relo_sec_name, sec_idx, sec_name);
4156         nrels = shdr->sh_size / shdr->sh_entsize;
4157
4158         for (i = 0; i < nrels; i++) {
4159                 rel = elf_rel_by_idx(data, i);
4160                 if (!rel) {
4161                         pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i);
4162                         return -LIBBPF_ERRNO__FORMAT;
4163                 }
4164
4165                 sym_idx = ELF64_R_SYM(rel->r_info);
4166                 sym = elf_sym_by_idx(obj, sym_idx);
4167                 if (!sym) {
4168                         pr_warn("sec '%s': symbol #%zu not found for relo #%d\n",
4169                                 relo_sec_name, sym_idx, i);
4170                         return -LIBBPF_ERRNO__FORMAT;
4171                 }
4172
4173                 if (sym->st_shndx >= obj->efile.sec_cnt) {
4174                         pr_warn("sec '%s': corrupted symbol #%zu pointing to invalid section #%zu for relo #%d\n",
4175                                 relo_sec_name, sym_idx, (size_t)sym->st_shndx, i);
4176                         return -LIBBPF_ERRNO__FORMAT;
4177                 }
4178
4179                 if (rel->r_offset % BPF_INSN_SZ || rel->r_offset >= scn_data->d_size) {
4180                         pr_warn("sec '%s': invalid offset 0x%zx for relo #%d\n",
4181                                 relo_sec_name, (size_t)rel->r_offset, i);
4182                         return -LIBBPF_ERRNO__FORMAT;
4183                 }
4184
4185                 insn_idx = rel->r_offset / BPF_INSN_SZ;
4186                 /* relocations against static functions are recorded as
4187                  * relocations against the section that contains a function;
4188                  * in such case, symbol will be STT_SECTION and sym.st_name
4189                  * will point to empty string (0), so fetch section name
4190                  * instead
4191                  */
4192                 if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_name == 0)
4193                         sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx));
4194                 else
4195                         sym_name = elf_sym_str(obj, sym->st_name);
4196                 sym_name = sym_name ?: "<?";
4197
4198                 pr_debug("sec '%s': relo #%d: insn #%u against '%s'\n",
4199                          relo_sec_name, i, insn_idx, sym_name);
4200
4201                 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
4202                 if (!prog) {
4203                         pr_debug("sec '%s': relo #%d: couldn't find program in section '%s' for insn #%u, probably overridden weak function, skipping...\n",
4204                                 relo_sec_name, i, sec_name, insn_idx);
4205                         continue;
4206                 }
4207
4208                 relos = libbpf_reallocarray(prog->reloc_desc,
4209                                             prog->nr_reloc + 1, sizeof(*relos));
4210                 if (!relos)
4211                         return -ENOMEM;
4212                 prog->reloc_desc = relos;
4213
4214                 /* adjust insn_idx to local BPF program frame of reference */
4215                 insn_idx -= prog->sec_insn_off;
4216                 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc],
4217                                                 insn_idx, sym_name, sym, rel);
4218                 if (err)
4219                         return err;
4220
4221                 prog->nr_reloc++;
4222         }
4223         return 0;
4224 }
4225
4226 static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map)
4227 {
4228         int id;
4229
4230         if (!obj->btf)
4231                 return -ENOENT;
4232
4233         /* if it's BTF-defined map, we don't need to search for type IDs.
4234          * For struct_ops map, it does not need btf_key_type_id and
4235          * btf_value_type_id.
4236          */
4237         if (map->sec_idx == obj->efile.btf_maps_shndx || bpf_map__is_struct_ops(map))
4238                 return 0;
4239
4240         /*
4241          * LLVM annotates global data differently in BTF, that is,
4242          * only as '.data', '.bss' or '.rodata'.
4243          */
4244         if (!bpf_map__is_internal(map))
4245                 return -ENOENT;
4246
4247         id = btf__find_by_name(obj->btf, map->real_name);
4248         if (id < 0)
4249                 return id;
4250
4251         map->btf_key_type_id = 0;
4252         map->btf_value_type_id = id;
4253         return 0;
4254 }
4255
4256 static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info)
4257 {
4258         char file[PATH_MAX], buff[4096];
4259         FILE *fp;
4260         __u32 val;
4261         int err;
4262
4263         snprintf(file, sizeof(file), "/proc/%d/fdinfo/%d", getpid(), fd);
4264         memset(info, 0, sizeof(*info));
4265
4266         fp = fopen(file, "r");
4267         if (!fp) {
4268                 err = -errno;
4269                 pr_warn("failed to open %s: %d. No procfs support?\n", file,
4270                         err);
4271                 return err;
4272         }
4273
4274         while (fgets(buff, sizeof(buff), fp)) {
4275                 if (sscanf(buff, "map_type:\t%u", &val) == 1)
4276                         info->type = val;
4277                 else if (sscanf(buff, "key_size:\t%u", &val) == 1)
4278                         info->key_size = val;
4279                 else if (sscanf(buff, "value_size:\t%u", &val) == 1)
4280                         info->value_size = val;
4281                 else if (sscanf(buff, "max_entries:\t%u", &val) == 1)
4282                         info->max_entries = val;
4283                 else if (sscanf(buff, "map_flags:\t%i", &val) == 1)
4284                         info->map_flags = val;
4285         }
4286
4287         fclose(fp);
4288
4289         return 0;
4290 }
4291
4292 bool bpf_map__autocreate(const struct bpf_map *map)
4293 {
4294         return map->autocreate;
4295 }
4296
4297 int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate)
4298 {
4299         if (map->obj->loaded)
4300                 return libbpf_err(-EBUSY);
4301
4302         map->autocreate = autocreate;
4303         return 0;
4304 }
4305
4306 int bpf_map__reuse_fd(struct bpf_map *map, int fd)
4307 {
4308         struct bpf_map_info info;
4309         __u32 len = sizeof(info), name_len;
4310         int new_fd, err;
4311         char *new_name;
4312
4313         memset(&info, 0, len);
4314         err = bpf_obj_get_info_by_fd(fd, &info, &len);
4315         if (err && errno == EINVAL)
4316                 err = bpf_get_map_info_from_fdinfo(fd, &info);
4317         if (err)
4318                 return libbpf_err(err);
4319
4320         name_len = strlen(info.name);
4321         if (name_len == BPF_OBJ_NAME_LEN - 1 && strncmp(map->name, info.name, name_len) == 0)
4322                 new_name = strdup(map->name);
4323         else
4324                 new_name = strdup(info.name);
4325
4326         if (!new_name)
4327                 return libbpf_err(-errno);
4328
4329         new_fd = open("/", O_RDONLY | O_CLOEXEC);
4330         if (new_fd < 0) {
4331                 err = -errno;
4332                 goto err_free_new_name;
4333         }
4334
4335         new_fd = dup3(fd, new_fd, O_CLOEXEC);
4336         if (new_fd < 0) {
4337                 err = -errno;
4338                 goto err_close_new_fd;
4339         }
4340
4341         err = zclose(map->fd);
4342         if (err) {
4343                 err = -errno;
4344                 goto err_close_new_fd;
4345         }
4346         free(map->name);
4347
4348         map->fd = new_fd;
4349         map->name = new_name;
4350         map->def.type = info.type;
4351         map->def.key_size = info.key_size;
4352         map->def.value_size = info.value_size;
4353         map->def.max_entries = info.max_entries;
4354         map->def.map_flags = info.map_flags;
4355         map->btf_key_type_id = info.btf_key_type_id;
4356         map->btf_value_type_id = info.btf_value_type_id;
4357         map->reused = true;
4358         map->map_extra = info.map_extra;
4359
4360         return 0;
4361
4362 err_close_new_fd:
4363         close(new_fd);
4364 err_free_new_name:
4365         free(new_name);
4366         return libbpf_err(err);
4367 }
4368
4369 __u32 bpf_map__max_entries(const struct bpf_map *map)
4370 {
4371         return map->def.max_entries;
4372 }
4373
4374 struct bpf_map *bpf_map__inner_map(struct bpf_map *map)
4375 {
4376         if (!bpf_map_type__is_map_in_map(map->def.type))
4377                 return errno = EINVAL, NULL;
4378
4379         return map->inner_map;
4380 }
4381
4382 int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries)
4383 {
4384         if (map->obj->loaded)
4385                 return libbpf_err(-EBUSY);
4386
4387         map->def.max_entries = max_entries;
4388
4389         /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
4390         if (map_is_ringbuf(map))
4391                 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
4392
4393         return 0;
4394 }
4395
4396 static int
4397 bpf_object__probe_loading(struct bpf_object *obj)
4398 {
4399         char *cp, errmsg[STRERR_BUFSIZE];
4400         struct bpf_insn insns[] = {
4401                 BPF_MOV64_IMM(BPF_REG_0, 0),
4402                 BPF_EXIT_INSN(),
4403         };
4404         int ret, insn_cnt = ARRAY_SIZE(insns);
4405
4406         if (obj->gen_loader)
4407                 return 0;
4408
4409         ret = bump_rlimit_memlock();
4410         if (ret)
4411                 pr_warn("Failed to bump RLIMIT_MEMLOCK (err = %d), you might need to do it explicitly!\n", ret);
4412
4413         /* make sure basic loading works */
4414         ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL);
4415         if (ret < 0)
4416                 ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL);
4417         if (ret < 0) {
4418                 ret = errno;
4419                 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
4420                 pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF "
4421                         "program. Make sure your kernel supports BPF "
4422                         "(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is "
4423                         "set to big enough value.\n", __func__, cp, ret);
4424                 return -ret;
4425         }
4426         close(ret);
4427
4428         return 0;
4429 }
4430
4431 static int probe_fd(int fd)
4432 {
4433         if (fd >= 0)
4434                 close(fd);
4435         return fd >= 0;
4436 }
4437
4438 static int probe_kern_prog_name(void)
4439 {
4440         const size_t attr_sz = offsetofend(union bpf_attr, prog_name);
4441         struct bpf_insn insns[] = {
4442                 BPF_MOV64_IMM(BPF_REG_0, 0),
4443                 BPF_EXIT_INSN(),
4444         };
4445         union bpf_attr attr;
4446         int ret;
4447
4448         memset(&attr, 0, attr_sz);
4449         attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
4450         attr.license = ptr_to_u64("GPL");
4451         attr.insns = ptr_to_u64(insns);
4452         attr.insn_cnt = (__u32)ARRAY_SIZE(insns);
4453         libbpf_strlcpy(attr.prog_name, "libbpf_nametest", sizeof(attr.prog_name));
4454
4455         /* make sure loading with name works */
4456         ret = sys_bpf_prog_load(&attr, attr_sz, PROG_LOAD_ATTEMPTS);
4457         return probe_fd(ret);
4458 }
4459
4460 static int probe_kern_global_data(void)
4461 {
4462         char *cp, errmsg[STRERR_BUFSIZE];
4463         struct bpf_insn insns[] = {
4464                 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16),
4465                 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42),
4466                 BPF_MOV64_IMM(BPF_REG_0, 0),
4467                 BPF_EXIT_INSN(),
4468         };
4469         int ret, map, insn_cnt = ARRAY_SIZE(insns);
4470
4471         map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_global", sizeof(int), 32, 1, NULL);
4472         if (map < 0) {
4473                 ret = -errno;
4474                 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
4475                 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
4476                         __func__, cp, -ret);
4477                 return ret;
4478         }
4479
4480         insns[0].imm = map;
4481
4482         ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL);
4483         close(map);
4484         return probe_fd(ret);
4485 }
4486
4487 static int probe_kern_btf(void)
4488 {
4489         static const char strs[] = "\0int";
4490         __u32 types[] = {
4491                 /* int */
4492                 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
4493         };
4494
4495         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4496                                              strs, sizeof(strs)));
4497 }
4498
4499 static int probe_kern_btf_func(void)
4500 {
4501         static const char strs[] = "\0int\0x\0a";
4502         /* void x(int a) {} */
4503         __u32 types[] = {
4504                 /* int */
4505                 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
4506                 /* FUNC_PROTO */                                /* [2] */
4507                 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
4508                 BTF_PARAM_ENC(7, 1),
4509                 /* FUNC x */                                    /* [3] */
4510                 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2),
4511         };
4512
4513         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4514                                              strs, sizeof(strs)));
4515 }
4516
4517 static int probe_kern_btf_func_global(void)
4518 {
4519         static const char strs[] = "\0int\0x\0a";
4520         /* static void x(int a) {} */
4521         __u32 types[] = {
4522                 /* int */
4523                 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
4524                 /* FUNC_PROTO */                                /* [2] */
4525                 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
4526                 BTF_PARAM_ENC(7, 1),
4527                 /* FUNC x BTF_FUNC_GLOBAL */                    /* [3] */
4528                 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2),
4529         };
4530
4531         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4532                                              strs, sizeof(strs)));
4533 }
4534
4535 static int probe_kern_btf_datasec(void)
4536 {
4537         static const char strs[] = "\0x\0.data";
4538         /* static int a; */
4539         __u32 types[] = {
4540                 /* int */
4541                 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
4542                 /* VAR x */                                     /* [2] */
4543                 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1),
4544                 BTF_VAR_STATIC,
4545                 /* DATASEC val */                               /* [3] */
4546                 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4),
4547                 BTF_VAR_SECINFO_ENC(2, 0, 4),
4548         };
4549
4550         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4551                                              strs, sizeof(strs)));
4552 }
4553
4554 static int probe_kern_btf_float(void)
4555 {
4556         static const char strs[] = "\0float";
4557         __u32 types[] = {
4558                 /* float */
4559                 BTF_TYPE_FLOAT_ENC(1, 4),
4560         };
4561
4562         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4563                                              strs, sizeof(strs)));
4564 }
4565
4566 static int probe_kern_btf_decl_tag(void)
4567 {
4568         static const char strs[] = "\0tag";
4569         __u32 types[] = {
4570                 /* int */
4571                 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
4572                 /* VAR x */                                     /* [2] */
4573                 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1),
4574                 BTF_VAR_STATIC,
4575                 /* attr */
4576                 BTF_TYPE_DECL_TAG_ENC(1, 2, -1),
4577         };
4578
4579         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4580                                              strs, sizeof(strs)));
4581 }
4582
4583 static int probe_kern_btf_type_tag(void)
4584 {
4585         static const char strs[] = "\0tag";
4586         __u32 types[] = {
4587                 /* int */
4588                 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),          /* [1] */
4589                 /* attr */
4590                 BTF_TYPE_TYPE_TAG_ENC(1, 1),                            /* [2] */
4591                 /* ptr */
4592                 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), 2),   /* [3] */
4593         };
4594
4595         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4596                                              strs, sizeof(strs)));
4597 }
4598
4599 static int probe_kern_array_mmap(void)
4600 {
4601         LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_MMAPABLE);
4602         int fd;
4603
4604         fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_mmap", sizeof(int), sizeof(int), 1, &opts);
4605         return probe_fd(fd);
4606 }
4607
4608 static int probe_kern_exp_attach_type(void)
4609 {
4610         LIBBPF_OPTS(bpf_prog_load_opts, opts, .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE);
4611         struct bpf_insn insns[] = {
4612                 BPF_MOV64_IMM(BPF_REG_0, 0),
4613                 BPF_EXIT_INSN(),
4614         };
4615         int fd, insn_cnt = ARRAY_SIZE(insns);
4616
4617         /* use any valid combination of program type and (optional)
4618          * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS)
4619          * to see if kernel supports expected_attach_type field for
4620          * BPF_PROG_LOAD command
4621          */
4622         fd = bpf_prog_load(BPF_PROG_TYPE_CGROUP_SOCK, NULL, "GPL", insns, insn_cnt, &opts);
4623         return probe_fd(fd);
4624 }
4625
4626 static int probe_kern_probe_read_kernel(void)
4627 {
4628         struct bpf_insn insns[] = {
4629                 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),   /* r1 = r10 (fp) */
4630                 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8),  /* r1 += -8 */
4631                 BPF_MOV64_IMM(BPF_REG_2, 8),            /* r2 = 8 */
4632                 BPF_MOV64_IMM(BPF_REG_3, 0),            /* r3 = 0 */
4633                 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_probe_read_kernel),
4634                 BPF_EXIT_INSN(),
4635         };
4636         int fd, insn_cnt = ARRAY_SIZE(insns);
4637
4638         fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL);
4639         return probe_fd(fd);
4640 }
4641
4642 static int probe_prog_bind_map(void)
4643 {
4644         char *cp, errmsg[STRERR_BUFSIZE];
4645         struct bpf_insn insns[] = {
4646                 BPF_MOV64_IMM(BPF_REG_0, 0),
4647                 BPF_EXIT_INSN(),
4648         };
4649         int ret, map, prog, insn_cnt = ARRAY_SIZE(insns);
4650
4651         map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_bind", sizeof(int), 32, 1, NULL);
4652         if (map < 0) {
4653                 ret = -errno;
4654                 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
4655                 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
4656                         __func__, cp, -ret);
4657                 return ret;
4658         }
4659
4660         prog = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL);
4661         if (prog < 0) {
4662                 close(map);
4663                 return 0;
4664         }
4665
4666         ret = bpf_prog_bind_map(prog, map, NULL);
4667
4668         close(map);
4669         close(prog);
4670
4671         return ret >= 0;
4672 }
4673
4674 static int probe_module_btf(void)
4675 {
4676         static const char strs[] = "\0int";
4677         __u32 types[] = {
4678                 /* int */
4679                 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
4680         };
4681         struct bpf_btf_info info;
4682         __u32 len = sizeof(info);
4683         char name[16];
4684         int fd, err;
4685
4686         fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs));
4687         if (fd < 0)
4688                 return 0; /* BTF not supported at all */
4689
4690         memset(&info, 0, sizeof(info));
4691         info.name = ptr_to_u64(name);
4692         info.name_len = sizeof(name);
4693
4694         /* check that BPF_OBJ_GET_INFO_BY_FD supports specifying name pointer;
4695          * kernel's module BTF support coincides with support for
4696          * name/name_len fields in struct bpf_btf_info.
4697          */
4698         err = bpf_obj_get_info_by_fd(fd, &info, &len);
4699         close(fd);
4700         return !err;
4701 }
4702
4703 static int probe_perf_link(void)
4704 {
4705         struct bpf_insn insns[] = {
4706                 BPF_MOV64_IMM(BPF_REG_0, 0),
4707                 BPF_EXIT_INSN(),
4708         };
4709         int prog_fd, link_fd, err;
4710
4711         prog_fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL",
4712                                 insns, ARRAY_SIZE(insns), NULL);
4713         if (prog_fd < 0)
4714                 return -errno;
4715
4716         /* use invalid perf_event FD to get EBADF, if link is supported;
4717          * otherwise EINVAL should be returned
4718          */
4719         link_fd = bpf_link_create(prog_fd, -1, BPF_PERF_EVENT, NULL);
4720         err = -errno; /* close() can clobber errno */
4721
4722         if (link_fd >= 0)
4723                 close(link_fd);
4724         close(prog_fd);
4725
4726         return link_fd < 0 && err == -EBADF;
4727 }
4728
4729 static int probe_kern_bpf_cookie(void)
4730 {
4731         struct bpf_insn insns[] = {
4732                 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_attach_cookie),
4733                 BPF_EXIT_INSN(),
4734         };
4735         int ret, insn_cnt = ARRAY_SIZE(insns);
4736
4737         ret = bpf_prog_load(BPF_PROG_TYPE_KPROBE, NULL, "GPL", insns, insn_cnt, NULL);
4738         return probe_fd(ret);
4739 }
4740
4741 static int probe_kern_btf_enum64(void)
4742 {
4743         static const char strs[] = "\0enum64";
4744         __u32 types[] = {
4745                 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 0), 8),
4746         };
4747
4748         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4749                                              strs, sizeof(strs)));
4750 }
4751
4752 static int probe_kern_syscall_wrapper(void);
4753
4754 enum kern_feature_result {
4755         FEAT_UNKNOWN = 0,
4756         FEAT_SUPPORTED = 1,
4757         FEAT_MISSING = 2,
4758 };
4759
4760 typedef int (*feature_probe_fn)(void);
4761
4762 static struct kern_feature_desc {
4763         const char *desc;
4764         feature_probe_fn probe;
4765         enum kern_feature_result res;
4766 } feature_probes[__FEAT_CNT] = {
4767         [FEAT_PROG_NAME] = {
4768                 "BPF program name", probe_kern_prog_name,
4769         },
4770         [FEAT_GLOBAL_DATA] = {
4771                 "global variables", probe_kern_global_data,
4772         },
4773         [FEAT_BTF] = {
4774                 "minimal BTF", probe_kern_btf,
4775         },
4776         [FEAT_BTF_FUNC] = {
4777                 "BTF functions", probe_kern_btf_func,
4778         },
4779         [FEAT_BTF_GLOBAL_FUNC] = {
4780                 "BTF global function", probe_kern_btf_func_global,
4781         },
4782         [FEAT_BTF_DATASEC] = {
4783                 "BTF data section and variable", probe_kern_btf_datasec,
4784         },
4785         [FEAT_ARRAY_MMAP] = {
4786                 "ARRAY map mmap()", probe_kern_array_mmap,
4787         },
4788         [FEAT_EXP_ATTACH_TYPE] = {
4789                 "BPF_PROG_LOAD expected_attach_type attribute",
4790                 probe_kern_exp_attach_type,
4791         },
4792         [FEAT_PROBE_READ_KERN] = {
4793                 "bpf_probe_read_kernel() helper", probe_kern_probe_read_kernel,
4794         },
4795         [FEAT_PROG_BIND_MAP] = {
4796                 "BPF_PROG_BIND_MAP support", probe_prog_bind_map,
4797         },
4798         [FEAT_MODULE_BTF] = {
4799                 "module BTF support", probe_module_btf,
4800         },
4801         [FEAT_BTF_FLOAT] = {
4802                 "BTF_KIND_FLOAT support", probe_kern_btf_float,
4803         },
4804         [FEAT_PERF_LINK] = {
4805                 "BPF perf link support", probe_perf_link,
4806         },
4807         [FEAT_BTF_DECL_TAG] = {
4808                 "BTF_KIND_DECL_TAG support", probe_kern_btf_decl_tag,
4809         },
4810         [FEAT_BTF_TYPE_TAG] = {
4811                 "BTF_KIND_TYPE_TAG support", probe_kern_btf_type_tag,
4812         },
4813         [FEAT_MEMCG_ACCOUNT] = {
4814                 "memcg-based memory accounting", probe_memcg_account,
4815         },
4816         [FEAT_BPF_COOKIE] = {
4817                 "BPF cookie support", probe_kern_bpf_cookie,
4818         },
4819         [FEAT_BTF_ENUM64] = {
4820                 "BTF_KIND_ENUM64 support", probe_kern_btf_enum64,
4821         },
4822         [FEAT_SYSCALL_WRAPPER] = {
4823                 "Kernel using syscall wrapper", probe_kern_syscall_wrapper,
4824         },
4825 };
4826
4827 bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id)
4828 {
4829         struct kern_feature_desc *feat = &feature_probes[feat_id];
4830         int ret;
4831
4832         if (obj && obj->gen_loader)
4833                 /* To generate loader program assume the latest kernel
4834                  * to avoid doing extra prog_load, map_create syscalls.
4835                  */
4836                 return true;
4837
4838         if (READ_ONCE(feat->res) == FEAT_UNKNOWN) {
4839                 ret = feat->probe();
4840                 if (ret > 0) {
4841                         WRITE_ONCE(feat->res, FEAT_SUPPORTED);
4842                 } else if (ret == 0) {
4843                         WRITE_ONCE(feat->res, FEAT_MISSING);
4844                 } else {
4845                         pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret);
4846                         WRITE_ONCE(feat->res, FEAT_MISSING);
4847                 }
4848         }
4849
4850         return READ_ONCE(feat->res) == FEAT_SUPPORTED;
4851 }
4852
4853 static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd)
4854 {
4855         struct bpf_map_info map_info;
4856         char msg[STRERR_BUFSIZE];
4857         __u32 map_info_len = sizeof(map_info);
4858         int err;
4859
4860         memset(&map_info, 0, map_info_len);
4861         err = bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len);
4862         if (err && errno == EINVAL)
4863                 err = bpf_get_map_info_from_fdinfo(map_fd, &map_info);
4864         if (err) {
4865                 pr_warn("failed to get map info for map FD %d: %s\n", map_fd,
4866                         libbpf_strerror_r(errno, msg, sizeof(msg)));
4867                 return false;
4868         }
4869
4870         return (map_info.type == map->def.type &&
4871                 map_info.key_size == map->def.key_size &&
4872                 map_info.value_size == map->def.value_size &&
4873                 map_info.max_entries == map->def.max_entries &&
4874                 map_info.map_flags == map->def.map_flags &&
4875                 map_info.map_extra == map->map_extra);
4876 }
4877
4878 static int
4879 bpf_object__reuse_map(struct bpf_map *map)
4880 {
4881         char *cp, errmsg[STRERR_BUFSIZE];
4882         int err, pin_fd;
4883
4884         pin_fd = bpf_obj_get(map->pin_path);
4885         if (pin_fd < 0) {
4886                 err = -errno;
4887                 if (err == -ENOENT) {
4888                         pr_debug("found no pinned map to reuse at '%s'\n",
4889                                  map->pin_path);
4890                         return 0;
4891                 }
4892
4893                 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
4894                 pr_warn("couldn't retrieve pinned map '%s': %s\n",
4895                         map->pin_path, cp);
4896                 return err;
4897         }
4898
4899         if (!map_is_reuse_compat(map, pin_fd)) {
4900                 pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n",
4901                         map->pin_path);
4902                 close(pin_fd);
4903                 return -EINVAL;
4904         }
4905
4906         err = bpf_map__reuse_fd(map, pin_fd);
4907         close(pin_fd);
4908         if (err) {
4909                 return err;
4910         }
4911         map->pinned = true;
4912         pr_debug("reused pinned map at '%s'\n", map->pin_path);
4913
4914         return 0;
4915 }
4916
4917 static int
4918 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map)
4919 {
4920         enum libbpf_map_type map_type = map->libbpf_type;
4921         char *cp, errmsg[STRERR_BUFSIZE];
4922         int err, zero = 0;
4923
4924         if (obj->gen_loader) {
4925                 bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps,
4926                                          map->mmaped, map->def.value_size);
4927                 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG)
4928                         bpf_gen__map_freeze(obj->gen_loader, map - obj->maps);
4929                 return 0;
4930         }
4931         err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0);
4932         if (err) {
4933                 err = -errno;
4934                 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4935                 pr_warn("Error setting initial map(%s) contents: %s\n",
4936                         map->name, cp);
4937                 return err;
4938         }
4939
4940         /* Freeze .rodata and .kconfig map as read-only from syscall side. */
4941         if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) {
4942                 err = bpf_map_freeze(map->fd);
4943                 if (err) {
4944                         err = -errno;
4945                         cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4946                         pr_warn("Error freezing map(%s) as read-only: %s\n",
4947                                 map->name, cp);
4948                         return err;
4949                 }
4950         }
4951         return 0;
4952 }
4953
4954 static void bpf_map__destroy(struct bpf_map *map);
4955
4956 static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, bool is_inner)
4957 {
4958         LIBBPF_OPTS(bpf_map_create_opts, create_attr);
4959         struct bpf_map_def *def = &map->def;
4960         const char *map_name = NULL;
4961         int err = 0;
4962
4963         if (kernel_supports(obj, FEAT_PROG_NAME))
4964                 map_name = map->name;
4965         create_attr.map_ifindex = map->map_ifindex;
4966         create_attr.map_flags = def->map_flags;
4967         create_attr.numa_node = map->numa_node;
4968         create_attr.map_extra = map->map_extra;
4969
4970         if (bpf_map__is_struct_ops(map))
4971                 create_attr.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;
4972
4973         if (obj->btf && btf__fd(obj->btf) >= 0) {
4974                 create_attr.btf_fd = btf__fd(obj->btf);
4975                 create_attr.btf_key_type_id = map->btf_key_type_id;
4976                 create_attr.btf_value_type_id = map->btf_value_type_id;
4977         }
4978
4979         if (bpf_map_type__is_map_in_map(def->type)) {
4980                 if (map->inner_map) {
4981                         err = bpf_object__create_map(obj, map->inner_map, true);
4982                         if (err) {
4983                                 pr_warn("map '%s': failed to create inner map: %d\n",
4984                                         map->name, err);
4985                                 return err;
4986                         }
4987                         map->inner_map_fd = bpf_map__fd(map->inner_map);
4988                 }
4989                 if (map->inner_map_fd >= 0)
4990                         create_attr.inner_map_fd = map->inner_map_fd;
4991         }
4992
4993         switch (def->type) {
4994         case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
4995         case BPF_MAP_TYPE_CGROUP_ARRAY:
4996         case BPF_MAP_TYPE_STACK_TRACE:
4997         case BPF_MAP_TYPE_ARRAY_OF_MAPS:
4998         case BPF_MAP_TYPE_HASH_OF_MAPS:
4999         case BPF_MAP_TYPE_DEVMAP:
5000         case BPF_MAP_TYPE_DEVMAP_HASH:
5001         case BPF_MAP_TYPE_CPUMAP:
5002         case BPF_MAP_TYPE_XSKMAP:
5003         case BPF_MAP_TYPE_SOCKMAP:
5004         case BPF_MAP_TYPE_SOCKHASH:
5005         case BPF_MAP_TYPE_QUEUE:
5006         case BPF_MAP_TYPE_STACK:
5007                 create_attr.btf_fd = 0;
5008                 create_attr.btf_key_type_id = 0;
5009                 create_attr.btf_value_type_id = 0;
5010                 map->btf_key_type_id = 0;
5011                 map->btf_value_type_id = 0;
5012         default:
5013                 break;
5014         }
5015
5016         if (obj->gen_loader) {
5017                 bpf_gen__map_create(obj->gen_loader, def->type, map_name,
5018                                     def->key_size, def->value_size, def->max_entries,
5019                                     &create_attr, is_inner ? -1 : map - obj->maps);
5020                 /* Pretend to have valid FD to pass various fd >= 0 checks.
5021                  * This fd == 0 will not be used with any syscall and will be reset to -1 eventually.
5022                  */
5023                 map->fd = 0;
5024         } else {
5025                 map->fd = bpf_map_create(def->type, map_name,
5026                                          def->key_size, def->value_size,
5027                                          def->max_entries, &create_attr);
5028         }
5029         if (map->fd < 0 && (create_attr.btf_key_type_id ||
5030                             create_attr.btf_value_type_id)) {
5031                 char *cp, errmsg[STRERR_BUFSIZE];
5032
5033                 err = -errno;
5034                 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
5035                 pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
5036                         map->name, cp, err);
5037                 create_attr.btf_fd = 0;
5038                 create_attr.btf_key_type_id = 0;
5039                 create_attr.btf_value_type_id = 0;
5040                 map->btf_key_type_id = 0;
5041                 map->btf_value_type_id = 0;
5042                 map->fd = bpf_map_create(def->type, map_name,
5043                                          def->key_size, def->value_size,
5044                                          def->max_entries, &create_attr);
5045         }
5046
5047         err = map->fd < 0 ? -errno : 0;
5048
5049         if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) {
5050                 if (obj->gen_loader)
5051                         map->inner_map->fd = -1;
5052                 bpf_map__destroy(map->inner_map);
5053                 zfree(&map->inner_map);
5054         }
5055
5056         return err;
5057 }
5058
5059 static int init_map_in_map_slots(struct bpf_object *obj, struct bpf_map *map)
5060 {
5061         const struct bpf_map *targ_map;
5062         unsigned int i;
5063         int fd, err = 0;
5064
5065         for (i = 0; i < map->init_slots_sz; i++) {
5066                 if (!map->init_slots[i])
5067                         continue;
5068
5069                 targ_map = map->init_slots[i];
5070                 fd = bpf_map__fd(targ_map);
5071
5072                 if (obj->gen_loader) {
5073                         bpf_gen__populate_outer_map(obj->gen_loader,
5074                                                     map - obj->maps, i,
5075                                                     targ_map - obj->maps);
5076                 } else {
5077                         err = bpf_map_update_elem(map->fd, &i, &fd, 0);
5078                 }
5079                 if (err) {
5080                         err = -errno;
5081                         pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n",
5082                                 map->name, i, targ_map->name, fd, err);
5083                         return err;
5084                 }
5085                 pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n",
5086                          map->name, i, targ_map->name, fd);
5087         }
5088
5089         zfree(&map->init_slots);
5090         map->init_slots_sz = 0;
5091
5092         return 0;
5093 }
5094
5095 static int init_prog_array_slots(struct bpf_object *obj, struct bpf_map *map)
5096 {
5097         const struct bpf_program *targ_prog;
5098         unsigned int i;
5099         int fd, err;
5100
5101         if (obj->gen_loader)
5102                 return -ENOTSUP;
5103
5104         for (i = 0; i < map->init_slots_sz; i++) {
5105                 if (!map->init_slots[i])
5106                         continue;
5107
5108                 targ_prog = map->init_slots[i];
5109                 fd = bpf_program__fd(targ_prog);
5110
5111                 err = bpf_map_update_elem(map->fd, &i, &fd, 0);
5112                 if (err) {
5113                         err = -errno;
5114                         pr_warn("map '%s': failed to initialize slot [%d] to prog '%s' fd=%d: %d\n",
5115                                 map->name, i, targ_prog->name, fd, err);
5116                         return err;
5117                 }
5118                 pr_debug("map '%s': slot [%d] set to prog '%s' fd=%d\n",
5119                          map->name, i, targ_prog->name, fd);
5120         }
5121
5122         zfree(&map->init_slots);
5123         map->init_slots_sz = 0;
5124
5125         return 0;
5126 }
5127
5128 static int bpf_object_init_prog_arrays(struct bpf_object *obj)
5129 {
5130         struct bpf_map *map;
5131         int i, err;
5132
5133         for (i = 0; i < obj->nr_maps; i++) {
5134                 map = &obj->maps[i];
5135
5136                 if (!map->init_slots_sz || map->def.type != BPF_MAP_TYPE_PROG_ARRAY)
5137                         continue;
5138
5139                 err = init_prog_array_slots(obj, map);
5140                 if (err < 0) {
5141                         zclose(map->fd);
5142                         return err;
5143                 }
5144         }
5145         return 0;
5146 }
5147
5148 static int map_set_def_max_entries(struct bpf_map *map)
5149 {
5150         if (map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !map->def.max_entries) {
5151                 int nr_cpus;
5152
5153                 nr_cpus = libbpf_num_possible_cpus();
5154                 if (nr_cpus < 0) {
5155                         pr_warn("map '%s': failed to determine number of system CPUs: %d\n",
5156                                 map->name, nr_cpus);
5157                         return nr_cpus;
5158                 }
5159                 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus);
5160                 map->def.max_entries = nr_cpus;
5161         }
5162
5163         return 0;
5164 }
5165
5166 static int
5167 bpf_object__create_maps(struct bpf_object *obj)
5168 {
5169         struct bpf_map *map;
5170         char *cp, errmsg[STRERR_BUFSIZE];
5171         unsigned int i, j;
5172         int err;
5173         bool retried;
5174
5175         for (i = 0; i < obj->nr_maps; i++) {
5176                 map = &obj->maps[i];
5177
5178                 /* To support old kernels, we skip creating global data maps
5179                  * (.rodata, .data, .kconfig, etc); later on, during program
5180                  * loading, if we detect that at least one of the to-be-loaded
5181                  * programs is referencing any global data map, we'll error
5182                  * out with program name and relocation index logged.
5183                  * This approach allows to accommodate Clang emitting
5184                  * unnecessary .rodata.str1.1 sections for string literals,
5185                  * but also it allows to have CO-RE applications that use
5186                  * global variables in some of BPF programs, but not others.
5187                  * If those global variable-using programs are not loaded at
5188                  * runtime due to bpf_program__set_autoload(prog, false),
5189                  * bpf_object loading will succeed just fine even on old
5190                  * kernels.
5191                  */
5192                 if (bpf_map__is_internal(map) && !kernel_supports(obj, FEAT_GLOBAL_DATA))
5193                         map->autocreate = false;
5194
5195                 if (!map->autocreate) {
5196                         pr_debug("map '%s': skipped auto-creating...\n", map->name);
5197                         continue;
5198                 }
5199
5200                 err = map_set_def_max_entries(map);
5201                 if (err)
5202                         goto err_out;
5203
5204                 retried = false;
5205 retry:
5206                 if (map->pin_path) {
5207                         err = bpf_object__reuse_map(map);
5208                         if (err) {
5209                                 pr_warn("map '%s': error reusing pinned map\n",
5210                                         map->name);
5211                                 goto err_out;
5212                         }
5213                         if (retried && map->fd < 0) {
5214                                 pr_warn("map '%s': cannot find pinned map\n",
5215                                         map->name);
5216                                 err = -ENOENT;
5217                                 goto err_out;
5218                         }
5219                 }
5220
5221                 if (map->fd >= 0) {
5222                         pr_debug("map '%s': skipping creation (preset fd=%d)\n",
5223                                  map->name, map->fd);
5224                 } else {
5225                         err = bpf_object__create_map(obj, map, false);
5226                         if (err)
5227                                 goto err_out;
5228
5229                         pr_debug("map '%s': created successfully, fd=%d\n",
5230                                  map->name, map->fd);
5231
5232                         if (bpf_map__is_internal(map)) {
5233                                 err = bpf_object__populate_internal_map(obj, map);
5234                                 if (err < 0) {
5235                                         zclose(map->fd);
5236                                         goto err_out;
5237                                 }
5238                         }
5239
5240                         if (map->init_slots_sz && map->def.type != BPF_MAP_TYPE_PROG_ARRAY) {
5241                                 err = init_map_in_map_slots(obj, map);
5242                                 if (err < 0) {
5243                                         zclose(map->fd);
5244                                         goto err_out;
5245                                 }
5246                         }
5247                 }
5248
5249                 if (map->pin_path && !map->pinned) {
5250                         err = bpf_map__pin(map, NULL);
5251                         if (err) {
5252                                 zclose(map->fd);
5253                                 if (!retried && err == -EEXIST) {
5254                                         retried = true;
5255                                         goto retry;
5256                                 }
5257                                 pr_warn("map '%s': failed to auto-pin at '%s': %d\n",
5258                                         map->name, map->pin_path, err);
5259                                 goto err_out;
5260                         }
5261                 }
5262         }
5263
5264         return 0;
5265
5266 err_out:
5267         cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
5268         pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err);
5269         pr_perm_msg(err);
5270         for (j = 0; j < i; j++)
5271                 zclose(obj->maps[j].fd);
5272         return err;
5273 }
5274
5275 static bool bpf_core_is_flavor_sep(const char *s)
5276 {
5277         /* check X___Y name pattern, where X and Y are not underscores */
5278         return s[0] != '_' &&                                 /* X */
5279                s[1] == '_' && s[2] == '_' && s[3] == '_' &&   /* ___ */
5280                s[4] != '_';                                   /* Y */
5281 }
5282
5283 /* Given 'some_struct_name___with_flavor' return the length of a name prefix
5284  * before last triple underscore. Struct name part after last triple
5285  * underscore is ignored by BPF CO-RE relocation during relocation matching.
5286  */
5287 size_t bpf_core_essential_name_len(const char *name)
5288 {
5289         size_t n = strlen(name);
5290         int i;
5291
5292         for (i = n - 5; i >= 0; i--) {
5293                 if (bpf_core_is_flavor_sep(name + i))
5294                         return i + 1;
5295         }
5296         return n;
5297 }
5298
5299 void bpf_core_free_cands(struct bpf_core_cand_list *cands)
5300 {
5301         if (!cands)
5302                 return;
5303
5304         free(cands->cands);
5305         free(cands);
5306 }
5307
5308 int bpf_core_add_cands(struct bpf_core_cand *local_cand,
5309                        size_t local_essent_len,
5310                        const struct btf *targ_btf,
5311                        const char *targ_btf_name,
5312                        int targ_start_id,
5313                        struct bpf_core_cand_list *cands)
5314 {
5315         struct bpf_core_cand *new_cands, *cand;
5316         const struct btf_type *t, *local_t;
5317         const char *targ_name, *local_name;
5318         size_t targ_essent_len;
5319         int n, i;
5320
5321         local_t = btf__type_by_id(local_cand->btf, local_cand->id);
5322         local_name = btf__str_by_offset(local_cand->btf, local_t->name_off);
5323
5324         n = btf__type_cnt(targ_btf);
5325         for (i = targ_start_id; i < n; i++) {
5326                 t = btf__type_by_id(targ_btf, i);
5327                 if (!btf_kind_core_compat(t, local_t))
5328                         continue;
5329
5330                 targ_name = btf__name_by_offset(targ_btf, t->name_off);
5331                 if (str_is_empty(targ_name))
5332                         continue;
5333
5334                 targ_essent_len = bpf_core_essential_name_len(targ_name);
5335                 if (targ_essent_len != local_essent_len)
5336                         continue;
5337
5338                 if (strncmp(local_name, targ_name, local_essent_len) != 0)
5339                         continue;
5340
5341                 pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n",
5342                          local_cand->id, btf_kind_str(local_t),
5343                          local_name, i, btf_kind_str(t), targ_name,
5344                          targ_btf_name);
5345                 new_cands = libbpf_reallocarray(cands->cands, cands->len + 1,
5346                                               sizeof(*cands->cands));
5347                 if (!new_cands)
5348                         return -ENOMEM;
5349
5350                 cand = &new_cands[cands->len];
5351                 cand->btf = targ_btf;
5352                 cand->id = i;
5353
5354                 cands->cands = new_cands;
5355                 cands->len++;
5356         }
5357         return 0;
5358 }
5359
5360 static int load_module_btfs(struct bpf_object *obj)
5361 {
5362         struct bpf_btf_info info;
5363         struct module_btf *mod_btf;
5364         struct btf *btf;
5365         char name[64];
5366         __u32 id = 0, len;
5367         int err, fd;
5368
5369         if (obj->btf_modules_loaded)
5370                 return 0;
5371
5372         if (obj->gen_loader)
5373                 return 0;
5374
5375         /* don't do this again, even if we find no module BTFs */
5376         obj->btf_modules_loaded = true;
5377
5378         /* kernel too old to support module BTFs */
5379         if (!kernel_supports(obj, FEAT_MODULE_BTF))
5380                 return 0;
5381
5382         while (true) {
5383                 err = bpf_btf_get_next_id(id, &id);
5384                 if (err && errno == ENOENT)
5385                         return 0;
5386                 if (err) {
5387                         err = -errno;
5388                         pr_warn("failed to iterate BTF objects: %d\n", err);
5389                         return err;
5390                 }
5391
5392                 fd = bpf_btf_get_fd_by_id(id);
5393                 if (fd < 0) {
5394                         if (errno == ENOENT)
5395                                 continue; /* expected race: BTF was unloaded */
5396                         err = -errno;
5397                         pr_warn("failed to get BTF object #%d FD: %d\n", id, err);
5398                         return err;
5399                 }
5400
5401                 len = sizeof(info);
5402                 memset(&info, 0, sizeof(info));
5403                 info.name = ptr_to_u64(name);
5404                 info.name_len = sizeof(name);
5405
5406                 err = bpf_obj_get_info_by_fd(fd, &info, &len);
5407                 if (err) {
5408                         err = -errno;
5409                         pr_warn("failed to get BTF object #%d info: %d\n", id, err);
5410                         goto err_out;
5411                 }
5412
5413                 /* ignore non-module BTFs */
5414                 if (!info.kernel_btf || strcmp(name, "vmlinux") == 0) {
5415                         close(fd);
5416                         continue;
5417                 }
5418
5419                 btf = btf_get_from_fd(fd, obj->btf_vmlinux);
5420                 err = libbpf_get_error(btf);
5421                 if (err) {
5422                         pr_warn("failed to load module [%s]'s BTF object #%d: %d\n",
5423                                 name, id, err);
5424                         goto err_out;
5425                 }
5426
5427                 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap,
5428                                         sizeof(*obj->btf_modules), obj->btf_module_cnt + 1);
5429                 if (err)
5430                         goto err_out;
5431
5432                 mod_btf = &obj->btf_modules[obj->btf_module_cnt++];
5433
5434                 mod_btf->btf = btf;
5435                 mod_btf->id = id;
5436                 mod_btf->fd = fd;
5437                 mod_btf->name = strdup(name);
5438                 if (!mod_btf->name) {
5439                         err = -ENOMEM;
5440                         goto err_out;
5441                 }
5442                 continue;
5443
5444 err_out:
5445                 close(fd);
5446                 return err;
5447         }
5448
5449         return 0;
5450 }
5451
5452 static struct bpf_core_cand_list *
5453 bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 local_type_id)
5454 {
5455         struct bpf_core_cand local_cand = {};
5456         struct bpf_core_cand_list *cands;
5457         const struct btf *main_btf;
5458         const struct btf_type *local_t;
5459         const char *local_name;
5460         size_t local_essent_len;
5461         int err, i;
5462
5463         local_cand.btf = local_btf;
5464         local_cand.id = local_type_id;
5465         local_t = btf__type_by_id(local_btf, local_type_id);
5466         if (!local_t)
5467                 return ERR_PTR(-EINVAL);
5468
5469         local_name = btf__name_by_offset(local_btf, local_t->name_off);
5470         if (str_is_empty(local_name))
5471                 return ERR_PTR(-EINVAL);
5472         local_essent_len = bpf_core_essential_name_len(local_name);
5473
5474         cands = calloc(1, sizeof(*cands));
5475         if (!cands)
5476                 return ERR_PTR(-ENOMEM);
5477
5478         /* Attempt to find target candidates in vmlinux BTF first */
5479         main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux;
5480         err = bpf_core_add_cands(&local_cand, local_essent_len, main_btf, "vmlinux", 1, cands);
5481         if (err)
5482                 goto err_out;
5483
5484         /* if vmlinux BTF has any candidate, don't got for module BTFs */
5485         if (cands->len)
5486                 return cands;
5487
5488         /* if vmlinux BTF was overridden, don't attempt to load module BTFs */
5489         if (obj->btf_vmlinux_override)
5490                 return cands;
5491
5492         /* now look through module BTFs, trying to still find candidates */
5493         err = load_module_btfs(obj);
5494         if (err)
5495                 goto err_out;
5496
5497         for (i = 0; i < obj->btf_module_cnt; i++) {
5498                 err = bpf_core_add_cands(&local_cand, local_essent_len,
5499                                          obj->btf_modules[i].btf,
5500                                          obj->btf_modules[i].name,
5501                                          btf__type_cnt(obj->btf_vmlinux),
5502                                          cands);
5503                 if (err)
5504                         goto err_out;
5505         }
5506
5507         return cands;
5508 err_out:
5509         bpf_core_free_cands(cands);
5510         return ERR_PTR(err);
5511 }
5512
5513 /* Check local and target types for compatibility. This check is used for
5514  * type-based CO-RE relocations and follow slightly different rules than
5515  * field-based relocations. This function assumes that root types were already
5516  * checked for name match. Beyond that initial root-level name check, names
5517  * are completely ignored. Compatibility rules are as follows:
5518  *   - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but
5519  *     kind should match for local and target types (i.e., STRUCT is not
5520  *     compatible with UNION);
5521  *   - for ENUMs, the size is ignored;
5522  *   - for INT, size and signedness are ignored;
5523  *   - for ARRAY, dimensionality is ignored, element types are checked for
5524  *     compatibility recursively;
5525  *   - CONST/VOLATILE/RESTRICT modifiers are ignored;
5526  *   - TYPEDEFs/PTRs are compatible if types they pointing to are compatible;
5527  *   - FUNC_PROTOs are compatible if they have compatible signature: same
5528  *     number of input args and compatible return and argument types.
5529  * These rules are not set in stone and probably will be adjusted as we get
5530  * more experience with using BPF CO-RE relocations.
5531  */
5532 int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id,
5533                               const struct btf *targ_btf, __u32 targ_id)
5534 {
5535         return __bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id, 32);
5536 }
5537
5538 int bpf_core_types_match(const struct btf *local_btf, __u32 local_id,
5539                          const struct btf *targ_btf, __u32 targ_id)
5540 {
5541         return __bpf_core_types_match(local_btf, local_id, targ_btf, targ_id, false, 32);
5542 }
5543
5544 static size_t bpf_core_hash_fn(const void *key, void *ctx)
5545 {
5546         return (size_t)key;
5547 }
5548
5549 static bool bpf_core_equal_fn(const void *k1, const void *k2, void *ctx)
5550 {
5551         return k1 == k2;
5552 }
5553
5554 static void *u32_as_hash_key(__u32 x)
5555 {
5556         return (void *)(uintptr_t)x;
5557 }
5558
5559 static int record_relo_core(struct bpf_program *prog,
5560                             const struct bpf_core_relo *core_relo, int insn_idx)
5561 {
5562         struct reloc_desc *relos, *relo;
5563
5564         relos = libbpf_reallocarray(prog->reloc_desc,
5565                                     prog->nr_reloc + 1, sizeof(*relos));
5566         if (!relos)
5567                 return -ENOMEM;
5568         relo = &relos[prog->nr_reloc];
5569         relo->type = RELO_CORE;
5570         relo->insn_idx = insn_idx;
5571         relo->core_relo = core_relo;
5572         prog->reloc_desc = relos;
5573         prog->nr_reloc++;
5574         return 0;
5575 }
5576
5577 static const struct bpf_core_relo *find_relo_core(struct bpf_program *prog, int insn_idx)
5578 {
5579         struct reloc_desc *relo;
5580         int i;
5581
5582         for (i = 0; i < prog->nr_reloc; i++) {
5583                 relo = &prog->reloc_desc[i];
5584                 if (relo->type != RELO_CORE || relo->insn_idx != insn_idx)
5585                         continue;
5586
5587                 return relo->core_relo;
5588         }
5589
5590         return NULL;
5591 }
5592
5593 static int bpf_core_resolve_relo(struct bpf_program *prog,
5594                                  const struct bpf_core_relo *relo,
5595                                  int relo_idx,
5596                                  const struct btf *local_btf,
5597                                  struct hashmap *cand_cache,
5598                                  struct bpf_core_relo_res *targ_res)
5599 {
5600         struct bpf_core_spec specs_scratch[3] = {};
5601         const void *type_key = u32_as_hash_key(relo->type_id);
5602         struct bpf_core_cand_list *cands = NULL;
5603         const char *prog_name = prog->name;
5604         const struct btf_type *local_type;
5605         const char *local_name;
5606         __u32 local_id = relo->type_id;
5607         int err;
5608
5609         local_type = btf__type_by_id(local_btf, local_id);
5610         if (!local_type)
5611                 return -EINVAL;
5612
5613         local_name = btf__name_by_offset(local_btf, local_type->name_off);
5614         if (!local_name)
5615                 return -EINVAL;
5616
5617         if (relo->kind != BPF_CORE_TYPE_ID_LOCAL &&
5618             !hashmap__find(cand_cache, type_key, (void **)&cands)) {
5619                 cands = bpf_core_find_cands(prog->obj, local_btf, local_id);
5620                 if (IS_ERR(cands)) {
5621                         pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n",
5622                                 prog_name, relo_idx, local_id, btf_kind_str(local_type),
5623                                 local_name, PTR_ERR(cands));
5624                         return PTR_ERR(cands);
5625                 }
5626                 err = hashmap__set(cand_cache, type_key, cands, NULL, NULL);
5627                 if (err) {
5628                         bpf_core_free_cands(cands);
5629                         return err;
5630                 }
5631         }
5632
5633         return bpf_core_calc_relo_insn(prog_name, relo, relo_idx, local_btf, cands, specs_scratch,
5634                                        targ_res);
5635 }
5636
5637 static int
5638 bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path)
5639 {
5640         const struct btf_ext_info_sec *sec;
5641         struct bpf_core_relo_res targ_res;
5642         const struct bpf_core_relo *rec;
5643         const struct btf_ext_info *seg;
5644         struct hashmap_entry *entry;
5645         struct hashmap *cand_cache = NULL;
5646         struct bpf_program *prog;
5647         struct bpf_insn *insn;
5648         const char *sec_name;
5649         int i, err = 0, insn_idx, sec_idx, sec_num;
5650
5651         if (obj->btf_ext->core_relo_info.len == 0)
5652                 return 0;
5653
5654         if (targ_btf_path) {
5655                 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL);
5656                 err = libbpf_get_error(obj->btf_vmlinux_override);
5657                 if (err) {
5658                         pr_warn("failed to parse target BTF: %d\n", err);
5659                         return err;
5660                 }
5661         }
5662
5663         cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL);
5664         if (IS_ERR(cand_cache)) {
5665                 err = PTR_ERR(cand_cache);
5666                 goto out;
5667         }
5668
5669         seg = &obj->btf_ext->core_relo_info;
5670         sec_num = 0;
5671         for_each_btf_ext_sec(seg, sec) {
5672                 sec_idx = seg->sec_idxs[sec_num];
5673                 sec_num++;
5674
5675                 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
5676                 if (str_is_empty(sec_name)) {
5677                         err = -EINVAL;
5678                         goto out;
5679                 }
5680
5681                 pr_debug("sec '%s': found %d CO-RE relocations\n", sec_name, sec->num_info);
5682
5683                 for_each_btf_ext_rec(seg, sec, i, rec) {
5684                         if (rec->insn_off % BPF_INSN_SZ)
5685                                 return -EINVAL;
5686                         insn_idx = rec->insn_off / BPF_INSN_SZ;
5687                         prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
5688                         if (!prog) {
5689                                 /* When __weak subprog is "overridden" by another instance
5690                                  * of the subprog from a different object file, linker still
5691                                  * appends all the .BTF.ext info that used to belong to that
5692                                  * eliminated subprogram.
5693                                  * This is similar to what x86-64 linker does for relocations.
5694                                  * So just ignore such relocations just like we ignore
5695                                  * subprog instructions when discovering subprograms.
5696                                  */
5697                                 pr_debug("sec '%s': skipping CO-RE relocation #%d for insn #%d belonging to eliminated weak subprogram\n",
5698                                          sec_name, i, insn_idx);
5699                                 continue;
5700                         }
5701                         /* no need to apply CO-RE relocation if the program is
5702                          * not going to be loaded
5703                          */
5704                         if (!prog->autoload)
5705                                 continue;
5706
5707                         /* adjust insn_idx from section frame of reference to the local
5708                          * program's frame of reference; (sub-)program code is not yet
5709                          * relocated, so it's enough to just subtract in-section offset
5710                          */
5711                         insn_idx = insn_idx - prog->sec_insn_off;
5712                         if (insn_idx >= prog->insns_cnt)
5713                                 return -EINVAL;
5714                         insn = &prog->insns[insn_idx];
5715
5716                         err = record_relo_core(prog, rec, insn_idx);
5717                         if (err) {
5718                                 pr_warn("prog '%s': relo #%d: failed to record relocation: %d\n",
5719                                         prog->name, i, err);
5720                                 goto out;
5721                         }
5722
5723                         if (prog->obj->gen_loader)
5724                                 continue;
5725
5726                         err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res);
5727                         if (err) {
5728                                 pr_warn("prog '%s': relo #%d: failed to relocate: %d\n",
5729                                         prog->name, i, err);
5730                                 goto out;
5731                         }
5732
5733                         err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res);
5734                         if (err) {
5735                                 pr_warn("prog '%s': relo #%d: failed to patch insn #%u: %d\n",
5736                                         prog->name, i, insn_idx, err);
5737                                 goto out;
5738                         }
5739                 }
5740         }
5741
5742 out:
5743         /* obj->btf_vmlinux and module BTFs are freed after object load */
5744         btf__free(obj->btf_vmlinux_override);
5745         obj->btf_vmlinux_override = NULL;
5746
5747         if (!IS_ERR_OR_NULL(cand_cache)) {
5748                 hashmap__for_each_entry(cand_cache, entry, i) {
5749                         bpf_core_free_cands(entry->value);
5750                 }
5751                 hashmap__free(cand_cache);
5752         }
5753         return err;
5754 }
5755
5756 /* base map load ldimm64 special constant, used also for log fixup logic */
5757 #define MAP_LDIMM64_POISON_BASE 2001000000
5758 #define MAP_LDIMM64_POISON_PFX "200100"
5759
5760 static void poison_map_ldimm64(struct bpf_program *prog, int relo_idx,
5761                                int insn_idx, struct bpf_insn *insn,
5762                                int map_idx, const struct bpf_map *map)
5763 {
5764         int i;
5765
5766         pr_debug("prog '%s': relo #%d: poisoning insn #%d that loads map #%d '%s'\n",
5767                  prog->name, relo_idx, insn_idx, map_idx, map->name);
5768
5769         /* we turn single ldimm64 into two identical invalid calls */
5770         for (i = 0; i < 2; i++) {
5771                 insn->code = BPF_JMP | BPF_CALL;
5772                 insn->dst_reg = 0;
5773                 insn->src_reg = 0;
5774                 insn->off = 0;
5775                 /* if this instruction is reachable (not a dead code),
5776                  * verifier will complain with something like:
5777                  * invalid func unknown#2001000123
5778                  * where lower 123 is map index into obj->maps[] array
5779                  */
5780                 insn->imm = MAP_LDIMM64_POISON_BASE + map_idx;
5781
5782                 insn++;
5783         }
5784 }
5785
5786 /* Relocate data references within program code:
5787  *  - map references;
5788  *  - global variable references;
5789  *  - extern references.
5790  */
5791 static int
5792 bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog)
5793 {
5794         int i;
5795
5796         for (i = 0; i < prog->nr_reloc; i++) {
5797                 struct reloc_desc *relo = &prog->reloc_desc[i];
5798                 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
5799                 const struct bpf_map *map;
5800                 struct extern_desc *ext;
5801
5802                 switch (relo->type) {
5803                 case RELO_LD64:
5804                         map = &obj->maps[relo->map_idx];
5805                         if (obj->gen_loader) {
5806                                 insn[0].src_reg = BPF_PSEUDO_MAP_IDX;
5807                                 insn[0].imm = relo->map_idx;
5808                         } else if (map->autocreate) {
5809                                 insn[0].src_reg = BPF_PSEUDO_MAP_FD;
5810                                 insn[0].imm = map->fd;
5811                         } else {
5812                                 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
5813                                                    relo->map_idx, map);
5814                         }
5815                         break;
5816                 case RELO_DATA:
5817                         map = &obj->maps[relo->map_idx];
5818                         insn[1].imm = insn[0].imm + relo->sym_off;
5819                         if (obj->gen_loader) {
5820                                 insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE;
5821                                 insn[0].imm = relo->map_idx;
5822                         } else if (map->autocreate) {
5823                                 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
5824                                 insn[0].imm = map->fd;
5825                         } else {
5826                                 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
5827                                                    relo->map_idx, map);
5828                         }
5829                         break;
5830                 case RELO_EXTERN_VAR:
5831                         ext = &obj->externs[relo->sym_off];
5832                         if (ext->type == EXT_KCFG) {
5833                                 if (obj->gen_loader) {
5834                                         insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE;
5835                                         insn[0].imm = obj->kconfig_map_idx;
5836                                 } else {
5837                                         insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
5838                                         insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;
5839                                 }
5840                                 insn[1].imm = ext->kcfg.data_off;
5841                         } else /* EXT_KSYM */ {
5842                                 if (ext->ksym.type_id && ext->is_set) { /* typed ksyms */
5843                                         insn[0].src_reg = BPF_PSEUDO_BTF_ID;
5844                                         insn[0].imm = ext->ksym.kernel_btf_id;
5845                                         insn[1].imm = ext->ksym.kernel_btf_obj_fd;
5846                                 } else { /* typeless ksyms or unresolved typed ksyms */
5847                                         insn[0].imm = (__u32)ext->ksym.addr;
5848                                         insn[1].imm = ext->ksym.addr >> 32;
5849                                 }
5850                         }
5851                         break;
5852                 case RELO_EXTERN_FUNC:
5853                         ext = &obj->externs[relo->sym_off];
5854                         insn[0].src_reg = BPF_PSEUDO_KFUNC_CALL;
5855                         if (ext->is_set) {
5856                                 insn[0].imm = ext->ksym.kernel_btf_id;
5857                                 insn[0].off = ext->ksym.btf_fd_idx;
5858                         } else { /* unresolved weak kfunc */
5859                                 insn[0].imm = 0;
5860                                 insn[0].off = 0;
5861                         }
5862                         break;
5863                 case RELO_SUBPROG_ADDR:
5864                         if (insn[0].src_reg != BPF_PSEUDO_FUNC) {
5865                                 pr_warn("prog '%s': relo #%d: bad insn\n",
5866                                         prog->name, i);
5867                                 return -EINVAL;
5868                         }
5869                         /* handled already */
5870                         break;
5871                 case RELO_CALL:
5872                         /* handled already */
5873                         break;
5874                 case RELO_CORE:
5875                         /* will be handled by bpf_program_record_relos() */
5876                         break;
5877                 default:
5878                         pr_warn("prog '%s': relo #%d: bad relo type %d\n",
5879                                 prog->name, i, relo->type);
5880                         return -EINVAL;
5881                 }
5882         }
5883
5884         return 0;
5885 }
5886
5887 static int adjust_prog_btf_ext_info(const struct bpf_object *obj,
5888                                     const struct bpf_program *prog,
5889                                     const struct btf_ext_info *ext_info,
5890                                     void **prog_info, __u32 *prog_rec_cnt,
5891                                     __u32 *prog_rec_sz)
5892 {
5893         void *copy_start = NULL, *copy_end = NULL;
5894         void *rec, *rec_end, *new_prog_info;
5895         const struct btf_ext_info_sec *sec;
5896         size_t old_sz, new_sz;
5897         int i, sec_num, sec_idx, off_adj;
5898
5899         sec_num = 0;
5900         for_each_btf_ext_sec(ext_info, sec) {
5901                 sec_idx = ext_info->sec_idxs[sec_num];
5902                 sec_num++;
5903                 if (prog->sec_idx != sec_idx)
5904                         continue;
5905
5906                 for_each_btf_ext_rec(ext_info, sec, i, rec) {
5907                         __u32 insn_off = *(__u32 *)rec / BPF_INSN_SZ;
5908
5909                         if (insn_off < prog->sec_insn_off)
5910                                 continue;
5911                         if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt)
5912                                 break;
5913
5914                         if (!copy_start)
5915                                 copy_start = rec;
5916                         copy_end = rec + ext_info->rec_size;
5917                 }
5918
5919                 if (!copy_start)
5920                         return -ENOENT;
5921
5922                 /* append func/line info of a given (sub-)program to the main
5923                  * program func/line info
5924                  */
5925                 old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size;
5926                 new_sz = old_sz + (copy_end - copy_start);
5927                 new_prog_info = realloc(*prog_info, new_sz);
5928                 if (!new_prog_info)
5929                         return -ENOMEM;
5930                 *prog_info = new_prog_info;
5931                 *prog_rec_cnt = new_sz / ext_info->rec_size;
5932                 memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start);
5933
5934                 /* Kernel instruction offsets are in units of 8-byte
5935                  * instructions, while .BTF.ext instruction offsets generated
5936                  * by Clang are in units of bytes. So convert Clang offsets
5937                  * into kernel offsets and adjust offset according to program
5938                  * relocated position.
5939                  */
5940                 off_adj = prog->sub_insn_off - prog->sec_insn_off;
5941                 rec = new_prog_info + old_sz;
5942                 rec_end = new_prog_info + new_sz;
5943                 for (; rec < rec_end; rec += ext_info->rec_size) {
5944                         __u32 *insn_off = rec;
5945
5946                         *insn_off = *insn_off / BPF_INSN_SZ + off_adj;
5947                 }
5948                 *prog_rec_sz = ext_info->rec_size;
5949                 return 0;
5950         }
5951
5952         return -ENOENT;
5953 }
5954
5955 static int
5956 reloc_prog_func_and_line_info(const struct bpf_object *obj,
5957                               struct bpf_program *main_prog,
5958                               const struct bpf_program *prog)
5959 {
5960         int err;
5961
5962         /* no .BTF.ext relocation if .BTF.ext is missing or kernel doesn't
5963          * supprot func/line info
5964          */
5965         if (!obj->btf_ext || !kernel_supports(obj, FEAT_BTF_FUNC))
5966                 return 0;
5967
5968         /* only attempt func info relocation if main program's func_info
5969          * relocation was successful
5970          */
5971         if (main_prog != prog && !main_prog->func_info)
5972                 goto line_info;
5973
5974         err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info,
5975                                        &main_prog->func_info,
5976                                        &main_prog->func_info_cnt,
5977                                        &main_prog->func_info_rec_size);
5978         if (err) {
5979                 if (err != -ENOENT) {
5980                         pr_warn("prog '%s': error relocating .BTF.ext function info: %d\n",
5981                                 prog->name, err);
5982                         return err;
5983                 }
5984                 if (main_prog->func_info) {
5985                         /*
5986                          * Some info has already been found but has problem
5987                          * in the last btf_ext reloc. Must have to error out.
5988                          */
5989                         pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name);
5990                         return err;
5991                 }
5992                 /* Have problem loading the very first info. Ignore the rest. */
5993                 pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext func info.\n",
5994                         prog->name);
5995         }
5996
5997 line_info:
5998         /* don't relocate line info if main program's relocation failed */
5999         if (main_prog != prog && !main_prog->line_info)
6000                 return 0;
6001
6002         err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info,
6003                                        &main_prog->line_info,
6004                                        &main_prog->line_info_cnt,
6005                                        &main_prog->line_info_rec_size);
6006         if (err) {
6007                 if (err != -ENOENT) {
6008                         pr_warn("prog '%s': error relocating .BTF.ext line info: %d\n",
6009                                 prog->name, err);
6010                         return err;
6011                 }
6012                 if (main_prog->line_info) {
6013                         /*
6014                          * Some info has already been found but has problem
6015                          * in the last btf_ext reloc. Must have to error out.
6016                          */
6017                         pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name);
6018                         return err;
6019                 }
6020                 /* Have problem loading the very first info. Ignore the rest. */
6021                 pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line info.\n",
6022                         prog->name);
6023         }
6024         return 0;
6025 }
6026
6027 static int cmp_relo_by_insn_idx(const void *key, const void *elem)
6028 {
6029         size_t insn_idx = *(const size_t *)key;
6030         const struct reloc_desc *relo = elem;
6031
6032         if (insn_idx == relo->insn_idx)
6033                 return 0;
6034         return insn_idx < relo->insn_idx ? -1 : 1;
6035 }
6036
6037 static struct reloc_desc *find_prog_insn_relo(const struct bpf_program *prog, size_t insn_idx)
6038 {
6039         if (!prog->nr_reloc)
6040                 return NULL;
6041         return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc,
6042                        sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx);
6043 }
6044
6045 static int append_subprog_relos(struct bpf_program *main_prog, struct bpf_program *subprog)
6046 {
6047         int new_cnt = main_prog->nr_reloc + subprog->nr_reloc;
6048         struct reloc_desc *relos;
6049         int i;
6050
6051         if (main_prog == subprog)
6052                 return 0;
6053         relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos));
6054         if (!relos)
6055                 return -ENOMEM;
6056         if (subprog->nr_reloc)
6057                 memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc,
6058                        sizeof(*relos) * subprog->nr_reloc);
6059
6060         for (i = main_prog->nr_reloc; i < new_cnt; i++)
6061                 relos[i].insn_idx += subprog->sub_insn_off;
6062         /* After insn_idx adjustment the 'relos' array is still sorted
6063          * by insn_idx and doesn't break bsearch.
6064          */
6065         main_prog->reloc_desc = relos;
6066         main_prog->nr_reloc = new_cnt;
6067         return 0;
6068 }
6069
6070 static int
6071 bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog,
6072                        struct bpf_program *prog)
6073 {
6074         size_t sub_insn_idx, insn_idx, new_cnt;
6075         struct bpf_program *subprog;
6076         struct bpf_insn *insns, *insn;
6077         struct reloc_desc *relo;
6078         int err;
6079
6080         err = reloc_prog_func_and_line_info(obj, main_prog, prog);
6081         if (err)
6082                 return err;
6083
6084         for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) {
6085                 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6086                 if (!insn_is_subprog_call(insn) && !insn_is_pseudo_func(insn))
6087                         continue;
6088
6089                 relo = find_prog_insn_relo(prog, insn_idx);
6090                 if (relo && relo->type == RELO_EXTERN_FUNC)
6091                         /* kfunc relocations will be handled later
6092                          * in bpf_object__relocate_data()
6093                          */
6094                         continue;
6095                 if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) {
6096                         pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n",
6097                                 prog->name, insn_idx, relo->type);
6098                         return -LIBBPF_ERRNO__RELOC;
6099                 }
6100                 if (relo) {
6101                         /* sub-program instruction index is a combination of
6102                          * an offset of a symbol pointed to by relocation and
6103                          * call instruction's imm field; for global functions,
6104                          * call always has imm = -1, but for static functions
6105                          * relocation is against STT_SECTION and insn->imm
6106                          * points to a start of a static function
6107                          *
6108                          * for subprog addr relocation, the relo->sym_off + insn->imm is
6109                          * the byte offset in the corresponding section.
6110                          */
6111                         if (relo->type == RELO_CALL)
6112                                 sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1;
6113                         else
6114                                 sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ;
6115                 } else if (insn_is_pseudo_func(insn)) {
6116                         /*
6117                          * RELO_SUBPROG_ADDR relo is always emitted even if both
6118                          * functions are in the same section, so it shouldn't reach here.
6119                          */
6120                         pr_warn("prog '%s': missing subprog addr relo for insn #%zu\n",
6121                                 prog->name, insn_idx);
6122                         return -LIBBPF_ERRNO__RELOC;
6123                 } else {
6124                         /* if subprogram call is to a static function within
6125                          * the same ELF section, there won't be any relocation
6126                          * emitted, but it also means there is no additional
6127                          * offset necessary, insns->imm is relative to
6128                          * instruction's original position within the section
6129                          */
6130                         sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1;
6131                 }
6132
6133                 /* we enforce that sub-programs should be in .text section */
6134                 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx);
6135                 if (!subprog) {
6136                         pr_warn("prog '%s': no .text section found yet sub-program call exists\n",
6137                                 prog->name);
6138                         return -LIBBPF_ERRNO__RELOC;
6139                 }
6140
6141                 /* if it's the first call instruction calling into this
6142                  * subprogram (meaning this subprog hasn't been processed
6143                  * yet) within the context of current main program:
6144                  *   - append it at the end of main program's instructions blog;
6145                  *   - process is recursively, while current program is put on hold;
6146                  *   - if that subprogram calls some other not yet processes
6147                  *   subprogram, same thing will happen recursively until
6148                  *   there are no more unprocesses subprograms left to append
6149                  *   and relocate.
6150                  */
6151                 if (subprog->sub_insn_off == 0) {
6152                         subprog->sub_insn_off = main_prog->insns_cnt;
6153
6154                         new_cnt = main_prog->insns_cnt + subprog->insns_cnt;
6155                         insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns));
6156                         if (!insns) {
6157                                 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name);
6158                                 return -ENOMEM;
6159                         }
6160                         main_prog->insns = insns;
6161                         main_prog->insns_cnt = new_cnt;
6162
6163                         memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns,
6164                                subprog->insns_cnt * sizeof(*insns));
6165
6166                         pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n",
6167                                  main_prog->name, subprog->insns_cnt, subprog->name);
6168
6169                         /* The subprog insns are now appended. Append its relos too. */
6170                         err = append_subprog_relos(main_prog, subprog);
6171                         if (err)
6172                                 return err;
6173                         err = bpf_object__reloc_code(obj, main_prog, subprog);
6174                         if (err)
6175                                 return err;
6176                 }
6177
6178                 /* main_prog->insns memory could have been re-allocated, so
6179                  * calculate pointer again
6180                  */
6181                 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6182                 /* calculate correct instruction position within current main
6183                  * prog; each main prog can have a different set of
6184                  * subprograms appended (potentially in different order as
6185                  * well), so position of any subprog can be different for
6186                  * different main programs */
6187                 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1;
6188
6189                 pr_debug("prog '%s': insn #%zu relocated, imm %d points to subprog '%s' (now at %zu offset)\n",
6190                          prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off);
6191         }
6192
6193         return 0;
6194 }
6195
6196 /*
6197  * Relocate sub-program calls.
6198  *
6199  * Algorithm operates as follows. Each entry-point BPF program (referred to as
6200  * main prog) is processed separately. For each subprog (non-entry functions,
6201  * that can be called from either entry progs or other subprogs) gets their
6202  * sub_insn_off reset to zero. This serves as indicator that this subprogram
6203  * hasn't been yet appended and relocated within current main prog. Once its
6204  * relocated, sub_insn_off will point at the position within current main prog
6205  * where given subprog was appended. This will further be used to relocate all
6206  * the call instructions jumping into this subprog.
6207  *
6208  * We start with main program and process all call instructions. If the call
6209  * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off
6210  * is zero), subprog instructions are appended at the end of main program's
6211  * instruction array. Then main program is "put on hold" while we recursively
6212  * process newly appended subprogram. If that subprogram calls into another
6213  * subprogram that hasn't been appended, new subprogram is appended again to
6214  * the *main* prog's instructions (subprog's instructions are always left
6215  * untouched, as they need to be in unmodified state for subsequent main progs
6216  * and subprog instructions are always sent only as part of a main prog) and
6217  * the process continues recursively. Once all the subprogs called from a main
6218  * prog or any of its subprogs are appended (and relocated), all their
6219  * positions within finalized instructions array are known, so it's easy to
6220  * rewrite call instructions with correct relative offsets, corresponding to
6221  * desired target subprog.
6222  *
6223  * Its important to realize that some subprogs might not be called from some
6224  * main prog and any of its called/used subprogs. Those will keep their
6225  * subprog->sub_insn_off as zero at all times and won't be appended to current
6226  * main prog and won't be relocated within the context of current main prog.
6227  * They might still be used from other main progs later.
6228  *
6229  * Visually this process can be shown as below. Suppose we have two main
6230  * programs mainA and mainB and BPF object contains three subprogs: subA,
6231  * subB, and subC. mainA calls only subA, mainB calls only subC, but subA and
6232  * subC both call subB:
6233  *
6234  *        +--------+ +-------+
6235  *        |        v v       |
6236  *     +--+---+ +--+-+-+ +---+--+
6237  *     | subA | | subB | | subC |
6238  *     +--+---+ +------+ +---+--+
6239  *        ^                  ^
6240  *        |                  |
6241  *    +---+-------+   +------+----+
6242  *    |   mainA   |   |   mainB   |
6243  *    +-----------+   +-----------+
6244  *
6245  * We'll start relocating mainA, will find subA, append it and start
6246  * processing sub A recursively:
6247  *
6248  *    +-----------+------+
6249  *    |   mainA   | subA |
6250  *    +-----------+------+
6251  *
6252  * At this point we notice that subB is used from subA, so we append it and
6253  * relocate (there are no further subcalls from subB):
6254  *
6255  *    +-----------+------+------+
6256  *    |   mainA   | subA | subB |
6257  *    +-----------+------+------+
6258  *
6259  * At this point, we relocate subA calls, then go one level up and finish with
6260  * relocatin mainA calls. mainA is done.
6261  *
6262  * For mainB process is similar but results in different order. We start with
6263  * mainB and skip subA and subB, as mainB never calls them (at least
6264  * directly), but we see subC is needed, so we append and start processing it:
6265  *
6266  *    +-----------+------+
6267  *    |   mainB   | subC |
6268  *    +-----------+------+
6269  * Now we see subC needs subB, so we go back to it, append and relocate it:
6270  *
6271  *    +-----------+------+------+
6272  *    |   mainB   | subC | subB |
6273  *    +-----------+------+------+
6274  *
6275  * At this point we unwind recursion, relocate calls in subC, then in mainB.
6276  */
6277 static int
6278 bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog)
6279 {
6280         struct bpf_program *subprog;
6281         int i, err;
6282
6283         /* mark all subprogs as not relocated (yet) within the context of
6284          * current main program
6285          */
6286         for (i = 0; i < obj->nr_programs; i++) {
6287                 subprog = &obj->programs[i];
6288                 if (!prog_is_subprog(obj, subprog))
6289                         continue;
6290
6291                 subprog->sub_insn_off = 0;
6292         }
6293
6294         err = bpf_object__reloc_code(obj, prog, prog);
6295         if (err)
6296                 return err;
6297
6298         return 0;
6299 }
6300
6301 static void
6302 bpf_object__free_relocs(struct bpf_object *obj)
6303 {
6304         struct bpf_program *prog;
6305         int i;
6306
6307         /* free up relocation descriptors */
6308         for (i = 0; i < obj->nr_programs; i++) {
6309                 prog = &obj->programs[i];
6310                 zfree(&prog->reloc_desc);
6311                 prog->nr_reloc = 0;
6312         }
6313 }
6314
6315 static int cmp_relocs(const void *_a, const void *_b)
6316 {
6317         const struct reloc_desc *a = _a;
6318         const struct reloc_desc *b = _b;
6319
6320         if (a->insn_idx != b->insn_idx)
6321                 return a->insn_idx < b->insn_idx ? -1 : 1;
6322
6323         /* no two relocations should have the same insn_idx, but ... */
6324         if (a->type != b->type)
6325                 return a->type < b->type ? -1 : 1;
6326
6327         return 0;
6328 }
6329
6330 static void bpf_object__sort_relos(struct bpf_object *obj)
6331 {
6332         int i;
6333
6334         for (i = 0; i < obj->nr_programs; i++) {
6335                 struct bpf_program *p = &obj->programs[i];
6336
6337                 if (!p->nr_reloc)
6338                         continue;
6339
6340                 qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs);
6341         }
6342 }
6343
6344 static int
6345 bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path)
6346 {
6347         struct bpf_program *prog;
6348         size_t i, j;
6349         int err;
6350
6351         if (obj->btf_ext) {
6352                 err = bpf_object__relocate_core(obj, targ_btf_path);
6353                 if (err) {
6354                         pr_warn("failed to perform CO-RE relocations: %d\n",
6355                                 err);
6356                         return err;
6357                 }
6358                 bpf_object__sort_relos(obj);
6359         }
6360
6361         /* Before relocating calls pre-process relocations and mark
6362          * few ld_imm64 instructions that points to subprogs.
6363          * Otherwise bpf_object__reloc_code() later would have to consider
6364          * all ld_imm64 insns as relocation candidates. That would
6365          * reduce relocation speed, since amount of find_prog_insn_relo()
6366          * would increase and most of them will fail to find a relo.
6367          */
6368         for (i = 0; i < obj->nr_programs; i++) {
6369                 prog = &obj->programs[i];
6370                 for (j = 0; j < prog->nr_reloc; j++) {
6371                         struct reloc_desc *relo = &prog->reloc_desc[j];
6372                         struct bpf_insn *insn = &prog->insns[relo->insn_idx];
6373
6374                         /* mark the insn, so it's recognized by insn_is_pseudo_func() */
6375                         if (relo->type == RELO_SUBPROG_ADDR)
6376                                 insn[0].src_reg = BPF_PSEUDO_FUNC;
6377                 }
6378         }
6379
6380         /* relocate subprogram calls and append used subprograms to main
6381          * programs; each copy of subprogram code needs to be relocated
6382          * differently for each main program, because its code location might
6383          * have changed.
6384          * Append subprog relos to main programs to allow data relos to be
6385          * processed after text is completely relocated.
6386          */
6387         for (i = 0; i < obj->nr_programs; i++) {
6388                 prog = &obj->programs[i];
6389                 /* sub-program's sub-calls are relocated within the context of
6390                  * its main program only
6391                  */
6392                 if (prog_is_subprog(obj, prog))
6393                         continue;
6394                 if (!prog->autoload)
6395                         continue;
6396
6397                 err = bpf_object__relocate_calls(obj, prog);
6398                 if (err) {
6399                         pr_warn("prog '%s': failed to relocate calls: %d\n",
6400                                 prog->name, err);
6401                         return err;
6402                 }
6403         }
6404         /* Process data relos for main programs */
6405         for (i = 0; i < obj->nr_programs; i++) {
6406                 prog = &obj->programs[i];
6407                 if (prog_is_subprog(obj, prog))
6408                         continue;
6409                 if (!prog->autoload)
6410                         continue;
6411                 err = bpf_object__relocate_data(obj, prog);
6412                 if (err) {
6413                         pr_warn("prog '%s': failed to relocate data references: %d\n",
6414                                 prog->name, err);
6415                         return err;
6416                 }
6417         }
6418
6419         return 0;
6420 }
6421
6422 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
6423                                             Elf64_Shdr *shdr, Elf_Data *data);
6424
6425 static int bpf_object__collect_map_relos(struct bpf_object *obj,
6426                                          Elf64_Shdr *shdr, Elf_Data *data)
6427 {
6428         const int bpf_ptr_sz = 8, host_ptr_sz = sizeof(void *);
6429         int i, j, nrels, new_sz;
6430         const struct btf_var_secinfo *vi = NULL;
6431         const struct btf_type *sec, *var, *def;
6432         struct bpf_map *map = NULL, *targ_map = NULL;
6433         struct bpf_program *targ_prog = NULL;
6434         bool is_prog_array, is_map_in_map;
6435         const struct btf_member *member;
6436         const char *name, *mname, *type;
6437         unsigned int moff;
6438         Elf64_Sym *sym;
6439         Elf64_Rel *rel;
6440         void *tmp;
6441
6442         if (!obj->efile.btf_maps_sec_btf_id || !obj->btf)
6443                 return -EINVAL;
6444         sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id);
6445         if (!sec)
6446                 return -EINVAL;
6447
6448         nrels = shdr->sh_size / shdr->sh_entsize;
6449         for (i = 0; i < nrels; i++) {
6450                 rel = elf_rel_by_idx(data, i);
6451                 if (!rel) {
6452                         pr_warn(".maps relo #%d: failed to get ELF relo\n", i);
6453                         return -LIBBPF_ERRNO__FORMAT;
6454                 }
6455
6456                 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
6457                 if (!sym) {
6458                         pr_warn(".maps relo #%d: symbol %zx not found\n",
6459                                 i, (size_t)ELF64_R_SYM(rel->r_info));
6460                         return -LIBBPF_ERRNO__FORMAT;
6461                 }
6462                 name = elf_sym_str(obj, sym->st_name) ?: "<?>";
6463
6464                 pr_debug(".maps relo #%d: for %zd value %zd rel->r_offset %zu name %d ('%s')\n",
6465                          i, (ssize_t)(rel->r_info >> 32), (size_t)sym->st_value,
6466                          (size_t)rel->r_offset, sym->st_name, name);
6467
6468                 for (j = 0; j < obj->nr_maps; j++) {
6469                         map = &obj->maps[j];
6470                         if (map->sec_idx != obj->efile.btf_maps_shndx)
6471                                 continue;
6472
6473                         vi = btf_var_secinfos(sec) + map->btf_var_idx;
6474                         if (vi->offset <= rel->r_offset &&
6475                             rel->r_offset + bpf_ptr_sz <= vi->offset + vi->size)
6476                                 break;
6477                 }
6478                 if (j == obj->nr_maps) {
6479                         pr_warn(".maps relo #%d: cannot find map '%s' at rel->r_offset %zu\n",
6480                                 i, name, (size_t)rel->r_offset);
6481                         return -EINVAL;
6482                 }
6483
6484                 is_map_in_map = bpf_map_type__is_map_in_map(map->def.type);
6485                 is_prog_array = map->def.type == BPF_MAP_TYPE_PROG_ARRAY;
6486                 type = is_map_in_map ? "map" : "prog";
6487                 if (is_map_in_map) {
6488                         if (sym->st_shndx != obj->efile.btf_maps_shndx) {
6489                                 pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n",
6490                                         i, name);
6491                                 return -LIBBPF_ERRNO__RELOC;
6492                         }
6493                         if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS &&
6494                             map->def.key_size != sizeof(int)) {
6495                                 pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n",
6496                                         i, map->name, sizeof(int));
6497                                 return -EINVAL;
6498                         }
6499                         targ_map = bpf_object__find_map_by_name(obj, name);
6500                         if (!targ_map) {
6501                                 pr_warn(".maps relo #%d: '%s' isn't a valid map reference\n",
6502                                         i, name);
6503                                 return -ESRCH;
6504                         }
6505                 } else if (is_prog_array) {
6506                         targ_prog = bpf_object__find_program_by_name(obj, name);
6507                         if (!targ_prog) {
6508                                 pr_warn(".maps relo #%d: '%s' isn't a valid program reference\n",
6509                                         i, name);
6510                                 return -ESRCH;
6511                         }
6512                         if (targ_prog->sec_idx != sym->st_shndx ||
6513                             targ_prog->sec_insn_off * 8 != sym->st_value ||
6514                             prog_is_subprog(obj, targ_prog)) {
6515                                 pr_warn(".maps relo #%d: '%s' isn't an entry-point program\n",
6516                                         i, name);
6517                                 return -LIBBPF_ERRNO__RELOC;
6518                         }
6519                 } else {
6520                         return -EINVAL;
6521                 }
6522
6523                 var = btf__type_by_id(obj->btf, vi->type);
6524                 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
6525                 if (btf_vlen(def) == 0)
6526                         return -EINVAL;
6527                 member = btf_members(def) + btf_vlen(def) - 1;
6528                 mname = btf__name_by_offset(obj->btf, member->name_off);
6529                 if (strcmp(mname, "values"))
6530                         return -EINVAL;
6531
6532                 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8;
6533                 if (rel->r_offset - vi->offset < moff)
6534                         return -EINVAL;
6535
6536                 moff = rel->r_offset - vi->offset - moff;
6537                 /* here we use BPF pointer size, which is always 64 bit, as we
6538                  * are parsing ELF that was built for BPF target
6539                  */
6540                 if (moff % bpf_ptr_sz)
6541                         return -EINVAL;
6542                 moff /= bpf_ptr_sz;
6543                 if (moff >= map->init_slots_sz) {
6544                         new_sz = moff + 1;
6545                         tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz);
6546                         if (!tmp)
6547                                 return -ENOMEM;
6548                         map->init_slots = tmp;
6549                         memset(map->init_slots + map->init_slots_sz, 0,
6550                                (new_sz - map->init_slots_sz) * host_ptr_sz);
6551                         map->init_slots_sz = new_sz;
6552                 }
6553                 map->init_slots[moff] = is_map_in_map ? (void *)targ_map : (void *)targ_prog;
6554
6555                 pr_debug(".maps relo #%d: map '%s' slot [%d] points to %s '%s'\n",
6556                          i, map->name, moff, type, name);
6557         }
6558
6559         return 0;
6560 }
6561
6562 static int bpf_object__collect_relos(struct bpf_object *obj)
6563 {
6564         int i, err;
6565
6566         for (i = 0; i < obj->efile.sec_cnt; i++) {
6567                 struct elf_sec_desc *sec_desc = &obj->efile.secs[i];
6568                 Elf64_Shdr *shdr;
6569                 Elf_Data *data;
6570                 int idx;
6571
6572                 if (sec_desc->sec_type != SEC_RELO)
6573                         continue;
6574
6575                 shdr = sec_desc->shdr;
6576                 data = sec_desc->data;
6577                 idx = shdr->sh_info;
6578
6579                 if (shdr->sh_type != SHT_REL) {
6580                         pr_warn("internal error at %d\n", __LINE__);
6581                         return -LIBBPF_ERRNO__INTERNAL;
6582                 }
6583
6584                 if (idx == obj->efile.st_ops_shndx)
6585                         err = bpf_object__collect_st_ops_relos(obj, shdr, data);
6586                 else if (idx == obj->efile.btf_maps_shndx)
6587                         err = bpf_object__collect_map_relos(obj, shdr, data);
6588                 else
6589                         err = bpf_object__collect_prog_relos(obj, shdr, data);
6590                 if (err)
6591                         return err;
6592         }
6593
6594         bpf_object__sort_relos(obj);
6595         return 0;
6596 }
6597
6598 static bool insn_is_helper_call(struct bpf_insn *insn, enum bpf_func_id *func_id)
6599 {
6600         if (BPF_CLASS(insn->code) == BPF_JMP &&
6601             BPF_OP(insn->code) == BPF_CALL &&
6602             BPF_SRC(insn->code) == BPF_K &&
6603             insn->src_reg == 0 &&
6604             insn->dst_reg == 0) {
6605                     *func_id = insn->imm;
6606                     return true;
6607         }
6608         return false;
6609 }
6610
6611 static int bpf_object__sanitize_prog(struct bpf_object *obj, struct bpf_program *prog)
6612 {
6613         struct bpf_insn *insn = prog->insns;
6614         enum bpf_func_id func_id;
6615         int i;
6616
6617         if (obj->gen_loader)
6618                 return 0;
6619
6620         for (i = 0; i < prog->insns_cnt; i++, insn++) {
6621                 if (!insn_is_helper_call(insn, &func_id))
6622                         continue;
6623
6624                 /* on kernels that don't yet support
6625                  * bpf_probe_read_{kernel,user}[_str] helpers, fall back
6626                  * to bpf_probe_read() which works well for old kernels
6627                  */
6628                 switch (func_id) {
6629                 case BPF_FUNC_probe_read_kernel:
6630                 case BPF_FUNC_probe_read_user:
6631                         if (!kernel_supports(obj, FEAT_PROBE_READ_KERN))
6632                                 insn->imm = BPF_FUNC_probe_read;
6633                         break;
6634                 case BPF_FUNC_probe_read_kernel_str:
6635                 case BPF_FUNC_probe_read_user_str:
6636                         if (!kernel_supports(obj, FEAT_PROBE_READ_KERN))
6637                                 insn->imm = BPF_FUNC_probe_read_str;
6638                         break;
6639                 default:
6640                         break;
6641                 }
6642         }
6643         return 0;
6644 }
6645
6646 static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name,
6647                                      int *btf_obj_fd, int *btf_type_id);
6648
6649 /* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */
6650 static int libbpf_prepare_prog_load(struct bpf_program *prog,
6651                                     struct bpf_prog_load_opts *opts, long cookie)
6652 {
6653         enum sec_def_flags def = cookie;
6654
6655         /* old kernels might not support specifying expected_attach_type */
6656         if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE))
6657                 opts->expected_attach_type = 0;
6658
6659         if (def & SEC_SLEEPABLE)
6660                 opts->prog_flags |= BPF_F_SLEEPABLE;
6661
6662         if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS))
6663                 opts->prog_flags |= BPF_F_XDP_HAS_FRAGS;
6664
6665         if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {
6666                 int btf_obj_fd = 0, btf_type_id = 0, err;
6667                 const char *attach_name;
6668
6669                 attach_name = strchr(prog->sec_name, '/');
6670                 if (!attach_name) {
6671                         /* if BPF program is annotated with just SEC("fentry")
6672                          * (or similar) without declaratively specifying
6673                          * target, then it is expected that target will be
6674                          * specified with bpf_program__set_attach_target() at
6675                          * runtime before BPF object load step. If not, then
6676                          * there is nothing to load into the kernel as BPF
6677                          * verifier won't be able to validate BPF program
6678                          * correctness anyways.
6679                          */
6680                         pr_warn("prog '%s': no BTF-based attach target is specified, use bpf_program__set_attach_target()\n",
6681                                 prog->name);
6682                         return -EINVAL;
6683                 }
6684                 attach_name++; /* skip over / */
6685
6686                 err = libbpf_find_attach_btf_id(prog, attach_name, &btf_obj_fd, &btf_type_id);
6687                 if (err)
6688                         return err;
6689
6690                 /* cache resolved BTF FD and BTF type ID in the prog */
6691                 prog->attach_btf_obj_fd = btf_obj_fd;
6692                 prog->attach_btf_id = btf_type_id;
6693
6694                 /* but by now libbpf common logic is not utilizing
6695                  * prog->atach_btf_obj_fd/prog->attach_btf_id anymore because
6696                  * this callback is called after opts were populated by
6697                  * libbpf, so this callback has to update opts explicitly here
6698                  */
6699                 opts->attach_btf_obj_fd = btf_obj_fd;
6700                 opts->attach_btf_id = btf_type_id;
6701         }
6702         return 0;
6703 }
6704
6705 static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz);
6706
6707 static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog,
6708                                 struct bpf_insn *insns, int insns_cnt,
6709                                 const char *license, __u32 kern_version, int *prog_fd)
6710 {
6711         LIBBPF_OPTS(bpf_prog_load_opts, load_attr);
6712         const char *prog_name = NULL;
6713         char *cp, errmsg[STRERR_BUFSIZE];
6714         size_t log_buf_size = 0;
6715         char *log_buf = NULL, *tmp;
6716         int btf_fd, ret, err;
6717         bool own_log_buf = true;
6718         __u32 log_level = prog->log_level;
6719
6720         if (prog->type == BPF_PROG_TYPE_UNSPEC) {
6721                 /*
6722                  * The program type must be set.  Most likely we couldn't find a proper
6723                  * section definition at load time, and thus we didn't infer the type.
6724                  */
6725                 pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n",
6726                         prog->name, prog->sec_name);
6727                 return -EINVAL;
6728         }
6729
6730         if (!insns || !insns_cnt)
6731                 return -EINVAL;
6732
6733         load_attr.expected_attach_type = prog->expected_attach_type;
6734         if (kernel_supports(obj, FEAT_PROG_NAME))
6735                 prog_name = prog->name;
6736         load_attr.attach_prog_fd = prog->attach_prog_fd;
6737         load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
6738         load_attr.attach_btf_id = prog->attach_btf_id;
6739         load_attr.kern_version = kern_version;
6740         load_attr.prog_ifindex = prog->prog_ifindex;
6741
6742         /* specify func_info/line_info only if kernel supports them */
6743         btf_fd = bpf_object__btf_fd(obj);
6744         if (btf_fd >= 0 && kernel_supports(obj, FEAT_BTF_FUNC)) {
6745                 load_attr.prog_btf_fd = btf_fd;
6746                 load_attr.func_info = prog->func_info;
6747                 load_attr.func_info_rec_size = prog->func_info_rec_size;
6748                 load_attr.func_info_cnt = prog->func_info_cnt;
6749                 load_attr.line_info = prog->line_info;
6750                 load_attr.line_info_rec_size = prog->line_info_rec_size;
6751                 load_attr.line_info_cnt = prog->line_info_cnt;
6752         }
6753         load_attr.log_level = log_level;
6754         load_attr.prog_flags = prog->prog_flags;
6755         load_attr.fd_array = obj->fd_array;
6756
6757         /* adjust load_attr if sec_def provides custom preload callback */
6758         if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) {
6759                 err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie);
6760                 if (err < 0) {
6761                         pr_warn("prog '%s': failed to prepare load attributes: %d\n",
6762                                 prog->name, err);
6763                         return err;
6764                 }
6765                 insns = prog->insns;
6766                 insns_cnt = prog->insns_cnt;
6767         }
6768
6769         if (obj->gen_loader) {
6770                 bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,
6771                                    license, insns, insns_cnt, &load_attr,
6772                                    prog - obj->programs);
6773                 *prog_fd = -1;
6774                 return 0;
6775         }
6776
6777 retry_load:
6778         /* if log_level is zero, we don't request logs initially even if
6779          * custom log_buf is specified; if the program load fails, then we'll
6780          * bump log_level to 1 and use either custom log_buf or we'll allocate
6781          * our own and retry the load to get details on what failed
6782          */
6783         if (log_level) {
6784                 if (prog->log_buf) {
6785                         log_buf = prog->log_buf;
6786                         log_buf_size = prog->log_size;
6787                         own_log_buf = false;
6788                 } else if (obj->log_buf) {
6789                         log_buf = obj->log_buf;
6790                         log_buf_size = obj->log_size;
6791                         own_log_buf = false;
6792                 } else {
6793                         log_buf_size = max((size_t)BPF_LOG_BUF_SIZE, log_buf_size * 2);
6794                         tmp = realloc(log_buf, log_buf_size);
6795                         if (!tmp) {
6796                                 ret = -ENOMEM;
6797                                 goto out;
6798                         }
6799                         log_buf = tmp;
6800                         log_buf[0] = '\0';
6801                         own_log_buf = true;
6802                 }
6803         }
6804
6805         load_attr.log_buf = log_buf;
6806         load_attr.log_size = log_buf_size;
6807         load_attr.log_level = log_level;
6808
6809         ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr);
6810         if (ret >= 0) {
6811                 if (log_level && own_log_buf) {
6812                         pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
6813                                  prog->name, log_buf);
6814                 }
6815
6816                 if (obj->has_rodata && kernel_supports(obj, FEAT_PROG_BIND_MAP)) {
6817                         struct bpf_map *map;
6818                         int i;
6819
6820                         for (i = 0; i < obj->nr_maps; i++) {
6821                                 map = &prog->obj->maps[i];
6822                                 if (map->libbpf_type != LIBBPF_MAP_RODATA)
6823                                         continue;
6824
6825                                 if (bpf_prog_bind_map(ret, bpf_map__fd(map), NULL)) {
6826                                         cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
6827                                         pr_warn("prog '%s': failed to bind map '%s': %s\n",
6828                                                 prog->name, map->real_name, cp);
6829                                         /* Don't fail hard if can't bind rodata. */
6830                                 }
6831                         }
6832                 }
6833
6834                 *prog_fd = ret;
6835                 ret = 0;
6836                 goto out;
6837         }
6838
6839         if (log_level == 0) {
6840                 log_level = 1;
6841                 goto retry_load;
6842         }
6843         /* On ENOSPC, increase log buffer size and retry, unless custom
6844          * log_buf is specified.
6845          * Be careful to not overflow u32, though. Kernel's log buf size limit
6846          * isn't part of UAPI so it can always be bumped to full 4GB. So don't
6847          * multiply by 2 unless we are sure we'll fit within 32 bits.
6848          * Currently, we'll get -EINVAL when we reach (UINT_MAX >> 2).
6849          */
6850         if (own_log_buf && errno == ENOSPC && log_buf_size <= UINT_MAX / 2)
6851                 goto retry_load;
6852
6853         ret = -errno;
6854
6855         /* post-process verifier log to improve error descriptions */
6856         fixup_verifier_log(prog, log_buf, log_buf_size);
6857
6858         cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
6859         pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, cp);
6860         pr_perm_msg(ret);
6861
6862         if (own_log_buf && log_buf && log_buf[0] != '\0') {
6863                 pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
6864                         prog->name, log_buf);
6865         }
6866
6867 out:
6868         if (own_log_buf)
6869                 free(log_buf);
6870         return ret;
6871 }
6872
6873 static char *find_prev_line(char *buf, char *cur)
6874 {
6875         char *p;
6876
6877         if (cur == buf) /* end of a log buf */
6878                 return NULL;
6879
6880         p = cur - 1;
6881         while (p - 1 >= buf && *(p - 1) != '\n')
6882                 p--;
6883
6884         return p;
6885 }
6886
6887 static void patch_log(char *buf, size_t buf_sz, size_t log_sz,
6888                       char *orig, size_t orig_sz, const char *patch)
6889 {
6890         /* size of the remaining log content to the right from the to-be-replaced part */
6891         size_t rem_sz = (buf + log_sz) - (orig + orig_sz);
6892         size_t patch_sz = strlen(patch);
6893
6894         if (patch_sz != orig_sz) {
6895                 /* If patch line(s) are longer than original piece of verifier log,
6896                  * shift log contents by (patch_sz - orig_sz) bytes to the right
6897                  * starting from after to-be-replaced part of the log.
6898                  *
6899                  * If patch line(s) are shorter than original piece of verifier log,
6900                  * shift log contents by (orig_sz - patch_sz) bytes to the left
6901                  * starting from after to-be-replaced part of the log
6902                  *
6903                  * We need to be careful about not overflowing available
6904                  * buf_sz capacity. If that's the case, we'll truncate the end
6905                  * of the original log, as necessary.
6906                  */
6907                 if (patch_sz > orig_sz) {
6908                         if (orig + patch_sz >= buf + buf_sz) {
6909                                 /* patch is big enough to cover remaining space completely */
6910                                 patch_sz -= (orig + patch_sz) - (buf + buf_sz) + 1;
6911                                 rem_sz = 0;
6912                         } else if (patch_sz - orig_sz > buf_sz - log_sz) {
6913                                 /* patch causes part of remaining log to be truncated */
6914                                 rem_sz -= (patch_sz - orig_sz) - (buf_sz - log_sz);
6915                         }
6916                 }
6917                 /* shift remaining log to the right by calculated amount */
6918                 memmove(orig + patch_sz, orig + orig_sz, rem_sz);
6919         }
6920
6921         memcpy(orig, patch, patch_sz);
6922 }
6923
6924 static void fixup_log_failed_core_relo(struct bpf_program *prog,
6925                                        char *buf, size_t buf_sz, size_t log_sz,
6926                                        char *line1, char *line2, char *line3)
6927 {
6928         /* Expected log for failed and not properly guarded CO-RE relocation:
6929          * line1 -> 123: (85) call unknown#195896080
6930          * line2 -> invalid func unknown#195896080
6931          * line3 -> <anything else or end of buffer>
6932          *
6933          * "123" is the index of the instruction that was poisoned. We extract
6934          * instruction index to find corresponding CO-RE relocation and
6935          * replace this part of the log with more relevant information about
6936          * failed CO-RE relocation.
6937          */
6938         const struct bpf_core_relo *relo;
6939         struct bpf_core_spec spec;
6940         char patch[512], spec_buf[256];
6941         int insn_idx, err, spec_len;
6942
6943         if (sscanf(line1, "%d: (%*d) call unknown#195896080\n", &insn_idx) != 1)
6944                 return;
6945
6946         relo = find_relo_core(prog, insn_idx);
6947         if (!relo)
6948                 return;
6949
6950         err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec);
6951         if (err)
6952                 return;
6953
6954         spec_len = bpf_core_format_spec(spec_buf, sizeof(spec_buf), &spec);
6955         snprintf(patch, sizeof(patch),
6956                  "%d: <invalid CO-RE relocation>\n"
6957                  "failed to resolve CO-RE relocation %s%s\n",
6958                  insn_idx, spec_buf, spec_len >= sizeof(spec_buf) ? "..." : "");
6959
6960         patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
6961 }
6962
6963 static void fixup_log_missing_map_load(struct bpf_program *prog,
6964                                        char *buf, size_t buf_sz, size_t log_sz,
6965                                        char *line1, char *line2, char *line3)
6966 {
6967         /* Expected log for failed and not properly guarded CO-RE relocation:
6968          * line1 -> 123: (85) call unknown#2001000345
6969          * line2 -> invalid func unknown#2001000345
6970          * line3 -> <anything else or end of buffer>
6971          *
6972          * "123" is the index of the instruction that was poisoned.
6973          * "345" in "2001000345" are map index in obj->maps to fetch map name.
6974          */
6975         struct bpf_object *obj = prog->obj;
6976         const struct bpf_map *map;
6977         int insn_idx, map_idx;
6978         char patch[128];
6979
6980         if (sscanf(line1, "%d: (%*d) call unknown#%d\n", &insn_idx, &map_idx) != 2)
6981                 return;
6982
6983         map_idx -= MAP_LDIMM64_POISON_BASE;
6984         if (map_idx < 0 || map_idx >= obj->nr_maps)
6985                 return;
6986         map = &obj->maps[map_idx];
6987
6988         snprintf(patch, sizeof(patch),
6989                  "%d: <invalid BPF map reference>\n"
6990                  "BPF map '%s' is referenced but wasn't created\n",
6991                  insn_idx, map->name);
6992
6993         patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
6994 }
6995
6996 static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz)
6997 {
6998         /* look for familiar error patterns in last N lines of the log */
6999         const size_t max_last_line_cnt = 10;
7000         char *prev_line, *cur_line, *next_line;
7001         size_t log_sz;
7002         int i;
7003
7004         if (!buf)
7005                 return;
7006
7007         log_sz = strlen(buf) + 1;
7008         next_line = buf + log_sz - 1;
7009
7010         for (i = 0; i < max_last_line_cnt; i++, next_line = cur_line) {
7011                 cur_line = find_prev_line(buf, next_line);
7012                 if (!cur_line)
7013                         return;
7014
7015                 /* failed CO-RE relocation case */
7016                 if (str_has_pfx(cur_line, "invalid func unknown#195896080\n")) {
7017                         prev_line = find_prev_line(buf, cur_line);
7018                         if (!prev_line)
7019                                 continue;
7020
7021                         fixup_log_failed_core_relo(prog, buf, buf_sz, log_sz,
7022                                                    prev_line, cur_line, next_line);
7023                         return;
7024                 } else if (str_has_pfx(cur_line, "invalid func unknown#"MAP_LDIMM64_POISON_PFX)) {
7025                         prev_line = find_prev_line(buf, cur_line);
7026                         if (!prev_line)
7027                                 continue;
7028
7029                         fixup_log_missing_map_load(prog, buf, buf_sz, log_sz,
7030                                                    prev_line, cur_line, next_line);
7031                         return;
7032                 }
7033         }
7034 }
7035
7036 static int bpf_program_record_relos(struct bpf_program *prog)
7037 {
7038         struct bpf_object *obj = prog->obj;
7039         int i;
7040
7041         for (i = 0; i < prog->nr_reloc; i++) {
7042                 struct reloc_desc *relo = &prog->reloc_desc[i];
7043                 struct extern_desc *ext = &obj->externs[relo->sym_off];
7044
7045                 switch (relo->type) {
7046                 case RELO_EXTERN_VAR:
7047                         if (ext->type != EXT_KSYM)
7048                                 continue;
7049                         bpf_gen__record_extern(obj->gen_loader, ext->name,
7050                                                ext->is_weak, !ext->ksym.type_id,
7051                                                BTF_KIND_VAR, relo->insn_idx);
7052                         break;
7053                 case RELO_EXTERN_FUNC:
7054                         bpf_gen__record_extern(obj->gen_loader, ext->name,
7055                                                ext->is_weak, false, BTF_KIND_FUNC,
7056                                                relo->insn_idx);
7057                         break;
7058                 case RELO_CORE: {
7059                         struct bpf_core_relo cr = {
7060                                 .insn_off = relo->insn_idx * 8,
7061                                 .type_id = relo->core_relo->type_id,
7062                                 .access_str_off = relo->core_relo->access_str_off,
7063                                 .kind = relo->core_relo->kind,
7064                         };
7065
7066                         bpf_gen__record_relo_core(obj->gen_loader, &cr);
7067                         break;
7068                 }
7069                 default:
7070                         continue;
7071                 }
7072         }
7073         return 0;
7074 }
7075
7076 static int
7077 bpf_object__load_progs(struct bpf_object *obj, int log_level)
7078 {
7079         struct bpf_program *prog;
7080         size_t i;
7081         int err;
7082
7083         for (i = 0; i < obj->nr_programs; i++) {
7084                 prog = &obj->programs[i];
7085                 err = bpf_object__sanitize_prog(obj, prog);
7086                 if (err)
7087                         return err;
7088         }
7089
7090         for (i = 0; i < obj->nr_programs; i++) {
7091                 prog = &obj->programs[i];
7092                 if (prog_is_subprog(obj, prog))
7093                         continue;
7094                 if (!prog->autoload) {
7095                         pr_debug("prog '%s': skipped loading\n", prog->name);
7096                         continue;
7097                 }
7098                 prog->log_level |= log_level;
7099
7100                 if (obj->gen_loader)
7101                         bpf_program_record_relos(prog);
7102
7103                 err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt,
7104                                            obj->license, obj->kern_version, &prog->fd);
7105                 if (err) {
7106                         pr_warn("prog '%s': failed to load: %d\n", prog->name, err);
7107                         return err;
7108                 }
7109         }
7110
7111         bpf_object__free_relocs(obj);
7112         return 0;
7113 }
7114
7115 static const struct bpf_sec_def *find_sec_def(const char *sec_name);
7116
7117 static int bpf_object_init_progs(struct bpf_object *obj, const struct bpf_object_open_opts *opts)
7118 {
7119         struct bpf_program *prog;
7120         int err;
7121
7122         bpf_object__for_each_program(prog, obj) {
7123                 prog->sec_def = find_sec_def(prog->sec_name);
7124                 if (!prog->sec_def) {
7125                         /* couldn't guess, but user might manually specify */
7126                         pr_debug("prog '%s': unrecognized ELF section name '%s'\n",
7127                                 prog->name, prog->sec_name);
7128                         continue;
7129                 }
7130
7131                 prog->type = prog->sec_def->prog_type;
7132                 prog->expected_attach_type = prog->sec_def->expected_attach_type;
7133
7134                 /* sec_def can have custom callback which should be called
7135                  * after bpf_program is initialized to adjust its properties
7136                  */
7137                 if (prog->sec_def->prog_setup_fn) {
7138                         err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie);
7139                         if (err < 0) {
7140                                 pr_warn("prog '%s': failed to initialize: %d\n",
7141                                         prog->name, err);
7142                                 return err;
7143                         }
7144                 }
7145         }
7146
7147         return 0;
7148 }
7149
7150 static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, size_t obj_buf_sz,
7151                                           const struct bpf_object_open_opts *opts)
7152 {
7153         const char *obj_name, *kconfig, *btf_tmp_path;
7154         struct bpf_object *obj;
7155         char tmp_name[64];
7156         int err;
7157         char *log_buf;
7158         size_t log_size;
7159         __u32 log_level;
7160
7161         if (elf_version(EV_CURRENT) == EV_NONE) {
7162                 pr_warn("failed to init libelf for %s\n",
7163                         path ? : "(mem buf)");
7164                 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
7165         }
7166
7167         if (!OPTS_VALID(opts, bpf_object_open_opts))
7168                 return ERR_PTR(-EINVAL);
7169
7170         obj_name = OPTS_GET(opts, object_name, NULL);
7171         if (obj_buf) {
7172                 if (!obj_name) {
7173                         snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
7174                                  (unsigned long)obj_buf,
7175                                  (unsigned long)obj_buf_sz);
7176                         obj_name = tmp_name;
7177                 }
7178                 path = obj_name;
7179                 pr_debug("loading object '%s' from buffer\n", obj_name);
7180         }
7181
7182         log_buf = OPTS_GET(opts, kernel_log_buf, NULL);
7183         log_size = OPTS_GET(opts, kernel_log_size, 0);
7184         log_level = OPTS_GET(opts, kernel_log_level, 0);
7185         if (log_size > UINT_MAX)
7186                 return ERR_PTR(-EINVAL);
7187         if (log_size && !log_buf)
7188                 return ERR_PTR(-EINVAL);
7189
7190         obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name);
7191         if (IS_ERR(obj))
7192                 return obj;
7193
7194         obj->log_buf = log_buf;
7195         obj->log_size = log_size;
7196         obj->log_level = log_level;
7197
7198         btf_tmp_path = OPTS_GET(opts, btf_custom_path, NULL);
7199         if (btf_tmp_path) {
7200                 if (strlen(btf_tmp_path) >= PATH_MAX) {
7201                         err = -ENAMETOOLONG;
7202                         goto out;
7203                 }
7204                 obj->btf_custom_path = strdup(btf_tmp_path);
7205                 if (!obj->btf_custom_path) {
7206                         err = -ENOMEM;
7207                         goto out;
7208                 }
7209         }
7210
7211         kconfig = OPTS_GET(opts, kconfig, NULL);
7212         if (kconfig) {
7213                 obj->kconfig = strdup(kconfig);
7214                 if (!obj->kconfig) {
7215                         err = -ENOMEM;
7216                         goto out;
7217                 }
7218         }
7219
7220         err = bpf_object__elf_init(obj);
7221         err = err ? : bpf_object__check_endianness(obj);
7222         err = err ? : bpf_object__elf_collect(obj);
7223         err = err ? : bpf_object__collect_externs(obj);
7224         err = err ? : bpf_object__finalize_btf(obj);
7225         err = err ? : bpf_object__init_maps(obj, opts);
7226         err = err ? : bpf_object_init_progs(obj, opts);
7227         err = err ? : bpf_object__collect_relos(obj);
7228         if (err)
7229                 goto out;
7230
7231         bpf_object__elf_finish(obj);
7232
7233         return obj;
7234 out:
7235         bpf_object__close(obj);
7236         return ERR_PTR(err);
7237 }
7238
7239 struct bpf_object *
7240 bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts)
7241 {
7242         if (!path)
7243                 return libbpf_err_ptr(-EINVAL);
7244
7245         pr_debug("loading %s\n", path);
7246
7247         return libbpf_ptr(bpf_object_open(path, NULL, 0, opts));
7248 }
7249
7250 struct bpf_object *bpf_object__open(const char *path)
7251 {
7252         return bpf_object__open_file(path, NULL);
7253 }
7254
7255 struct bpf_object *
7256 bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,
7257                      const struct bpf_object_open_opts *opts)
7258 {
7259         if (!obj_buf || obj_buf_sz == 0)
7260                 return libbpf_err_ptr(-EINVAL);
7261
7262         return libbpf_ptr(bpf_object_open(NULL, obj_buf, obj_buf_sz, opts));
7263 }
7264
7265 static int bpf_object_unload(struct bpf_object *obj)
7266 {
7267         size_t i;
7268
7269         if (!obj)
7270                 return libbpf_err(-EINVAL);
7271
7272         for (i = 0; i < obj->nr_maps; i++) {
7273                 zclose(obj->maps[i].fd);
7274                 if (obj->maps[i].st_ops)
7275                         zfree(&obj->maps[i].st_ops->kern_vdata);
7276         }
7277
7278         for (i = 0; i < obj->nr_programs; i++)
7279                 bpf_program__unload(&obj->programs[i]);
7280
7281         return 0;
7282 }
7283
7284 static int bpf_object__sanitize_maps(struct bpf_object *obj)
7285 {
7286         struct bpf_map *m;
7287
7288         bpf_object__for_each_map(m, obj) {
7289                 if (!bpf_map__is_internal(m))
7290                         continue;
7291                 if (!kernel_supports(obj, FEAT_ARRAY_MMAP))
7292                         m->def.map_flags ^= BPF_F_MMAPABLE;
7293         }
7294
7295         return 0;
7296 }
7297
7298 int libbpf_kallsyms_parse(kallsyms_cb_t cb, void *ctx)
7299 {
7300         char sym_type, sym_name[500];
7301         unsigned long long sym_addr;
7302         int ret, err = 0;
7303         FILE *f;
7304
7305         f = fopen("/proc/kallsyms", "r");
7306         if (!f) {
7307                 err = -errno;
7308                 pr_warn("failed to open /proc/kallsyms: %d\n", err);
7309                 return err;
7310         }
7311
7312         while (true) {
7313                 ret = fscanf(f, "%llx %c %499s%*[^\n]\n",
7314                              &sym_addr, &sym_type, sym_name);
7315                 if (ret == EOF && feof(f))
7316                         break;
7317                 if (ret != 3) {
7318                         pr_warn("failed to read kallsyms entry: %d\n", ret);
7319                         err = -EINVAL;
7320                         break;
7321                 }
7322
7323                 err = cb(sym_addr, sym_type, sym_name, ctx);
7324                 if (err)
7325                         break;
7326         }
7327
7328         fclose(f);
7329         return err;
7330 }
7331
7332 static int kallsyms_cb(unsigned long long sym_addr, char sym_type,
7333                        const char *sym_name, void *ctx)
7334 {
7335         struct bpf_object *obj = ctx;
7336         const struct btf_type *t;
7337         struct extern_desc *ext;
7338
7339         ext = find_extern_by_name(obj, sym_name);
7340         if (!ext || ext->type != EXT_KSYM)
7341                 return 0;
7342
7343         t = btf__type_by_id(obj->btf, ext->btf_id);
7344         if (!btf_is_var(t))
7345                 return 0;
7346
7347         if (ext->is_set && ext->ksym.addr != sym_addr) {
7348                 pr_warn("extern (ksym) '%s': resolution is ambiguous: 0x%llx or 0x%llx\n",
7349                         sym_name, ext->ksym.addr, sym_addr);
7350                 return -EINVAL;
7351         }
7352         if (!ext->is_set) {
7353                 ext->is_set = true;
7354                 ext->ksym.addr = sym_addr;
7355                 pr_debug("extern (ksym) '%s': set to 0x%llx\n", sym_name, sym_addr);
7356         }
7357         return 0;
7358 }
7359
7360 static int bpf_object__read_kallsyms_file(struct bpf_object *obj)
7361 {
7362         return libbpf_kallsyms_parse(kallsyms_cb, obj);
7363 }
7364
7365 static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name,
7366                             __u16 kind, struct btf **res_btf,
7367                             struct module_btf **res_mod_btf)
7368 {
7369         struct module_btf *mod_btf;
7370         struct btf *btf;
7371         int i, id, err;
7372
7373         btf = obj->btf_vmlinux;
7374         mod_btf = NULL;
7375         id = btf__find_by_name_kind(btf, ksym_name, kind);
7376
7377         if (id == -ENOENT) {
7378                 err = load_module_btfs(obj);
7379                 if (err)
7380                         return err;
7381
7382                 for (i = 0; i < obj->btf_module_cnt; i++) {
7383                         /* we assume module_btf's BTF FD is always >0 */
7384                         mod_btf = &obj->btf_modules[i];
7385                         btf = mod_btf->btf;
7386                         id = btf__find_by_name_kind_own(btf, ksym_name, kind);
7387                         if (id != -ENOENT)
7388                                 break;
7389                 }
7390         }
7391         if (id <= 0)
7392                 return -ESRCH;
7393
7394         *res_btf = btf;
7395         *res_mod_btf = mod_btf;
7396         return id;
7397 }
7398
7399 static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj,
7400                                                struct extern_desc *ext)
7401 {
7402         const struct btf_type *targ_var, *targ_type;
7403         __u32 targ_type_id, local_type_id;
7404         struct module_btf *mod_btf = NULL;
7405         const char *targ_var_name;
7406         struct btf *btf = NULL;
7407         int id, err;
7408
7409         id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &mod_btf);
7410         if (id < 0) {
7411                 if (id == -ESRCH && ext->is_weak)
7412                         return 0;
7413                 pr_warn("extern (var ksym) '%s': not found in kernel BTF\n",
7414                         ext->name);
7415                 return id;
7416         }
7417
7418         /* find local type_id */
7419         local_type_id = ext->ksym.type_id;
7420
7421         /* find target type_id */
7422         targ_var = btf__type_by_id(btf, id);
7423         targ_var_name = btf__name_by_offset(btf, targ_var->name_off);
7424         targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id);
7425
7426         err = bpf_core_types_are_compat(obj->btf, local_type_id,
7427                                         btf, targ_type_id);
7428         if (err <= 0) {
7429                 const struct btf_type *local_type;
7430                 const char *targ_name, *local_name;
7431
7432                 local_type = btf__type_by_id(obj->btf, local_type_id);
7433                 local_name = btf__name_by_offset(obj->btf, local_type->name_off);
7434                 targ_name = btf__name_by_offset(btf, targ_type->name_off);
7435
7436                 pr_warn("extern (var ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n",
7437                         ext->name, local_type_id,
7438                         btf_kind_str(local_type), local_name, targ_type_id,
7439                         btf_kind_str(targ_type), targ_name);
7440                 return -EINVAL;
7441         }
7442
7443         ext->is_set = true;
7444         ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0;
7445         ext->ksym.kernel_btf_id = id;
7446         pr_debug("extern (var ksym) '%s': resolved to [%d] %s %s\n",
7447                  ext->name, id, btf_kind_str(targ_var), targ_var_name);
7448
7449         return 0;
7450 }
7451
7452 static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj,
7453                                                 struct extern_desc *ext)
7454 {
7455         int local_func_proto_id, kfunc_proto_id, kfunc_id;
7456         struct module_btf *mod_btf = NULL;
7457         const struct btf_type *kern_func;
7458         struct btf *kern_btf = NULL;
7459         int ret;
7460
7461         local_func_proto_id = ext->ksym.type_id;
7462
7463         kfunc_id = find_ksym_btf_id(obj, ext->name, BTF_KIND_FUNC, &kern_btf, &mod_btf);
7464         if (kfunc_id < 0) {
7465                 if (kfunc_id == -ESRCH && ext->is_weak)
7466                         return 0;
7467                 pr_warn("extern (func ksym) '%s': not found in kernel or module BTFs\n",
7468                         ext->name);
7469                 return kfunc_id;
7470         }
7471
7472         kern_func = btf__type_by_id(kern_btf, kfunc_id);
7473         kfunc_proto_id = kern_func->type;
7474
7475         ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id,
7476                                         kern_btf, kfunc_proto_id);
7477         if (ret <= 0) {
7478                 pr_warn("extern (func ksym) '%s': func_proto [%d] incompatible with kernel [%d]\n",
7479                         ext->name, local_func_proto_id, kfunc_proto_id);
7480                 return -EINVAL;
7481         }
7482
7483         /* set index for module BTF fd in fd_array, if unset */
7484         if (mod_btf && !mod_btf->fd_array_idx) {
7485                 /* insn->off is s16 */
7486                 if (obj->fd_array_cnt == INT16_MAX) {
7487                         pr_warn("extern (func ksym) '%s': module BTF fd index %d too big to fit in bpf_insn offset\n",
7488                                 ext->name, mod_btf->fd_array_idx);
7489                         return -E2BIG;
7490                 }
7491                 /* Cannot use index 0 for module BTF fd */
7492                 if (!obj->fd_array_cnt)
7493                         obj->fd_array_cnt = 1;
7494
7495                 ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int),
7496                                         obj->fd_array_cnt + 1);
7497                 if (ret)
7498                         return ret;
7499                 mod_btf->fd_array_idx = obj->fd_array_cnt;
7500                 /* we assume module BTF FD is always >0 */
7501                 obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd;
7502         }
7503
7504         ext->is_set = true;
7505         ext->ksym.kernel_btf_id = kfunc_id;
7506         ext->ksym.btf_fd_idx = mod_btf ? mod_btf->fd_array_idx : 0;
7507         pr_debug("extern (func ksym) '%s': resolved to kernel [%d]\n",
7508                  ext->name, kfunc_id);
7509
7510         return 0;
7511 }
7512
7513 static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj)
7514 {
7515         const struct btf_type *t;
7516         struct extern_desc *ext;
7517         int i, err;
7518
7519         for (i = 0; i < obj->nr_extern; i++) {
7520                 ext = &obj->externs[i];
7521                 if (ext->type != EXT_KSYM || !ext->ksym.type_id)
7522                         continue;
7523
7524                 if (obj->gen_loader) {
7525                         ext->is_set = true;
7526                         ext->ksym.kernel_btf_obj_fd = 0;
7527                         ext->ksym.kernel_btf_id = 0;
7528                         continue;
7529                 }
7530                 t = btf__type_by_id(obj->btf, ext->btf_id);
7531                 if (btf_is_var(t))
7532                         err = bpf_object__resolve_ksym_var_btf_id(obj, ext);
7533                 else
7534                         err = bpf_object__resolve_ksym_func_btf_id(obj, ext);
7535                 if (err)
7536                         return err;
7537         }
7538         return 0;
7539 }
7540
7541 static int bpf_object__resolve_externs(struct bpf_object *obj,
7542                                        const char *extra_kconfig)
7543 {
7544         bool need_config = false, need_kallsyms = false;
7545         bool need_vmlinux_btf = false;
7546         struct extern_desc *ext;
7547         void *kcfg_data = NULL;
7548         int err, i;
7549
7550         if (obj->nr_extern == 0)
7551                 return 0;
7552
7553         if (obj->kconfig_map_idx >= 0)
7554                 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped;
7555
7556         for (i = 0; i < obj->nr_extern; i++) {
7557                 ext = &obj->externs[i];
7558
7559                 if (ext->type == EXT_KSYM) {
7560                         if (ext->ksym.type_id)
7561                                 need_vmlinux_btf = true;
7562                         else
7563                                 need_kallsyms = true;
7564                         continue;
7565                 } else if (ext->type == EXT_KCFG) {
7566                         void *ext_ptr = kcfg_data + ext->kcfg.data_off;
7567                         __u64 value = 0;
7568
7569                         /* Kconfig externs need actual /proc/config.gz */
7570                         if (str_has_pfx(ext->name, "CONFIG_")) {
7571                                 need_config = true;
7572                                 continue;
7573                         }
7574
7575                         /* Virtual kcfg externs are customly handled by libbpf */
7576                         if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) {
7577                                 value = get_kernel_version();
7578                                 if (!value) {
7579                                         pr_warn("extern (kcfg) '%s': failed to get kernel version\n", ext->name);
7580                                         return -EINVAL;
7581                                 }
7582                         } else if (strcmp(ext->name, "LINUX_HAS_BPF_COOKIE") == 0) {
7583                                 value = kernel_supports(obj, FEAT_BPF_COOKIE);
7584                         } else if (strcmp(ext->name, "LINUX_HAS_SYSCALL_WRAPPER") == 0) {
7585                                 value = kernel_supports(obj, FEAT_SYSCALL_WRAPPER);
7586                         } else if (!str_has_pfx(ext->name, "LINUX_") || !ext->is_weak) {
7587                                 /* Currently libbpf supports only CONFIG_ and LINUX_ prefixed
7588                                  * __kconfig externs, where LINUX_ ones are virtual and filled out
7589                                  * customly by libbpf (their values don't come from Kconfig).
7590                                  * If LINUX_xxx variable is not recognized by libbpf, but is marked
7591                                  * __weak, it defaults to zero value, just like for CONFIG_xxx
7592                                  * externs.
7593                                  */
7594                                 pr_warn("extern (kcfg) '%s': unrecognized virtual extern\n", ext->name);
7595                                 return -EINVAL;
7596                         }
7597
7598                         err = set_kcfg_value_num(ext, ext_ptr, value);
7599                         if (err)
7600                                 return err;
7601                         pr_debug("extern (kcfg) '%s': set to 0x%llx\n",
7602                                  ext->name, (long long)value);
7603                 } else {
7604                         pr_warn("extern '%s': unrecognized extern kind\n", ext->name);
7605                         return -EINVAL;
7606                 }
7607         }
7608         if (need_config && extra_kconfig) {
7609                 err = bpf_object__read_kconfig_mem(obj, extra_kconfig, kcfg_data);
7610                 if (err)
7611                         return -EINVAL;
7612                 need_config = false;
7613                 for (i = 0; i < obj->nr_extern; i++) {
7614                         ext = &obj->externs[i];
7615                         if (ext->type == EXT_KCFG && !ext->is_set) {
7616                                 need_config = true;
7617                                 break;
7618                         }
7619                 }
7620         }
7621         if (need_config) {
7622                 err = bpf_object__read_kconfig_file(obj, kcfg_data);
7623                 if (err)
7624                         return -EINVAL;
7625         }
7626         if (need_kallsyms) {
7627                 err = bpf_object__read_kallsyms_file(obj);
7628                 if (err)
7629                         return -EINVAL;
7630         }
7631         if (need_vmlinux_btf) {
7632                 err = bpf_object__resolve_ksyms_btf_id(obj);
7633                 if (err)
7634                         return -EINVAL;
7635         }
7636         for (i = 0; i < obj->nr_extern; i++) {
7637                 ext = &obj->externs[i];
7638
7639                 if (!ext->is_set && !ext->is_weak) {
7640                         pr_warn("extern '%s' (strong): not resolved\n", ext->name);
7641                         return -ESRCH;
7642                 } else if (!ext->is_set) {
7643                         pr_debug("extern '%s' (weak): not resolved, defaulting to zero\n",
7644                                  ext->name);
7645                 }
7646         }
7647
7648         return 0;
7649 }
7650
7651 static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const char *target_btf_path)
7652 {
7653         int err, i;
7654
7655         if (!obj)
7656                 return libbpf_err(-EINVAL);
7657
7658         if (obj->loaded) {
7659                 pr_warn("object '%s': load can't be attempted twice\n", obj->name);
7660                 return libbpf_err(-EINVAL);
7661         }
7662
7663         if (obj->gen_loader)
7664                 bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps);
7665
7666         err = bpf_object__probe_loading(obj);
7667         err = err ? : bpf_object__load_vmlinux_btf(obj, false);
7668         err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
7669         err = err ? : bpf_object__sanitize_and_load_btf(obj);
7670         err = err ? : bpf_object__sanitize_maps(obj);
7671         err = err ? : bpf_object__init_kern_struct_ops_maps(obj);
7672         err = err ? : bpf_object__create_maps(obj);
7673         err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path);
7674         err = err ? : bpf_object__load_progs(obj, extra_log_level);
7675         err = err ? : bpf_object_init_prog_arrays(obj);
7676
7677         if (obj->gen_loader) {
7678                 /* reset FDs */
7679                 if (obj->btf)
7680                         btf__set_fd(obj->btf, -1);
7681                 for (i = 0; i < obj->nr_maps; i++)
7682                         obj->maps[i].fd = -1;
7683                 if (!err)
7684                         err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps);
7685         }
7686
7687         /* clean up fd_array */
7688         zfree(&obj->fd_array);
7689
7690         /* clean up module BTFs */
7691         for (i = 0; i < obj->btf_module_cnt; i++) {
7692                 close(obj->btf_modules[i].fd);
7693                 btf__free(obj->btf_modules[i].btf);
7694                 free(obj->btf_modules[i].name);
7695         }
7696         free(obj->btf_modules);
7697
7698         /* clean up vmlinux BTF */
7699         btf__free(obj->btf_vmlinux);
7700         obj->btf_vmlinux = NULL;
7701
7702         obj->loaded = true; /* doesn't matter if successfully or not */
7703
7704         if (err)
7705                 goto out;
7706
7707         return 0;
7708 out:
7709         /* unpin any maps that were auto-pinned during load */
7710         for (i = 0; i < obj->nr_maps; i++)
7711                 if (obj->maps[i].pinned && !obj->maps[i].reused)
7712                         bpf_map__unpin(&obj->maps[i], NULL);
7713
7714         bpf_object_unload(obj);
7715         pr_warn("failed to load object '%s'\n", obj->path);
7716         return libbpf_err(err);
7717 }
7718
7719 int bpf_object__load(struct bpf_object *obj)
7720 {
7721         return bpf_object_load(obj, 0, NULL);
7722 }
7723
7724 static int make_parent_dir(const char *path)
7725 {
7726         char *cp, errmsg[STRERR_BUFSIZE];
7727         char *dname, *dir;
7728         int err = 0;
7729
7730         dname = strdup(path);
7731         if (dname == NULL)
7732                 return -ENOMEM;
7733
7734         dir = dirname(dname);
7735         if (mkdir(dir, 0700) && errno != EEXIST)
7736                 err = -errno;
7737
7738         free(dname);
7739         if (err) {
7740                 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
7741                 pr_warn("failed to mkdir %s: %s\n", path, cp);
7742         }
7743         return err;
7744 }
7745
7746 static int check_path(const char *path)
7747 {
7748         char *cp, errmsg[STRERR_BUFSIZE];
7749         struct statfs st_fs;
7750         char *dname, *dir;
7751         int err = 0;
7752
7753         if (path == NULL)
7754                 return -EINVAL;
7755
7756         dname = strdup(path);
7757         if (dname == NULL)
7758                 return -ENOMEM;
7759
7760         dir = dirname(dname);
7761         if (statfs(dir, &st_fs)) {
7762                 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
7763                 pr_warn("failed to statfs %s: %s\n", dir, cp);
7764                 err = -errno;
7765         }
7766         free(dname);
7767
7768         if (!err && st_fs.f_type != BPF_FS_MAGIC) {
7769                 pr_warn("specified path %s is not on BPF FS\n", path);
7770                 err = -EINVAL;
7771         }
7772
7773         return err;
7774 }
7775
7776 int bpf_program__pin(struct bpf_program *prog, const char *path)
7777 {
7778         char *cp, errmsg[STRERR_BUFSIZE];
7779         int err;
7780
7781         if (prog->fd < 0) {
7782                 pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name);
7783                 return libbpf_err(-EINVAL);
7784         }
7785
7786         err = make_parent_dir(path);
7787         if (err)
7788                 return libbpf_err(err);
7789
7790         err = check_path(path);
7791         if (err)
7792                 return libbpf_err(err);
7793
7794         if (bpf_obj_pin(prog->fd, path)) {
7795                 err = -errno;
7796                 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
7797                 pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, cp);
7798                 return libbpf_err(err);
7799         }
7800
7801         pr_debug("prog '%s': pinned at '%s'\n", prog->name, path);
7802         return 0;
7803 }
7804
7805 int bpf_program__unpin(struct bpf_program *prog, const char *path)
7806 {
7807         int err;
7808
7809         if (prog->fd < 0) {
7810                 pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name);
7811                 return libbpf_err(-EINVAL);
7812         }
7813
7814         err = check_path(path);
7815         if (err)
7816                 return libbpf_err(err);
7817
7818         err = unlink(path);
7819         if (err)
7820                 return libbpf_err(-errno);
7821
7822         pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path);
7823         return 0;
7824 }
7825
7826 int bpf_map__pin(struct bpf_map *map, const char *path)
7827 {
7828         char *cp, errmsg[STRERR_BUFSIZE];
7829         int err;
7830
7831         if (map == NULL) {
7832                 pr_warn("invalid map pointer\n");
7833                 return libbpf_err(-EINVAL);
7834         }
7835
7836         if (map->pin_path) {
7837                 if (path && strcmp(path, map->pin_path)) {
7838                         pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
7839                                 bpf_map__name(map), map->pin_path, path);
7840                         return libbpf_err(-EINVAL);
7841                 } else if (map->pinned) {
7842                         pr_debug("map '%s' already pinned at '%s'; not re-pinning\n",
7843                                  bpf_map__name(map), map->pin_path);
7844                         return 0;
7845                 }
7846         } else {
7847                 if (!path) {
7848                         pr_warn("missing a path to pin map '%s' at\n",
7849                                 bpf_map__name(map));
7850                         return libbpf_err(-EINVAL);
7851                 } else if (map->pinned) {
7852                         pr_warn("map '%s' already pinned\n", bpf_map__name(map));
7853                         return libbpf_err(-EEXIST);
7854                 }
7855
7856                 map->pin_path = strdup(path);
7857                 if (!map->pin_path) {
7858                         err = -errno;
7859                         goto out_err;
7860                 }
7861         }
7862
7863         err = make_parent_dir(map->pin_path);
7864         if (err)
7865                 return libbpf_err(err);
7866
7867         err = check_path(map->pin_path);
7868         if (err)
7869                 return libbpf_err(err);
7870
7871         if (bpf_obj_pin(map->fd, map->pin_path)) {
7872                 err = -errno;
7873                 goto out_err;
7874         }
7875
7876         map->pinned = true;
7877         pr_debug("pinned map '%s'\n", map->pin_path);
7878
7879         return 0;
7880
7881 out_err:
7882         cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
7883         pr_warn("failed to pin map: %s\n", cp);
7884         return libbpf_err(err);
7885 }
7886
7887 int bpf_map__unpin(struct bpf_map *map, const char *path)
7888 {
7889         int err;
7890
7891         if (map == NULL) {
7892                 pr_warn("invalid map pointer\n");
7893                 return libbpf_err(-EINVAL);
7894         }
7895
7896         if (map->pin_path) {
7897                 if (path && strcmp(path, map->pin_path)) {
7898                         pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
7899                                 bpf_map__name(map), map->pin_path, path);
7900                         return libbpf_err(-EINVAL);
7901                 }
7902                 path = map->pin_path;
7903         } else if (!path) {
7904                 pr_warn("no path to unpin map '%s' from\n",
7905                         bpf_map__name(map));
7906                 return libbpf_err(-EINVAL);
7907         }
7908
7909         err = check_path(path);
7910         if (err)
7911                 return libbpf_err(err);
7912
7913         err = unlink(path);
7914         if (err != 0)
7915                 return libbpf_err(-errno);
7916
7917         map->pinned = false;
7918         pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path);
7919
7920         return 0;
7921 }
7922
7923 int bpf_map__set_pin_path(struct bpf_map *map, const char *path)
7924 {
7925         char *new = NULL;
7926
7927         if (path) {
7928                 new = strdup(path);
7929                 if (!new)
7930                         return libbpf_err(-errno);
7931         }
7932
7933         free(map->pin_path);
7934         map->pin_path = new;
7935         return 0;
7936 }
7937
7938 __alias(bpf_map__pin_path)
7939 const char *bpf_map__get_pin_path(const struct bpf_map *map);
7940
7941 const char *bpf_map__pin_path(const struct bpf_map *map)
7942 {
7943         return map->pin_path;
7944 }
7945
7946 bool bpf_map__is_pinned(const struct bpf_map *map)
7947 {
7948         return map->pinned;
7949 }
7950
7951 static void sanitize_pin_path(char *s)
7952 {
7953         /* bpffs disallows periods in path names */
7954         while (*s) {
7955                 if (*s == '.')
7956                         *s = '_';
7957                 s++;
7958         }
7959 }
7960
7961 int bpf_object__pin_maps(struct bpf_object *obj, const char *path)
7962 {
7963         struct bpf_map *map;
7964         int err;
7965
7966         if (!obj)
7967                 return libbpf_err(-ENOENT);
7968
7969         if (!obj->loaded) {
7970                 pr_warn("object not yet loaded; load it first\n");
7971                 return libbpf_err(-ENOENT);
7972         }
7973
7974         bpf_object__for_each_map(map, obj) {
7975                 char *pin_path = NULL;
7976                 char buf[PATH_MAX];
7977
7978                 if (!map->autocreate)
7979                         continue;
7980
7981                 if (path) {
7982                         err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map));
7983                         if (err)
7984                                 goto err_unpin_maps;
7985                         sanitize_pin_path(buf);
7986                         pin_path = buf;
7987                 } else if (!map->pin_path) {
7988                         continue;
7989                 }
7990
7991                 err = bpf_map__pin(map, pin_path);
7992                 if (err)
7993                         goto err_unpin_maps;
7994         }
7995
7996         return 0;
7997
7998 err_unpin_maps:
7999         while ((map = bpf_object__prev_map(obj, map))) {
8000                 if (!map->pin_path)
8001                         continue;
8002
8003                 bpf_map__unpin(map, NULL);
8004         }
8005
8006         return libbpf_err(err);
8007 }
8008
8009 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path)
8010 {
8011         struct bpf_map *map;
8012         int err;
8013
8014         if (!obj)
8015                 return libbpf_err(-ENOENT);
8016
8017         bpf_object__for_each_map(map, obj) {
8018                 char *pin_path = NULL;
8019                 char buf[PATH_MAX];
8020
8021                 if (path) {
8022                         err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map));
8023                         if (err)
8024                                 return libbpf_err(err);
8025                         sanitize_pin_path(buf);
8026                         pin_path = buf;
8027                 } else if (!map->pin_path) {
8028                         continue;
8029                 }
8030
8031                 err = bpf_map__unpin(map, pin_path);
8032                 if (err)
8033                         return libbpf_err(err);
8034         }
8035
8036         return 0;
8037 }
8038
8039 int bpf_object__pin_programs(struct bpf_object *obj, const char *path)
8040 {
8041         struct bpf_program *prog;
8042         char buf[PATH_MAX];
8043         int err;
8044
8045         if (!obj)
8046                 return libbpf_err(-ENOENT);
8047
8048         if (!obj->loaded) {
8049                 pr_warn("object not yet loaded; load it first\n");
8050                 return libbpf_err(-ENOENT);
8051         }
8052
8053         bpf_object__for_each_program(prog, obj) {
8054                 err = pathname_concat(buf, sizeof(buf), path, prog->name);
8055                 if (err)
8056                         goto err_unpin_programs;
8057
8058                 err = bpf_program__pin(prog, buf);
8059                 if (err)
8060                         goto err_unpin_programs;
8061         }
8062
8063         return 0;
8064
8065 err_unpin_programs:
8066         while ((prog = bpf_object__prev_program(obj, prog))) {
8067                 if (pathname_concat(buf, sizeof(buf), path, prog->name))
8068                         continue;
8069
8070                 bpf_program__unpin(prog, buf);
8071         }
8072
8073         return libbpf_err(err);
8074 }
8075
8076 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path)
8077 {
8078         struct bpf_program *prog;
8079         int err;
8080
8081         if (!obj)
8082                 return libbpf_err(-ENOENT);
8083
8084         bpf_object__for_each_program(prog, obj) {
8085                 char buf[PATH_MAX];
8086
8087                 err = pathname_concat(buf, sizeof(buf), path, prog->name);
8088                 if (err)
8089                         return libbpf_err(err);
8090
8091                 err = bpf_program__unpin(prog, buf);
8092                 if (err)
8093                         return libbpf_err(err);
8094         }
8095
8096         return 0;
8097 }
8098
8099 int bpf_object__pin(struct bpf_object *obj, const char *path)
8100 {
8101         int err;
8102
8103         err = bpf_object__pin_maps(obj, path);
8104         if (err)
8105                 return libbpf_err(err);
8106
8107         err = bpf_object__pin_programs(obj, path);
8108         if (err) {
8109                 bpf_object__unpin_maps(obj, path);
8110                 return libbpf_err(err);
8111         }
8112
8113         return 0;
8114 }
8115
8116 static void bpf_map__destroy(struct bpf_map *map)
8117 {
8118         if (map->inner_map) {
8119                 bpf_map__destroy(map->inner_map);
8120                 zfree(&map->inner_map);
8121         }
8122
8123         zfree(&map->init_slots);
8124         map->init_slots_sz = 0;
8125
8126         if (map->mmaped) {
8127                 munmap(map->mmaped, bpf_map_mmap_sz(map));
8128                 map->mmaped = NULL;
8129         }
8130
8131         if (map->st_ops) {
8132                 zfree(&map->st_ops->data);
8133                 zfree(&map->st_ops->progs);
8134                 zfree(&map->st_ops->kern_func_off);
8135                 zfree(&map->st_ops);
8136         }
8137
8138         zfree(&map->name);
8139         zfree(&map->real_name);
8140         zfree(&map->pin_path);
8141
8142         if (map->fd >= 0)
8143                 zclose(map->fd);
8144 }
8145
8146 void bpf_object__close(struct bpf_object *obj)
8147 {
8148         size_t i;
8149
8150         if (IS_ERR_OR_NULL(obj))
8151                 return;
8152
8153         usdt_manager_free(obj->usdt_man);
8154         obj->usdt_man = NULL;
8155
8156         bpf_gen__free(obj->gen_loader);
8157         bpf_object__elf_finish(obj);
8158         bpf_object_unload(obj);
8159         btf__free(obj->btf);
8160         btf_ext__free(obj->btf_ext);
8161
8162         for (i = 0; i < obj->nr_maps; i++)
8163                 bpf_map__destroy(&obj->maps[i]);
8164
8165         zfree(&obj->btf_custom_path);
8166         zfree(&obj->kconfig);
8167         zfree(&obj->externs);
8168         obj->nr_extern = 0;
8169
8170         zfree(&obj->maps);
8171         obj->nr_maps = 0;
8172
8173         if (obj->programs && obj->nr_programs) {
8174                 for (i = 0; i < obj->nr_programs; i++)
8175                         bpf_program__exit(&obj->programs[i]);
8176         }
8177         zfree(&obj->programs);
8178
8179         free(obj);
8180 }
8181
8182 const char *bpf_object__name(const struct bpf_object *obj)
8183 {
8184         return obj ? obj->name : libbpf_err_ptr(-EINVAL);
8185 }
8186
8187 unsigned int bpf_object__kversion(const struct bpf_object *obj)
8188 {
8189         return obj ? obj->kern_version : 0;
8190 }
8191
8192 struct btf *bpf_object__btf(const struct bpf_object *obj)
8193 {
8194         return obj ? obj->btf : NULL;
8195 }
8196
8197 int bpf_object__btf_fd(const struct bpf_object *obj)
8198 {
8199         return obj->btf ? btf__fd(obj->btf) : -1;
8200 }
8201
8202 int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version)
8203 {
8204         if (obj->loaded)
8205                 return libbpf_err(-EINVAL);
8206
8207         obj->kern_version = kern_version;
8208
8209         return 0;
8210 }
8211
8212 int bpf_object__gen_loader(struct bpf_object *obj, struct gen_loader_opts *opts)
8213 {
8214         struct bpf_gen *gen;
8215
8216         if (!opts)
8217                 return -EFAULT;
8218         if (!OPTS_VALID(opts, gen_loader_opts))
8219                 return -EINVAL;
8220         gen = calloc(sizeof(*gen), 1);
8221         if (!gen)
8222                 return -ENOMEM;
8223         gen->opts = opts;
8224         obj->gen_loader = gen;
8225         return 0;
8226 }
8227
8228 static struct bpf_program *
8229 __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj,
8230                     bool forward)
8231 {
8232         size_t nr_programs = obj->nr_programs;
8233         ssize_t idx;
8234
8235         if (!nr_programs)
8236                 return NULL;
8237
8238         if (!p)
8239                 /* Iter from the beginning */
8240                 return forward ? &obj->programs[0] :
8241                         &obj->programs[nr_programs - 1];
8242
8243         if (p->obj != obj) {
8244                 pr_warn("error: program handler doesn't match object\n");
8245                 return errno = EINVAL, NULL;
8246         }
8247
8248         idx = (p - obj->programs) + (forward ? 1 : -1);
8249         if (idx >= obj->nr_programs || idx < 0)
8250                 return NULL;
8251         return &obj->programs[idx];
8252 }
8253
8254 struct bpf_program *
8255 bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prev)
8256 {
8257         struct bpf_program *prog = prev;
8258
8259         do {
8260                 prog = __bpf_program__iter(prog, obj, true);
8261         } while (prog && prog_is_subprog(obj, prog));
8262
8263         return prog;
8264 }
8265
8266 struct bpf_program *
8267 bpf_object__prev_program(const struct bpf_object *obj, struct bpf_program *next)
8268 {
8269         struct bpf_program *prog = next;
8270
8271         do {
8272                 prog = __bpf_program__iter(prog, obj, false);
8273         } while (prog && prog_is_subprog(obj, prog));
8274
8275         return prog;
8276 }
8277
8278 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
8279 {
8280         prog->prog_ifindex = ifindex;
8281 }
8282
8283 const char *bpf_program__name(const struct bpf_program *prog)
8284 {
8285         return prog->name;
8286 }
8287
8288 const char *bpf_program__section_name(const struct bpf_program *prog)
8289 {
8290         return prog->sec_name;
8291 }
8292
8293 bool bpf_program__autoload(const struct bpf_program *prog)
8294 {
8295         return prog->autoload;
8296 }
8297
8298 int bpf_program__set_autoload(struct bpf_program *prog, bool autoload)
8299 {
8300         if (prog->obj->loaded)
8301                 return libbpf_err(-EINVAL);
8302
8303         prog->autoload = autoload;
8304         return 0;
8305 }
8306
8307 bool bpf_program__autoattach(const struct bpf_program *prog)
8308 {
8309         return prog->autoattach;
8310 }
8311
8312 void bpf_program__set_autoattach(struct bpf_program *prog, bool autoattach)
8313 {
8314         prog->autoattach = autoattach;
8315 }
8316
8317 const struct bpf_insn *bpf_program__insns(const struct bpf_program *prog)
8318 {
8319         return prog->insns;
8320 }
8321
8322 size_t bpf_program__insn_cnt(const struct bpf_program *prog)
8323 {
8324         return prog->insns_cnt;
8325 }
8326
8327 int bpf_program__set_insns(struct bpf_program *prog,
8328                            struct bpf_insn *new_insns, size_t new_insn_cnt)
8329 {
8330         struct bpf_insn *insns;
8331
8332         if (prog->obj->loaded)
8333                 return -EBUSY;
8334
8335         insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns));
8336         if (!insns) {
8337                 pr_warn("prog '%s': failed to realloc prog code\n", prog->name);
8338                 return -ENOMEM;
8339         }
8340         memcpy(insns, new_insns, new_insn_cnt * sizeof(*insns));
8341
8342         prog->insns = insns;
8343         prog->insns_cnt = new_insn_cnt;
8344         return 0;
8345 }
8346
8347 int bpf_program__fd(const struct bpf_program *prog)
8348 {
8349         if (!prog)
8350                 return libbpf_err(-EINVAL);
8351
8352         if (prog->fd < 0)
8353                 return libbpf_err(-ENOENT);
8354
8355         return prog->fd;
8356 }
8357
8358 __alias(bpf_program__type)
8359 enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog);
8360
8361 enum bpf_prog_type bpf_program__type(const struct bpf_program *prog)
8362 {
8363         return prog->type;
8364 }
8365
8366 int bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
8367 {
8368         if (prog->obj->loaded)
8369                 return libbpf_err(-EBUSY);
8370
8371         prog->type = type;
8372         return 0;
8373 }
8374
8375 __alias(bpf_program__expected_attach_type)
8376 enum bpf_attach_type bpf_program__get_expected_attach_type(const struct bpf_program *prog);
8377
8378 enum bpf_attach_type bpf_program__expected_attach_type(const struct bpf_program *prog)
8379 {
8380         return prog->expected_attach_type;
8381 }
8382
8383 int bpf_program__set_expected_attach_type(struct bpf_program *prog,
8384                                            enum bpf_attach_type type)
8385 {
8386         if (prog->obj->loaded)
8387                 return libbpf_err(-EBUSY);
8388
8389         prog->expected_attach_type = type;
8390         return 0;
8391 }
8392
8393 __u32 bpf_program__flags(const struct bpf_program *prog)
8394 {
8395         return prog->prog_flags;
8396 }
8397
8398 int bpf_program__set_flags(struct bpf_program *prog, __u32 flags)
8399 {
8400         if (prog->obj->loaded)
8401                 return libbpf_err(-EBUSY);
8402
8403         prog->prog_flags = flags;
8404         return 0;
8405 }
8406
8407 __u32 bpf_program__log_level(const struct bpf_program *prog)
8408 {
8409         return prog->log_level;
8410 }
8411
8412 int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level)
8413 {
8414         if (prog->obj->loaded)
8415                 return libbpf_err(-EBUSY);
8416
8417         prog->log_level = log_level;
8418         return 0;
8419 }
8420
8421 const char *bpf_program__log_buf(const struct bpf_program *prog, size_t *log_size)
8422 {
8423         *log_size = prog->log_size;
8424         return prog->log_buf;
8425 }
8426
8427 int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size)
8428 {
8429         if (log_size && !log_buf)
8430                 return -EINVAL;
8431         if (prog->log_size > UINT_MAX)
8432                 return -EINVAL;
8433         if (prog->obj->loaded)
8434                 return -EBUSY;
8435
8436         prog->log_buf = log_buf;
8437         prog->log_size = log_size;
8438         return 0;
8439 }
8440
8441 #define SEC_DEF(sec_pfx, ptype, atype, flags, ...) {                        \
8442         .sec = (char *)sec_pfx,                                             \
8443         .prog_type = BPF_PROG_TYPE_##ptype,                                 \
8444         .expected_attach_type = atype,                                      \
8445         .cookie = (long)(flags),                                            \
8446         .prog_prepare_load_fn = libbpf_prepare_prog_load,                   \
8447         __VA_ARGS__                                                         \
8448 }
8449
8450 static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8451 static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8452 static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8453 static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8454 static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8455 static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8456 static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8457 static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8458 static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8459 static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8460
8461 static const struct bpf_sec_def section_defs[] = {
8462         SEC_DEF("socket",               SOCKET_FILTER, 0, SEC_NONE),
8463         SEC_DEF("sk_reuseport/migrate", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT_OR_MIGRATE, SEC_ATTACHABLE),
8464         SEC_DEF("sk_reuseport",         SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT, SEC_ATTACHABLE),
8465         SEC_DEF("kprobe+",              KPROBE, 0, SEC_NONE, attach_kprobe),
8466         SEC_DEF("uprobe+",              KPROBE, 0, SEC_NONE, attach_uprobe),
8467         SEC_DEF("uprobe.s+",            KPROBE, 0, SEC_SLEEPABLE, attach_uprobe),
8468         SEC_DEF("kretprobe+",           KPROBE, 0, SEC_NONE, attach_kprobe),
8469         SEC_DEF("uretprobe+",           KPROBE, 0, SEC_NONE, attach_uprobe),
8470         SEC_DEF("uretprobe.s+",         KPROBE, 0, SEC_SLEEPABLE, attach_uprobe),
8471         SEC_DEF("kprobe.multi+",        KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi),
8472         SEC_DEF("kretprobe.multi+",     KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi),
8473         SEC_DEF("ksyscall+",            KPROBE, 0, SEC_NONE, attach_ksyscall),
8474         SEC_DEF("kretsyscall+",         KPROBE, 0, SEC_NONE, attach_ksyscall),
8475         SEC_DEF("usdt+",                KPROBE, 0, SEC_NONE, attach_usdt),
8476         SEC_DEF("tc",                   SCHED_CLS, 0, SEC_NONE),
8477         SEC_DEF("classifier",           SCHED_CLS, 0, SEC_NONE),
8478         SEC_DEF("action",               SCHED_ACT, 0, SEC_NONE),
8479         SEC_DEF("tracepoint+",          TRACEPOINT, 0, SEC_NONE, attach_tp),
8480         SEC_DEF("tp+",                  TRACEPOINT, 0, SEC_NONE, attach_tp),
8481         SEC_DEF("raw_tracepoint+",      RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp),
8482         SEC_DEF("raw_tp+",              RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp),
8483         SEC_DEF("raw_tracepoint.w+",    RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp),
8484         SEC_DEF("raw_tp.w+",            RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp),
8485         SEC_DEF("tp_btf+",              TRACING, BPF_TRACE_RAW_TP, SEC_ATTACH_BTF, attach_trace),
8486         SEC_DEF("fentry+",              TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF, attach_trace),
8487         SEC_DEF("fmod_ret+",            TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF, attach_trace),
8488         SEC_DEF("fexit+",               TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF, attach_trace),
8489         SEC_DEF("fentry.s+",            TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace),
8490         SEC_DEF("fmod_ret.s+",          TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace),
8491         SEC_DEF("fexit.s+",             TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace),
8492         SEC_DEF("freplace+",            EXT, 0, SEC_ATTACH_BTF, attach_trace),
8493         SEC_DEF("lsm+",                 LSM, BPF_LSM_MAC, SEC_ATTACH_BTF, attach_lsm),
8494         SEC_DEF("lsm.s+",               LSM, BPF_LSM_MAC, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_lsm),
8495         SEC_DEF("lsm_cgroup+",          LSM, BPF_LSM_CGROUP, SEC_ATTACH_BTF),
8496         SEC_DEF("iter+",                TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF, attach_iter),
8497         SEC_DEF("iter.s+",              TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_iter),
8498         SEC_DEF("syscall",              SYSCALL, 0, SEC_SLEEPABLE),
8499         SEC_DEF("xdp.frags/devmap",     XDP, BPF_XDP_DEVMAP, SEC_XDP_FRAGS),
8500         SEC_DEF("xdp/devmap",           XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE),
8501         SEC_DEF("xdp.frags/cpumap",     XDP, BPF_XDP_CPUMAP, SEC_XDP_FRAGS),
8502         SEC_DEF("xdp/cpumap",           XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE),
8503         SEC_DEF("xdp.frags",            XDP, BPF_XDP, SEC_XDP_FRAGS),
8504         SEC_DEF("xdp",                  XDP, BPF_XDP, SEC_ATTACHABLE_OPT),
8505         SEC_DEF("perf_event",           PERF_EVENT, 0, SEC_NONE),
8506         SEC_DEF("lwt_in",               LWT_IN, 0, SEC_NONE),
8507         SEC_DEF("lwt_out",              LWT_OUT, 0, SEC_NONE),
8508         SEC_DEF("lwt_xmit",             LWT_XMIT, 0, SEC_NONE),
8509         SEC_DEF("lwt_seg6local",        LWT_SEG6LOCAL, 0, SEC_NONE),
8510         SEC_DEF("sockops",              SOCK_OPS, BPF_CGROUP_SOCK_OPS, SEC_ATTACHABLE_OPT),
8511         SEC_DEF("sk_skb/stream_parser", SK_SKB, BPF_SK_SKB_STREAM_PARSER, SEC_ATTACHABLE_OPT),
8512         SEC_DEF("sk_skb/stream_verdict",SK_SKB, BPF_SK_SKB_STREAM_VERDICT, SEC_ATTACHABLE_OPT),
8513         SEC_DEF("sk_skb",               SK_SKB, 0, SEC_NONE),
8514         SEC_DEF("sk_msg",               SK_MSG, BPF_SK_MSG_VERDICT, SEC_ATTACHABLE_OPT),
8515         SEC_DEF("lirc_mode2",           LIRC_MODE2, BPF_LIRC_MODE2, SEC_ATTACHABLE_OPT),
8516         SEC_DEF("flow_dissector",       FLOW_DISSECTOR, BPF_FLOW_DISSECTOR, SEC_ATTACHABLE_OPT),
8517         SEC_DEF("cgroup_skb/ingress",   CGROUP_SKB, BPF_CGROUP_INET_INGRESS, SEC_ATTACHABLE_OPT),
8518         SEC_DEF("cgroup_skb/egress",    CGROUP_SKB, BPF_CGROUP_INET_EGRESS, SEC_ATTACHABLE_OPT),
8519         SEC_DEF("cgroup/skb",           CGROUP_SKB, 0, SEC_NONE),
8520         SEC_DEF("cgroup/sock_create",   CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE),
8521         SEC_DEF("cgroup/sock_release",  CGROUP_SOCK, BPF_CGROUP_INET_SOCK_RELEASE, SEC_ATTACHABLE),
8522         SEC_DEF("cgroup/sock",          CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE_OPT),
8523         SEC_DEF("cgroup/post_bind4",    CGROUP_SOCK, BPF_CGROUP_INET4_POST_BIND, SEC_ATTACHABLE),
8524         SEC_DEF("cgroup/post_bind6",    CGROUP_SOCK, BPF_CGROUP_INET6_POST_BIND, SEC_ATTACHABLE),
8525         SEC_DEF("cgroup/bind4",         CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_BIND, SEC_ATTACHABLE),
8526         SEC_DEF("cgroup/bind6",         CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_BIND, SEC_ATTACHABLE),
8527         SEC_DEF("cgroup/connect4",      CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_CONNECT, SEC_ATTACHABLE),
8528         SEC_DEF("cgroup/connect6",      CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_CONNECT, SEC_ATTACHABLE),
8529         SEC_DEF("cgroup/sendmsg4",      CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_SENDMSG, SEC_ATTACHABLE),
8530         SEC_DEF("cgroup/sendmsg6",      CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_SENDMSG, SEC_ATTACHABLE),
8531         SEC_DEF("cgroup/recvmsg4",      CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_RECVMSG, SEC_ATTACHABLE),
8532         SEC_DEF("cgroup/recvmsg6",      CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_RECVMSG, SEC_ATTACHABLE),
8533         SEC_DEF("cgroup/getpeername4",  CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETPEERNAME, SEC_ATTACHABLE),
8534         SEC_DEF("cgroup/getpeername6",  CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETPEERNAME, SEC_ATTACHABLE),
8535         SEC_DEF("cgroup/getsockname4",  CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETSOCKNAME, SEC_ATTACHABLE),
8536         SEC_DEF("cgroup/getsockname6",  CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETSOCKNAME, SEC_ATTACHABLE),
8537         SEC_DEF("cgroup/sysctl",        CGROUP_SYSCTL, BPF_CGROUP_SYSCTL, SEC_ATTACHABLE),
8538         SEC_DEF("cgroup/getsockopt",    CGROUP_SOCKOPT, BPF_CGROUP_GETSOCKOPT, SEC_ATTACHABLE),
8539         SEC_DEF("cgroup/setsockopt",    CGROUP_SOCKOPT, BPF_CGROUP_SETSOCKOPT, SEC_ATTACHABLE),
8540         SEC_DEF("cgroup/dev",           CGROUP_DEVICE, BPF_CGROUP_DEVICE, SEC_ATTACHABLE_OPT),
8541         SEC_DEF("struct_ops+",          STRUCT_OPS, 0, SEC_NONE),
8542         SEC_DEF("sk_lookup",            SK_LOOKUP, BPF_SK_LOOKUP, SEC_ATTACHABLE),
8543 };
8544
8545 static size_t custom_sec_def_cnt;
8546 static struct bpf_sec_def *custom_sec_defs;
8547 static struct bpf_sec_def custom_fallback_def;
8548 static bool has_custom_fallback_def;
8549
8550 static int last_custom_sec_def_handler_id;
8551
8552 int libbpf_register_prog_handler(const char *sec,
8553                                  enum bpf_prog_type prog_type,
8554                                  enum bpf_attach_type exp_attach_type,
8555                                  const struct libbpf_prog_handler_opts *opts)
8556 {
8557         struct bpf_sec_def *sec_def;
8558
8559         if (!OPTS_VALID(opts, libbpf_prog_handler_opts))
8560                 return libbpf_err(-EINVAL);
8561
8562         if (last_custom_sec_def_handler_id == INT_MAX) /* prevent overflow */
8563                 return libbpf_err(-E2BIG);
8564
8565         if (sec) {
8566                 sec_def = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt + 1,
8567                                               sizeof(*sec_def));
8568                 if (!sec_def)
8569                         return libbpf_err(-ENOMEM);
8570
8571                 custom_sec_defs = sec_def;
8572                 sec_def = &custom_sec_defs[custom_sec_def_cnt];
8573         } else {
8574                 if (has_custom_fallback_def)
8575                         return libbpf_err(-EBUSY);
8576
8577                 sec_def = &custom_fallback_def;
8578         }
8579
8580         sec_def->sec = sec ? strdup(sec) : NULL;
8581         if (sec && !sec_def->sec)
8582                 return libbpf_err(-ENOMEM);
8583
8584         sec_def->prog_type = prog_type;
8585         sec_def->expected_attach_type = exp_attach_type;
8586         sec_def->cookie = OPTS_GET(opts, cookie, 0);
8587
8588         sec_def->prog_setup_fn = OPTS_GET(opts, prog_setup_fn, NULL);
8589         sec_def->prog_prepare_load_fn = OPTS_GET(opts, prog_prepare_load_fn, NULL);
8590         sec_def->prog_attach_fn = OPTS_GET(opts, prog_attach_fn, NULL);
8591
8592         sec_def->handler_id = ++last_custom_sec_def_handler_id;
8593
8594         if (sec)
8595                 custom_sec_def_cnt++;
8596         else
8597                 has_custom_fallback_def = true;
8598
8599         return sec_def->handler_id;
8600 }
8601
8602 int libbpf_unregister_prog_handler(int handler_id)
8603 {
8604         struct bpf_sec_def *sec_defs;
8605         int i;
8606
8607         if (handler_id <= 0)
8608                 return libbpf_err(-EINVAL);
8609
8610         if (has_custom_fallback_def && custom_fallback_def.handler_id == handler_id) {
8611                 memset(&custom_fallback_def, 0, sizeof(custom_fallback_def));
8612                 has_custom_fallback_def = false;
8613                 return 0;
8614         }
8615
8616         for (i = 0; i < custom_sec_def_cnt; i++) {
8617                 if (custom_sec_defs[i].handler_id == handler_id)
8618                         break;
8619         }
8620
8621         if (i == custom_sec_def_cnt)
8622                 return libbpf_err(-ENOENT);
8623
8624         free(custom_sec_defs[i].sec);
8625         for (i = i + 1; i < custom_sec_def_cnt; i++)
8626                 custom_sec_defs[i - 1] = custom_sec_defs[i];
8627         custom_sec_def_cnt--;
8628
8629         /* try to shrink the array, but it's ok if we couldn't */
8630         sec_defs = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt, sizeof(*sec_defs));
8631         if (sec_defs)
8632                 custom_sec_defs = sec_defs;
8633
8634         return 0;
8635 }
8636
8637 static bool sec_def_matches(const struct bpf_sec_def *sec_def, const char *sec_name)
8638 {
8639         size_t len = strlen(sec_def->sec);
8640
8641         /* "type/" always has to have proper SEC("type/extras") form */
8642         if (sec_def->sec[len - 1] == '/') {
8643                 if (str_has_pfx(sec_name, sec_def->sec))
8644                         return true;
8645                 return false;
8646         }
8647
8648         /* "type+" means it can be either exact SEC("type") or
8649          * well-formed SEC("type/extras") with proper '/' separator
8650          */
8651         if (sec_def->sec[len - 1] == '+') {
8652                 len--;
8653                 /* not even a prefix */
8654                 if (strncmp(sec_name, sec_def->sec, len) != 0)
8655                         return false;
8656                 /* exact match or has '/' separator */
8657                 if (sec_name[len] == '\0' || sec_name[len] == '/')
8658                         return true;
8659                 return false;
8660         }
8661
8662         return strcmp(sec_name, sec_def->sec) == 0;
8663 }
8664
8665 static const struct bpf_sec_def *find_sec_def(const char *sec_name)
8666 {
8667         const struct bpf_sec_def *sec_def;
8668         int i, n;
8669
8670         n = custom_sec_def_cnt;
8671         for (i = 0; i < n; i++) {
8672                 sec_def = &custom_sec_defs[i];
8673                 if (sec_def_matches(sec_def, sec_name))
8674                         return sec_def;
8675         }
8676
8677         n = ARRAY_SIZE(section_defs);
8678         for (i = 0; i < n; i++) {
8679                 sec_def = &section_defs[i];
8680                 if (sec_def_matches(sec_def, sec_name))
8681                         return sec_def;
8682         }
8683
8684         if (has_custom_fallback_def)
8685                 return &custom_fallback_def;
8686
8687         return NULL;
8688 }
8689
8690 #define MAX_TYPE_NAME_SIZE 32
8691
8692 static char *libbpf_get_type_names(bool attach_type)
8693 {
8694         int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE;
8695         char *buf;
8696
8697         buf = malloc(len);
8698         if (!buf)
8699                 return NULL;
8700
8701         buf[0] = '\0';
8702         /* Forge string buf with all available names */
8703         for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
8704                 const struct bpf_sec_def *sec_def = &section_defs[i];
8705
8706                 if (attach_type) {
8707                         if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)
8708                                 continue;
8709
8710                         if (!(sec_def->cookie & SEC_ATTACHABLE))
8711                                 continue;
8712                 }
8713
8714                 if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) {
8715                         free(buf);
8716                         return NULL;
8717                 }
8718                 strcat(buf, " ");
8719                 strcat(buf, section_defs[i].sec);
8720         }
8721
8722         return buf;
8723 }
8724
8725 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
8726                              enum bpf_attach_type *expected_attach_type)
8727 {
8728         const struct bpf_sec_def *sec_def;
8729         char *type_names;
8730
8731         if (!name)
8732                 return libbpf_err(-EINVAL);
8733
8734         sec_def = find_sec_def(name);
8735         if (sec_def) {
8736                 *prog_type = sec_def->prog_type;
8737                 *expected_attach_type = sec_def->expected_attach_type;
8738                 return 0;
8739         }
8740
8741         pr_debug("failed to guess program type from ELF section '%s'\n", name);
8742         type_names = libbpf_get_type_names(false);
8743         if (type_names != NULL) {
8744                 pr_debug("supported section(type) names are:%s\n", type_names);
8745                 free(type_names);
8746         }
8747
8748         return libbpf_err(-ESRCH);
8749 }
8750
8751 const char *libbpf_bpf_attach_type_str(enum bpf_attach_type t)
8752 {
8753         if (t < 0 || t >= ARRAY_SIZE(attach_type_name))
8754                 return NULL;
8755
8756         return attach_type_name[t];
8757 }
8758
8759 const char *libbpf_bpf_link_type_str(enum bpf_link_type t)
8760 {
8761         if (t < 0 || t >= ARRAY_SIZE(link_type_name))
8762                 return NULL;
8763
8764         return link_type_name[t];
8765 }
8766
8767 const char *libbpf_bpf_map_type_str(enum bpf_map_type t)
8768 {
8769         if (t < 0 || t >= ARRAY_SIZE(map_type_name))
8770                 return NULL;
8771
8772         return map_type_name[t];
8773 }
8774
8775 const char *libbpf_bpf_prog_type_str(enum bpf_prog_type t)
8776 {
8777         if (t < 0 || t >= ARRAY_SIZE(prog_type_name))
8778                 return NULL;
8779
8780         return prog_type_name[t];
8781 }
8782
8783 static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj,
8784                                                      size_t offset)
8785 {
8786         struct bpf_map *map;
8787         size_t i;
8788
8789         for (i = 0; i < obj->nr_maps; i++) {
8790                 map = &obj->maps[i];
8791                 if (!bpf_map__is_struct_ops(map))
8792                         continue;
8793                 if (map->sec_offset <= offset &&
8794                     offset - map->sec_offset < map->def.value_size)
8795                         return map;
8796         }
8797
8798         return NULL;
8799 }
8800
8801 /* Collect the reloc from ELF and populate the st_ops->progs[] */
8802 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
8803                                             Elf64_Shdr *shdr, Elf_Data *data)
8804 {
8805         const struct btf_member *member;
8806         struct bpf_struct_ops *st_ops;
8807         struct bpf_program *prog;
8808         unsigned int shdr_idx;
8809         const struct btf *btf;
8810         struct bpf_map *map;
8811         unsigned int moff, insn_idx;
8812         const char *name;
8813         __u32 member_idx;
8814         Elf64_Sym *sym;
8815         Elf64_Rel *rel;
8816         int i, nrels;
8817
8818         btf = obj->btf;
8819         nrels = shdr->sh_size / shdr->sh_entsize;
8820         for (i = 0; i < nrels; i++) {
8821                 rel = elf_rel_by_idx(data, i);
8822                 if (!rel) {
8823                         pr_warn("struct_ops reloc: failed to get %d reloc\n", i);
8824                         return -LIBBPF_ERRNO__FORMAT;
8825                 }
8826
8827                 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
8828                 if (!sym) {
8829                         pr_warn("struct_ops reloc: symbol %zx not found\n",
8830                                 (size_t)ELF64_R_SYM(rel->r_info));
8831                         return -LIBBPF_ERRNO__FORMAT;
8832                 }
8833
8834                 name = elf_sym_str(obj, sym->st_name) ?: "<?>";
8835                 map = find_struct_ops_map_by_offset(obj, rel->r_offset);
8836                 if (!map) {
8837                         pr_warn("struct_ops reloc: cannot find map at rel->r_offset %zu\n",
8838                                 (size_t)rel->r_offset);
8839                         return -EINVAL;
8840                 }
8841
8842                 moff = rel->r_offset - map->sec_offset;
8843                 shdr_idx = sym->st_shndx;
8844                 st_ops = map->st_ops;
8845                 pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %zu name %d (\'%s\')\n",
8846                          map->name,
8847                          (long long)(rel->r_info >> 32),
8848                          (long long)sym->st_value,
8849                          shdr_idx, (size_t)rel->r_offset,
8850                          map->sec_offset, sym->st_name, name);
8851
8852                 if (shdr_idx >= SHN_LORESERVE) {
8853                         pr_warn("struct_ops reloc %s: rel->r_offset %zu shdr_idx %u unsupported non-static function\n",
8854                                 map->name, (size_t)rel->r_offset, shdr_idx);
8855                         return -LIBBPF_ERRNO__RELOC;
8856                 }
8857                 if (sym->st_value % BPF_INSN_SZ) {
8858                         pr_warn("struct_ops reloc %s: invalid target program offset %llu\n",
8859                                 map->name, (unsigned long long)sym->st_value);
8860                         return -LIBBPF_ERRNO__FORMAT;
8861                 }
8862                 insn_idx = sym->st_value / BPF_INSN_SZ;
8863
8864                 member = find_member_by_offset(st_ops->type, moff * 8);
8865                 if (!member) {
8866                         pr_warn("struct_ops reloc %s: cannot find member at moff %u\n",
8867                                 map->name, moff);
8868                         return -EINVAL;
8869                 }
8870                 member_idx = member - btf_members(st_ops->type);
8871                 name = btf__name_by_offset(btf, member->name_off);
8872
8873                 if (!resolve_func_ptr(btf, member->type, NULL)) {
8874                         pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n",
8875                                 map->name, name);
8876                         return -EINVAL;
8877                 }
8878
8879                 prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx);
8880                 if (!prog) {
8881                         pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n",
8882                                 map->name, shdr_idx, name);
8883                         return -EINVAL;
8884                 }
8885
8886                 /* prevent the use of BPF prog with invalid type */
8887                 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) {
8888                         pr_warn("struct_ops reloc %s: prog %s is not struct_ops BPF program\n",
8889                                 map->name, prog->name);
8890                         return -EINVAL;
8891                 }
8892
8893                 /* if we haven't yet processed this BPF program, record proper
8894                  * attach_btf_id and member_idx
8895                  */
8896                 if (!prog->attach_btf_id) {
8897                         prog->attach_btf_id = st_ops->type_id;
8898                         prog->expected_attach_type = member_idx;
8899                 }
8900
8901                 /* struct_ops BPF prog can be re-used between multiple
8902                  * .struct_ops as long as it's the same struct_ops struct
8903                  * definition and the same function pointer field
8904                  */
8905                 if (prog->attach_btf_id != st_ops->type_id ||
8906                     prog->expected_attach_type != member_idx) {
8907                         pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n",
8908                                 map->name, prog->name, prog->sec_name, prog->type,
8909                                 prog->attach_btf_id, prog->expected_attach_type, name);
8910                         return -EINVAL;
8911                 }
8912
8913                 st_ops->progs[member_idx] = prog;
8914         }
8915
8916         return 0;
8917 }
8918
8919 #define BTF_TRACE_PREFIX "btf_trace_"
8920 #define BTF_LSM_PREFIX "bpf_lsm_"
8921 #define BTF_ITER_PREFIX "bpf_iter_"
8922 #define BTF_MAX_NAME_SIZE 128
8923
8924 void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type,
8925                                 const char **prefix, int *kind)
8926 {
8927         switch (attach_type) {
8928         case BPF_TRACE_RAW_TP:
8929                 *prefix = BTF_TRACE_PREFIX;
8930                 *kind = BTF_KIND_TYPEDEF;
8931                 break;
8932         case BPF_LSM_MAC:
8933         case BPF_LSM_CGROUP:
8934                 *prefix = BTF_LSM_PREFIX;
8935                 *kind = BTF_KIND_FUNC;
8936                 break;
8937         case BPF_TRACE_ITER:
8938                 *prefix = BTF_ITER_PREFIX;
8939                 *kind = BTF_KIND_FUNC;
8940                 break;
8941         default:
8942                 *prefix = "";
8943                 *kind = BTF_KIND_FUNC;
8944         }
8945 }
8946
8947 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
8948                                    const char *name, __u32 kind)
8949 {
8950         char btf_type_name[BTF_MAX_NAME_SIZE];
8951         int ret;
8952
8953         ret = snprintf(btf_type_name, sizeof(btf_type_name),
8954                        "%s%s", prefix, name);
8955         /* snprintf returns the number of characters written excluding the
8956          * terminating null. So, if >= BTF_MAX_NAME_SIZE are written, it
8957          * indicates truncation.
8958          */
8959         if (ret < 0 || ret >= sizeof(btf_type_name))
8960                 return -ENAMETOOLONG;
8961         return btf__find_by_name_kind(btf, btf_type_name, kind);
8962 }
8963
8964 static inline int find_attach_btf_id(struct btf *btf, const char *name,
8965                                      enum bpf_attach_type attach_type)
8966 {
8967         const char *prefix;
8968         int kind;
8969
8970         btf_get_kernel_prefix_kind(attach_type, &prefix, &kind);
8971         return find_btf_by_prefix_kind(btf, prefix, name, kind);
8972 }
8973
8974 int libbpf_find_vmlinux_btf_id(const char *name,
8975                                enum bpf_attach_type attach_type)
8976 {
8977         struct btf *btf;
8978         int err;
8979
8980         btf = btf__load_vmlinux_btf();
8981         err = libbpf_get_error(btf);
8982         if (err) {
8983                 pr_warn("vmlinux BTF is not found\n");
8984                 return libbpf_err(err);
8985         }
8986
8987         err = find_attach_btf_id(btf, name, attach_type);
8988         if (err <= 0)
8989                 pr_warn("%s is not found in vmlinux BTF\n", name);
8990
8991         btf__free(btf);
8992         return libbpf_err(err);
8993 }
8994
8995 static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd)
8996 {
8997         struct bpf_prog_info info;
8998         __u32 info_len = sizeof(info);
8999         struct btf *btf;
9000         int err;
9001
9002         memset(&info, 0, info_len);
9003         err = bpf_obj_get_info_by_fd(attach_prog_fd, &info, &info_len);
9004         if (err) {
9005                 pr_warn("failed bpf_obj_get_info_by_fd for FD %d: %d\n",
9006                         attach_prog_fd, err);
9007                 return err;
9008         }
9009
9010         err = -EINVAL;
9011         if (!info.btf_id) {
9012                 pr_warn("The target program doesn't have BTF\n");
9013                 goto out;
9014         }
9015         btf = btf__load_from_kernel_by_id(info.btf_id);
9016         err = libbpf_get_error(btf);
9017         if (err) {
9018                 pr_warn("Failed to get BTF %d of the program: %d\n", info.btf_id, err);
9019                 goto out;
9020         }
9021         err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
9022         btf__free(btf);
9023         if (err <= 0) {
9024                 pr_warn("%s is not found in prog's BTF\n", name);
9025                 goto out;
9026         }
9027 out:
9028         return err;
9029 }
9030
9031 static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name,
9032                               enum bpf_attach_type attach_type,
9033                               int *btf_obj_fd, int *btf_type_id)
9034 {
9035         int ret, i;
9036
9037         ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type);
9038         if (ret > 0) {
9039                 *btf_obj_fd = 0; /* vmlinux BTF */
9040                 *btf_type_id = ret;
9041                 return 0;
9042         }
9043         if (ret != -ENOENT)
9044                 return ret;
9045
9046         ret = load_module_btfs(obj);
9047         if (ret)
9048                 return ret;
9049
9050         for (i = 0; i < obj->btf_module_cnt; i++) {
9051                 const struct module_btf *mod = &obj->btf_modules[i];
9052
9053                 ret = find_attach_btf_id(mod->btf, attach_name, attach_type);
9054                 if (ret > 0) {
9055                         *btf_obj_fd = mod->fd;
9056                         *btf_type_id = ret;
9057                         return 0;
9058                 }
9059                 if (ret == -ENOENT)
9060                         continue;
9061
9062                 return ret;
9063         }
9064
9065         return -ESRCH;
9066 }
9067
9068 static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name,
9069                                      int *btf_obj_fd, int *btf_type_id)
9070 {
9071         enum bpf_attach_type attach_type = prog->expected_attach_type;
9072         __u32 attach_prog_fd = prog->attach_prog_fd;
9073         int err = 0;
9074
9075         /* BPF program's BTF ID */
9076         if (prog->type == BPF_PROG_TYPE_EXT || attach_prog_fd) {
9077                 if (!attach_prog_fd) {
9078                         pr_warn("prog '%s': attach program FD is not set\n", prog->name);
9079                         return -EINVAL;
9080                 }
9081                 err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd);
9082                 if (err < 0) {
9083                         pr_warn("prog '%s': failed to find BPF program (FD %d) BTF ID for '%s': %d\n",
9084                                  prog->name, attach_prog_fd, attach_name, err);
9085                         return err;
9086                 }
9087                 *btf_obj_fd = 0;
9088                 *btf_type_id = err;
9089                 return 0;
9090         }
9091
9092         /* kernel/module BTF ID */
9093         if (prog->obj->gen_loader) {
9094                 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type);
9095                 *btf_obj_fd = 0;
9096                 *btf_type_id = 1;
9097         } else {
9098                 err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id);
9099         }
9100         if (err) {
9101                 pr_warn("prog '%s': failed to find kernel BTF type ID of '%s': %d\n",
9102                         prog->name, attach_name, err);
9103                 return err;
9104         }
9105         return 0;
9106 }
9107
9108 int libbpf_attach_type_by_name(const char *name,
9109                                enum bpf_attach_type *attach_type)
9110 {
9111         char *type_names;
9112         const struct bpf_sec_def *sec_def;
9113
9114         if (!name)
9115                 return libbpf_err(-EINVAL);
9116
9117         sec_def = find_sec_def(name);
9118         if (!sec_def) {
9119                 pr_debug("failed to guess attach type based on ELF section name '%s'\n", name);
9120                 type_names = libbpf_get_type_names(true);
9121                 if (type_names != NULL) {
9122                         pr_debug("attachable section(type) names are:%s\n", type_names);
9123                         free(type_names);
9124                 }
9125
9126                 return libbpf_err(-EINVAL);
9127         }
9128
9129         if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)
9130                 return libbpf_err(-EINVAL);
9131         if (!(sec_def->cookie & SEC_ATTACHABLE))
9132                 return libbpf_err(-EINVAL);
9133
9134         *attach_type = sec_def->expected_attach_type;
9135         return 0;
9136 }
9137
9138 int bpf_map__fd(const struct bpf_map *map)
9139 {
9140         return map ? map->fd : libbpf_err(-EINVAL);
9141 }
9142
9143 static bool map_uses_real_name(const struct bpf_map *map)
9144 {
9145         /* Since libbpf started to support custom .data.* and .rodata.* maps,
9146          * their user-visible name differs from kernel-visible name. Users see
9147          * such map's corresponding ELF section name as a map name.
9148          * This check distinguishes .data/.rodata from .data.* and .rodata.*
9149          * maps to know which name has to be returned to the user.
9150          */
9151         if (map->libbpf_type == LIBBPF_MAP_DATA && strcmp(map->real_name, DATA_SEC) != 0)
9152                 return true;
9153         if (map->libbpf_type == LIBBPF_MAP_RODATA && strcmp(map->real_name, RODATA_SEC) != 0)
9154                 return true;
9155         return false;
9156 }
9157
9158 const char *bpf_map__name(const struct bpf_map *map)
9159 {
9160         if (!map)
9161                 return NULL;
9162
9163         if (map_uses_real_name(map))
9164                 return map->real_name;
9165
9166         return map->name;
9167 }
9168
9169 enum bpf_map_type bpf_map__type(const struct bpf_map *map)
9170 {
9171         return map->def.type;
9172 }
9173
9174 int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type)
9175 {
9176         if (map->fd >= 0)
9177                 return libbpf_err(-EBUSY);
9178         map->def.type = type;
9179         return 0;
9180 }
9181
9182 __u32 bpf_map__map_flags(const struct bpf_map *map)
9183 {
9184         return map->def.map_flags;
9185 }
9186
9187 int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags)
9188 {
9189         if (map->fd >= 0)
9190                 return libbpf_err(-EBUSY);
9191         map->def.map_flags = flags;
9192         return 0;
9193 }
9194
9195 __u64 bpf_map__map_extra(const struct bpf_map *map)
9196 {
9197         return map->map_extra;
9198 }
9199
9200 int bpf_map__set_map_extra(struct bpf_map *map, __u64 map_extra)
9201 {
9202         if (map->fd >= 0)
9203                 return libbpf_err(-EBUSY);
9204         map->map_extra = map_extra;
9205         return 0;
9206 }
9207
9208 __u32 bpf_map__numa_node(const struct bpf_map *map)
9209 {
9210         return map->numa_node;
9211 }
9212
9213 int bpf_map__set_numa_node(struct bpf_map *map, __u32 numa_node)
9214 {
9215         if (map->fd >= 0)
9216                 return libbpf_err(-EBUSY);
9217         map->numa_node = numa_node;
9218         return 0;
9219 }
9220
9221 __u32 bpf_map__key_size(const struct bpf_map *map)
9222 {
9223         return map->def.key_size;
9224 }
9225
9226 int bpf_map__set_key_size(struct bpf_map *map, __u32 size)
9227 {
9228         if (map->fd >= 0)
9229                 return libbpf_err(-EBUSY);
9230         map->def.key_size = size;
9231         return 0;
9232 }
9233
9234 __u32 bpf_map__value_size(const struct bpf_map *map)
9235 {
9236         return map->def.value_size;
9237 }
9238
9239 int bpf_map__set_value_size(struct bpf_map *map, __u32 size)
9240 {
9241         if (map->fd >= 0)
9242                 return libbpf_err(-EBUSY);
9243         map->def.value_size = size;
9244         return 0;
9245 }
9246
9247 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map)
9248 {
9249         return map ? map->btf_key_type_id : 0;
9250 }
9251
9252 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map)
9253 {
9254         return map ? map->btf_value_type_id : 0;
9255 }
9256
9257 int bpf_map__set_initial_value(struct bpf_map *map,
9258                                const void *data, size_t size)
9259 {
9260         if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG ||
9261             size != map->def.value_size || map->fd >= 0)
9262                 return libbpf_err(-EINVAL);
9263
9264         memcpy(map->mmaped, data, size);
9265         return 0;
9266 }
9267
9268 const void *bpf_map__initial_value(struct bpf_map *map, size_t *psize)
9269 {
9270         if (!map->mmaped)
9271                 return NULL;
9272         *psize = map->def.value_size;
9273         return map->mmaped;
9274 }
9275
9276 bool bpf_map__is_internal(const struct bpf_map *map)
9277 {
9278         return map->libbpf_type != LIBBPF_MAP_UNSPEC;
9279 }
9280
9281 __u32 bpf_map__ifindex(const struct bpf_map *map)
9282 {
9283         return map->map_ifindex;
9284 }
9285
9286 int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex)
9287 {
9288         if (map->fd >= 0)
9289                 return libbpf_err(-EBUSY);
9290         map->map_ifindex = ifindex;
9291         return 0;
9292 }
9293
9294 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd)
9295 {
9296         if (!bpf_map_type__is_map_in_map(map->def.type)) {
9297                 pr_warn("error: unsupported map type\n");
9298                 return libbpf_err(-EINVAL);
9299         }
9300         if (map->inner_map_fd != -1) {
9301                 pr_warn("error: inner_map_fd already specified\n");
9302                 return libbpf_err(-EINVAL);
9303         }
9304         if (map->inner_map) {
9305                 bpf_map__destroy(map->inner_map);
9306                 zfree(&map->inner_map);
9307         }
9308         map->inner_map_fd = fd;
9309         return 0;
9310 }
9311
9312 static struct bpf_map *
9313 __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i)
9314 {
9315         ssize_t idx;
9316         struct bpf_map *s, *e;
9317
9318         if (!obj || !obj->maps)
9319                 return errno = EINVAL, NULL;
9320
9321         s = obj->maps;
9322         e = obj->maps + obj->nr_maps;
9323
9324         if ((m < s) || (m >= e)) {
9325                 pr_warn("error in %s: map handler doesn't belong to object\n",
9326                          __func__);
9327                 return errno = EINVAL, NULL;
9328         }
9329
9330         idx = (m - obj->maps) + i;
9331         if (idx >= obj->nr_maps || idx < 0)
9332                 return NULL;
9333         return &obj->maps[idx];
9334 }
9335
9336 struct bpf_map *
9337 bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *prev)
9338 {
9339         if (prev == NULL)
9340                 return obj->maps;
9341
9342         return __bpf_map__iter(prev, obj, 1);
9343 }
9344
9345 struct bpf_map *
9346 bpf_object__prev_map(const struct bpf_object *obj, const struct bpf_map *next)
9347 {
9348         if (next == NULL) {
9349                 if (!obj->nr_maps)
9350                         return NULL;
9351                 return obj->maps + obj->nr_maps - 1;
9352         }
9353
9354         return __bpf_map__iter(next, obj, -1);
9355 }
9356
9357 struct bpf_map *
9358 bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name)
9359 {
9360         struct bpf_map *pos;
9361
9362         bpf_object__for_each_map(pos, obj) {
9363                 /* if it's a special internal map name (which always starts
9364                  * with dot) then check if that special name matches the
9365                  * real map name (ELF section name)
9366                  */
9367                 if (name[0] == '.') {
9368                         if (pos->real_name && strcmp(pos->real_name, name) == 0)
9369                                 return pos;
9370                         continue;
9371                 }
9372                 /* otherwise map name has to be an exact match */
9373                 if (map_uses_real_name(pos)) {
9374                         if (strcmp(pos->real_name, name) == 0)
9375                                 return pos;
9376                         continue;
9377                 }
9378                 if (strcmp(pos->name, name) == 0)
9379                         return pos;
9380         }
9381         return errno = ENOENT, NULL;
9382 }
9383
9384 int
9385 bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name)
9386 {
9387         return bpf_map__fd(bpf_object__find_map_by_name(obj, name));
9388 }
9389
9390 static int validate_map_op(const struct bpf_map *map, size_t key_sz,
9391                            size_t value_sz, bool check_value_sz)
9392 {
9393         if (map->fd <= 0)
9394                 return -ENOENT;
9395
9396         if (map->def.key_size != key_sz) {
9397                 pr_warn("map '%s': unexpected key size %zu provided, expected %u\n",
9398                         map->name, key_sz, map->def.key_size);
9399                 return -EINVAL;
9400         }
9401
9402         if (!check_value_sz)
9403                 return 0;
9404
9405         switch (map->def.type) {
9406         case BPF_MAP_TYPE_PERCPU_ARRAY:
9407         case BPF_MAP_TYPE_PERCPU_HASH:
9408         case BPF_MAP_TYPE_LRU_PERCPU_HASH:
9409         case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: {
9410                 int num_cpu = libbpf_num_possible_cpus();
9411                 size_t elem_sz = roundup(map->def.value_size, 8);
9412
9413                 if (value_sz != num_cpu * elem_sz) {
9414                         pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zd\n",
9415                                 map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz);
9416                         return -EINVAL;
9417                 }
9418                 break;
9419         }
9420         default:
9421                 if (map->def.value_size != value_sz) {
9422                         pr_warn("map '%s': unexpected value size %zu provided, expected %u\n",
9423                                 map->name, value_sz, map->def.value_size);
9424                         return -EINVAL;
9425                 }
9426                 break;
9427         }
9428         return 0;
9429 }
9430
9431 int bpf_map__lookup_elem(const struct bpf_map *map,
9432                          const void *key, size_t key_sz,
9433                          void *value, size_t value_sz, __u64 flags)
9434 {
9435         int err;
9436
9437         err = validate_map_op(map, key_sz, value_sz, true);
9438         if (err)
9439                 return libbpf_err(err);
9440
9441         return bpf_map_lookup_elem_flags(map->fd, key, value, flags);
9442 }
9443
9444 int bpf_map__update_elem(const struct bpf_map *map,
9445                          const void *key, size_t key_sz,
9446                          const void *value, size_t value_sz, __u64 flags)
9447 {
9448         int err;
9449
9450         err = validate_map_op(map, key_sz, value_sz, true);
9451         if (err)
9452                 return libbpf_err(err);
9453
9454         return bpf_map_update_elem(map->fd, key, value, flags);
9455 }
9456
9457 int bpf_map__delete_elem(const struct bpf_map *map,
9458                          const void *key, size_t key_sz, __u64 flags)
9459 {
9460         int err;
9461
9462         err = validate_map_op(map, key_sz, 0, false /* check_value_sz */);
9463         if (err)
9464                 return libbpf_err(err);
9465
9466         return bpf_map_delete_elem_flags(map->fd, key, flags);
9467 }
9468
9469 int bpf_map__lookup_and_delete_elem(const struct bpf_map *map,
9470                                     const void *key, size_t key_sz,
9471                                     void *value, size_t value_sz, __u64 flags)
9472 {
9473         int err;
9474
9475         err = validate_map_op(map, key_sz, value_sz, true);
9476         if (err)
9477                 return libbpf_err(err);
9478
9479         return bpf_map_lookup_and_delete_elem_flags(map->fd, key, value, flags);
9480 }
9481
9482 int bpf_map__get_next_key(const struct bpf_map *map,
9483                           const void *cur_key, void *next_key, size_t key_sz)
9484 {
9485         int err;
9486
9487         err = validate_map_op(map, key_sz, 0, false /* check_value_sz */);
9488         if (err)
9489                 return libbpf_err(err);
9490
9491         return bpf_map_get_next_key(map->fd, cur_key, next_key);
9492 }
9493
9494 long libbpf_get_error(const void *ptr)
9495 {
9496         if (!IS_ERR_OR_NULL(ptr))
9497                 return 0;
9498
9499         if (IS_ERR(ptr))
9500                 errno = -PTR_ERR(ptr);
9501
9502         /* If ptr == NULL, then errno should be already set by the failing
9503          * API, because libbpf never returns NULL on success and it now always
9504          * sets errno on error. So no extra errno handling for ptr == NULL
9505          * case.
9506          */
9507         return -errno;
9508 }
9509
9510 /* Replace link's underlying BPF program with the new one */
9511 int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog)
9512 {
9513         int ret;
9514
9515         ret = bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL);
9516         return libbpf_err_errno(ret);
9517 }
9518
9519 /* Release "ownership" of underlying BPF resource (typically, BPF program
9520  * attached to some BPF hook, e.g., tracepoint, kprobe, etc). Disconnected
9521  * link, when destructed through bpf_link__destroy() call won't attempt to
9522  * detach/unregisted that BPF resource. This is useful in situations where,
9523  * say, attached BPF program has to outlive userspace program that attached it
9524  * in the system. Depending on type of BPF program, though, there might be
9525  * additional steps (like pinning BPF program in BPF FS) necessary to ensure
9526  * exit of userspace program doesn't trigger automatic detachment and clean up
9527  * inside the kernel.
9528  */
9529 void bpf_link__disconnect(struct bpf_link *link)
9530 {
9531         link->disconnected = true;
9532 }
9533
9534 int bpf_link__destroy(struct bpf_link *link)
9535 {
9536         int err = 0;
9537
9538         if (IS_ERR_OR_NULL(link))
9539                 return 0;
9540
9541         if (!link->disconnected && link->detach)
9542                 err = link->detach(link);
9543         if (link->pin_path)
9544                 free(link->pin_path);
9545         if (link->dealloc)
9546                 link->dealloc(link);
9547         else
9548                 free(link);
9549
9550         return libbpf_err(err);
9551 }
9552
9553 int bpf_link__fd(const struct bpf_link *link)
9554 {
9555         return link->fd;
9556 }
9557
9558 const char *bpf_link__pin_path(const struct bpf_link *link)
9559 {
9560         return link->pin_path;
9561 }
9562
9563 static int bpf_link__detach_fd(struct bpf_link *link)
9564 {
9565         return libbpf_err_errno(close(link->fd));
9566 }
9567
9568 struct bpf_link *bpf_link__open(const char *path)
9569 {
9570         struct bpf_link *link;
9571         int fd;
9572
9573         fd = bpf_obj_get(path);
9574         if (fd < 0) {
9575                 fd = -errno;
9576                 pr_warn("failed to open link at %s: %d\n", path, fd);
9577                 return libbpf_err_ptr(fd);
9578         }
9579
9580         link = calloc(1, sizeof(*link));
9581         if (!link) {
9582                 close(fd);
9583                 return libbpf_err_ptr(-ENOMEM);
9584         }
9585         link->detach = &bpf_link__detach_fd;
9586         link->fd = fd;
9587
9588         link->pin_path = strdup(path);
9589         if (!link->pin_path) {
9590                 bpf_link__destroy(link);
9591                 return libbpf_err_ptr(-ENOMEM);
9592         }
9593
9594         return link;
9595 }
9596
9597 int bpf_link__detach(struct bpf_link *link)
9598 {
9599         return bpf_link_detach(link->fd) ? -errno : 0;
9600 }
9601
9602 int bpf_link__pin(struct bpf_link *link, const char *path)
9603 {
9604         int err;
9605
9606         if (link->pin_path)
9607                 return libbpf_err(-EBUSY);
9608         err = make_parent_dir(path);
9609         if (err)
9610                 return libbpf_err(err);
9611         err = check_path(path);
9612         if (err)
9613                 return libbpf_err(err);
9614
9615         link->pin_path = strdup(path);
9616         if (!link->pin_path)
9617                 return libbpf_err(-ENOMEM);
9618
9619         if (bpf_obj_pin(link->fd, link->pin_path)) {
9620                 err = -errno;
9621                 zfree(&link->pin_path);
9622                 return libbpf_err(err);
9623         }
9624
9625         pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path);
9626         return 0;
9627 }
9628
9629 int bpf_link__unpin(struct bpf_link *link)
9630 {
9631         int err;
9632
9633         if (!link->pin_path)
9634                 return libbpf_err(-EINVAL);
9635
9636         err = unlink(link->pin_path);
9637         if (err != 0)
9638                 return -errno;
9639
9640         pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path);
9641         zfree(&link->pin_path);
9642         return 0;
9643 }
9644
9645 struct bpf_link_perf {
9646         struct bpf_link link;
9647         int perf_event_fd;
9648         /* legacy kprobe support: keep track of probe identifier and type */
9649         char *legacy_probe_name;
9650         bool legacy_is_kprobe;
9651         bool legacy_is_retprobe;
9652 };
9653
9654 static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe);
9655 static int remove_uprobe_event_legacy(const char *probe_name, bool retprobe);
9656
9657 static int bpf_link_perf_detach(struct bpf_link *link)
9658 {
9659         struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
9660         int err = 0;
9661
9662         if (ioctl(perf_link->perf_event_fd, PERF_EVENT_IOC_DISABLE, 0) < 0)
9663                 err = -errno;
9664
9665         if (perf_link->perf_event_fd != link->fd)
9666                 close(perf_link->perf_event_fd);
9667         close(link->fd);
9668
9669         /* legacy uprobe/kprobe needs to be removed after perf event fd closure */
9670         if (perf_link->legacy_probe_name) {
9671                 if (perf_link->legacy_is_kprobe) {
9672                         err = remove_kprobe_event_legacy(perf_link->legacy_probe_name,
9673                                                          perf_link->legacy_is_retprobe);
9674                 } else {
9675                         err = remove_uprobe_event_legacy(perf_link->legacy_probe_name,
9676                                                          perf_link->legacy_is_retprobe);
9677                 }
9678         }
9679
9680         return err;
9681 }
9682
9683 static void bpf_link_perf_dealloc(struct bpf_link *link)
9684 {
9685         struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
9686
9687         free(perf_link->legacy_probe_name);
9688         free(perf_link);
9689 }
9690
9691 struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *prog, int pfd,
9692                                                      const struct bpf_perf_event_opts *opts)
9693 {
9694         char errmsg[STRERR_BUFSIZE];
9695         struct bpf_link_perf *link;
9696         int prog_fd, link_fd = -1, err;
9697
9698         if (!OPTS_VALID(opts, bpf_perf_event_opts))
9699                 return libbpf_err_ptr(-EINVAL);
9700
9701         if (pfd < 0) {
9702                 pr_warn("prog '%s': invalid perf event FD %d\n",
9703                         prog->name, pfd);
9704                 return libbpf_err_ptr(-EINVAL);
9705         }
9706         prog_fd = bpf_program__fd(prog);
9707         if (prog_fd < 0) {
9708                 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n",
9709                         prog->name);
9710                 return libbpf_err_ptr(-EINVAL);
9711         }
9712
9713         link = calloc(1, sizeof(*link));
9714         if (!link)
9715                 return libbpf_err_ptr(-ENOMEM);
9716         link->link.detach = &bpf_link_perf_detach;
9717         link->link.dealloc = &bpf_link_perf_dealloc;
9718         link->perf_event_fd = pfd;
9719
9720         if (kernel_supports(prog->obj, FEAT_PERF_LINK)) {
9721                 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_opts,
9722                         .perf_event.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0));
9723
9724                 link_fd = bpf_link_create(prog_fd, pfd, BPF_PERF_EVENT, &link_opts);
9725                 if (link_fd < 0) {
9726                         err = -errno;
9727                         pr_warn("prog '%s': failed to create BPF link for perf_event FD %d: %d (%s)\n",
9728                                 prog->name, pfd,
9729                                 err, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9730                         goto err_out;
9731                 }
9732                 link->link.fd = link_fd;
9733         } else {
9734                 if (OPTS_GET(opts, bpf_cookie, 0)) {
9735                         pr_warn("prog '%s': user context value is not supported\n", prog->name);
9736                         err = -EOPNOTSUPP;
9737                         goto err_out;
9738                 }
9739
9740                 if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) {
9741                         err = -errno;
9742                         pr_warn("prog '%s': failed to attach to perf_event FD %d: %s\n",
9743                                 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9744                         if (err == -EPROTO)
9745                                 pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n",
9746                                         prog->name, pfd);
9747                         goto err_out;
9748                 }
9749                 link->link.fd = pfd;
9750         }
9751         if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
9752                 err = -errno;
9753                 pr_warn("prog '%s': failed to enable perf_event FD %d: %s\n",
9754                         prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9755                 goto err_out;
9756         }
9757
9758         return &link->link;
9759 err_out:
9760         if (link_fd >= 0)
9761                 close(link_fd);
9762         free(link);
9763         return libbpf_err_ptr(err);
9764 }
9765
9766 struct bpf_link *bpf_program__attach_perf_event(const struct bpf_program *prog, int pfd)
9767 {
9768         return bpf_program__attach_perf_event_opts(prog, pfd, NULL);
9769 }
9770
9771 /*
9772  * this function is expected to parse integer in the range of [0, 2^31-1] from
9773  * given file using scanf format string fmt. If actual parsed value is
9774  * negative, the result might be indistinguishable from error
9775  */
9776 static int parse_uint_from_file(const char *file, const char *fmt)
9777 {
9778         char buf[STRERR_BUFSIZE];
9779         int err, ret;
9780         FILE *f;
9781
9782         f = fopen(file, "r");
9783         if (!f) {
9784                 err = -errno;
9785                 pr_debug("failed to open '%s': %s\n", file,
9786                          libbpf_strerror_r(err, buf, sizeof(buf)));
9787                 return err;
9788         }
9789         err = fscanf(f, fmt, &ret);
9790         if (err != 1) {
9791                 err = err == EOF ? -EIO : -errno;
9792                 pr_debug("failed to parse '%s': %s\n", file,
9793                         libbpf_strerror_r(err, buf, sizeof(buf)));
9794                 fclose(f);
9795                 return err;
9796         }
9797         fclose(f);
9798         return ret;
9799 }
9800
9801 static int determine_kprobe_perf_type(void)
9802 {
9803         const char *file = "/sys/bus/event_source/devices/kprobe/type";
9804
9805         return parse_uint_from_file(file, "%d\n");
9806 }
9807
9808 static int determine_uprobe_perf_type(void)
9809 {
9810         const char *file = "/sys/bus/event_source/devices/uprobe/type";
9811
9812         return parse_uint_from_file(file, "%d\n");
9813 }
9814
9815 static int determine_kprobe_retprobe_bit(void)
9816 {
9817         const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe";
9818
9819         return parse_uint_from_file(file, "config:%d\n");
9820 }
9821
9822 static int determine_uprobe_retprobe_bit(void)
9823 {
9824         const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe";
9825
9826         return parse_uint_from_file(file, "config:%d\n");
9827 }
9828
9829 #define PERF_UPROBE_REF_CTR_OFFSET_BITS 32
9830 #define PERF_UPROBE_REF_CTR_OFFSET_SHIFT 32
9831
9832 static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name,
9833                                  uint64_t offset, int pid, size_t ref_ctr_off)
9834 {
9835         const size_t attr_sz = sizeof(struct perf_event_attr);
9836         struct perf_event_attr attr;
9837         char errmsg[STRERR_BUFSIZE];
9838         int type, pfd;
9839
9840         if (ref_ctr_off >= (1ULL << PERF_UPROBE_REF_CTR_OFFSET_BITS))
9841                 return -EINVAL;
9842
9843         memset(&attr, 0, attr_sz);
9844
9845         type = uprobe ? determine_uprobe_perf_type()
9846                       : determine_kprobe_perf_type();
9847         if (type < 0) {
9848                 pr_warn("failed to determine %s perf type: %s\n",
9849                         uprobe ? "uprobe" : "kprobe",
9850                         libbpf_strerror_r(type, errmsg, sizeof(errmsg)));
9851                 return type;
9852         }
9853         if (retprobe) {
9854                 int bit = uprobe ? determine_uprobe_retprobe_bit()
9855                                  : determine_kprobe_retprobe_bit();
9856
9857                 if (bit < 0) {
9858                         pr_warn("failed to determine %s retprobe bit: %s\n",
9859                                 uprobe ? "uprobe" : "kprobe",
9860                                 libbpf_strerror_r(bit, errmsg, sizeof(errmsg)));
9861                         return bit;
9862                 }
9863                 attr.config |= 1 << bit;
9864         }
9865         attr.size = attr_sz;
9866         attr.type = type;
9867         attr.config |= (__u64)ref_ctr_off << PERF_UPROBE_REF_CTR_OFFSET_SHIFT;
9868         attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */
9869         attr.config2 = offset;           /* kprobe_addr or probe_offset */
9870
9871         /* pid filter is meaningful only for uprobes */
9872         pfd = syscall(__NR_perf_event_open, &attr,
9873                       pid < 0 ? -1 : pid /* pid */,
9874                       pid == -1 ? 0 : -1 /* cpu */,
9875                       -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
9876         return pfd >= 0 ? pfd : -errno;
9877 }
9878
9879 static int append_to_file(const char *file, const char *fmt, ...)
9880 {
9881         int fd, n, err = 0;
9882         va_list ap;
9883
9884         fd = open(file, O_WRONLY | O_APPEND | O_CLOEXEC, 0);
9885         if (fd < 0)
9886                 return -errno;
9887
9888         va_start(ap, fmt);
9889         n = vdprintf(fd, fmt, ap);
9890         va_end(ap);
9891
9892         if (n < 0)
9893                 err = -errno;
9894
9895         close(fd);
9896         return err;
9897 }
9898
9899 #define DEBUGFS "/sys/kernel/debug/tracing"
9900 #define TRACEFS "/sys/kernel/tracing"
9901
9902 static bool use_debugfs(void)
9903 {
9904         static int has_debugfs = -1;
9905
9906         if (has_debugfs < 0)
9907                 has_debugfs = faccessat(AT_FDCWD, DEBUGFS, F_OK, AT_EACCESS) == 0;
9908
9909         return has_debugfs == 1;
9910 }
9911
9912 static const char *tracefs_path(void)
9913 {
9914         return use_debugfs() ? DEBUGFS : TRACEFS;
9915 }
9916
9917 static const char *tracefs_kprobe_events(void)
9918 {
9919         return use_debugfs() ? DEBUGFS"/kprobe_events" : TRACEFS"/kprobe_events";
9920 }
9921
9922 static const char *tracefs_uprobe_events(void)
9923 {
9924         return use_debugfs() ? DEBUGFS"/uprobe_events" : TRACEFS"/uprobe_events";
9925 }
9926
9927 static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz,
9928                                          const char *kfunc_name, size_t offset)
9929 {
9930         static int index = 0;
9931
9932         snprintf(buf, buf_sz, "libbpf_%u_%s_0x%zx_%d", getpid(), kfunc_name, offset,
9933                  __sync_fetch_and_add(&index, 1));
9934 }
9935
9936 static int add_kprobe_event_legacy(const char *probe_name, bool retprobe,
9937                                    const char *kfunc_name, size_t offset)
9938 {
9939         return append_to_file(tracefs_kprobe_events(), "%c:%s/%s %s+0x%zx",
9940                               retprobe ? 'r' : 'p',
9941                               retprobe ? "kretprobes" : "kprobes",
9942                               probe_name, kfunc_name, offset);
9943 }
9944
9945 static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe)
9946 {
9947         return append_to_file(tracefs_kprobe_events(), "-:%s/%s",
9948                               retprobe ? "kretprobes" : "kprobes", probe_name);
9949 }
9950
9951 static int determine_kprobe_perf_type_legacy(const char *probe_name, bool retprobe)
9952 {
9953         char file[256];
9954
9955         snprintf(file, sizeof(file), "%s/events/%s/%s/id",
9956                  tracefs_path(), retprobe ? "kretprobes" : "kprobes", probe_name);
9957
9958         return parse_uint_from_file(file, "%d\n");
9959 }
9960
9961 static int perf_event_kprobe_open_legacy(const char *probe_name, bool retprobe,
9962                                          const char *kfunc_name, size_t offset, int pid)
9963 {
9964         const size_t attr_sz = sizeof(struct perf_event_attr);
9965         struct perf_event_attr attr;
9966         char errmsg[STRERR_BUFSIZE];
9967         int type, pfd, err;
9968
9969         err = add_kprobe_event_legacy(probe_name, retprobe, kfunc_name, offset);
9970         if (err < 0) {
9971                 pr_warn("failed to add legacy kprobe event for '%s+0x%zx': %s\n",
9972                         kfunc_name, offset,
9973                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9974                 return err;
9975         }
9976         type = determine_kprobe_perf_type_legacy(probe_name, retprobe);
9977         if (type < 0) {
9978                 err = type;
9979                 pr_warn("failed to determine legacy kprobe event id for '%s+0x%zx': %s\n",
9980                         kfunc_name, offset,
9981                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9982                 goto err_clean_legacy;
9983         }
9984
9985         memset(&attr, 0, attr_sz);
9986         attr.size = attr_sz;
9987         attr.config = type;
9988         attr.type = PERF_TYPE_TRACEPOINT;
9989
9990         pfd = syscall(__NR_perf_event_open, &attr,
9991                       pid < 0 ? -1 : pid, /* pid */
9992                       pid == -1 ? 0 : -1, /* cpu */
9993                       -1 /* group_fd */,  PERF_FLAG_FD_CLOEXEC);
9994         if (pfd < 0) {
9995                 err = -errno;
9996                 pr_warn("legacy kprobe perf_event_open() failed: %s\n",
9997                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9998                 goto err_clean_legacy;
9999         }
10000         return pfd;
10001
10002 err_clean_legacy:
10003         /* Clear the newly added legacy kprobe_event */
10004         remove_kprobe_event_legacy(probe_name, retprobe);
10005         return err;
10006 }
10007
10008 static const char *arch_specific_syscall_pfx(void)
10009 {
10010 #if defined(__x86_64__)
10011         return "x64";
10012 #elif defined(__i386__)
10013         return "ia32";
10014 #elif defined(__s390x__)
10015         return "s390x";
10016 #elif defined(__s390__)
10017         return "s390";
10018 #elif defined(__arm__)
10019         return "arm";
10020 #elif defined(__aarch64__)
10021         return "arm64";
10022 #elif defined(__mips__)
10023         return "mips";
10024 #elif defined(__riscv)
10025         return "riscv";
10026 #elif defined(__powerpc__)
10027         return "powerpc";
10028 #elif defined(__powerpc64__)
10029         return "powerpc64";
10030 #else
10031         return NULL;
10032 #endif
10033 }
10034
10035 static int probe_kern_syscall_wrapper(void)
10036 {
10037         char syscall_name[64];
10038         const char *ksys_pfx;
10039
10040         ksys_pfx = arch_specific_syscall_pfx();
10041         if (!ksys_pfx)
10042                 return 0;
10043
10044         snprintf(syscall_name, sizeof(syscall_name), "__%s_sys_bpf", ksys_pfx);
10045
10046         if (determine_kprobe_perf_type() >= 0) {
10047                 int pfd;
10048
10049                 pfd = perf_event_open_probe(false, false, syscall_name, 0, getpid(), 0);
10050                 if (pfd >= 0)
10051                         close(pfd);
10052
10053                 return pfd >= 0 ? 1 : 0;
10054         } else { /* legacy mode */
10055                 char probe_name[128];
10056
10057                 gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name), syscall_name, 0);
10058                 if (add_kprobe_event_legacy(probe_name, false, syscall_name, 0) < 0)
10059                         return 0;
10060
10061                 (void)remove_kprobe_event_legacy(probe_name, false);
10062                 return 1;
10063         }
10064 }
10065
10066 struct bpf_link *
10067 bpf_program__attach_kprobe_opts(const struct bpf_program *prog,
10068                                 const char *func_name,
10069                                 const struct bpf_kprobe_opts *opts)
10070 {
10071         DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
10072         char errmsg[STRERR_BUFSIZE];
10073         char *legacy_probe = NULL;
10074         struct bpf_link *link;
10075         size_t offset;
10076         bool retprobe, legacy;
10077         int pfd, err;
10078
10079         if (!OPTS_VALID(opts, bpf_kprobe_opts))
10080                 return libbpf_err_ptr(-EINVAL);
10081
10082         retprobe = OPTS_GET(opts, retprobe, false);
10083         offset = OPTS_GET(opts, offset, 0);
10084         pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
10085
10086         legacy = determine_kprobe_perf_type() < 0;
10087         if (!legacy) {
10088                 pfd = perf_event_open_probe(false /* uprobe */, retprobe,
10089                                             func_name, offset,
10090                                             -1 /* pid */, 0 /* ref_ctr_off */);
10091         } else {
10092                 char probe_name[256];
10093
10094                 gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name),
10095                                              func_name, offset);
10096
10097                 legacy_probe = strdup(probe_name);
10098                 if (!legacy_probe)
10099                         return libbpf_err_ptr(-ENOMEM);
10100
10101                 pfd = perf_event_kprobe_open_legacy(legacy_probe, retprobe, func_name,
10102                                                     offset, -1 /* pid */);
10103         }
10104         if (pfd < 0) {
10105                 err = -errno;
10106                 pr_warn("prog '%s': failed to create %s '%s+0x%zx' perf event: %s\n",
10107                         prog->name, retprobe ? "kretprobe" : "kprobe",
10108                         func_name, offset,
10109                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10110                 goto err_out;
10111         }
10112         link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
10113         err = libbpf_get_error(link);
10114         if (err) {
10115                 close(pfd);
10116                 pr_warn("prog '%s': failed to attach to %s '%s+0x%zx': %s\n",
10117                         prog->name, retprobe ? "kretprobe" : "kprobe",
10118                         func_name, offset,
10119                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10120                 goto err_clean_legacy;
10121         }
10122         if (legacy) {
10123                 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
10124
10125                 perf_link->legacy_probe_name = legacy_probe;
10126                 perf_link->legacy_is_kprobe = true;
10127                 perf_link->legacy_is_retprobe = retprobe;
10128         }
10129
10130         return link;
10131
10132 err_clean_legacy:
10133         if (legacy)
10134                 remove_kprobe_event_legacy(legacy_probe, retprobe);
10135 err_out:
10136         free(legacy_probe);
10137         return libbpf_err_ptr(err);
10138 }
10139
10140 struct bpf_link *bpf_program__attach_kprobe(const struct bpf_program *prog,
10141                                             bool retprobe,
10142                                             const char *func_name)
10143 {
10144         DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts,
10145                 .retprobe = retprobe,
10146         );
10147
10148         return bpf_program__attach_kprobe_opts(prog, func_name, &opts);
10149 }
10150
10151 struct bpf_link *bpf_program__attach_ksyscall(const struct bpf_program *prog,
10152                                               const char *syscall_name,
10153                                               const struct bpf_ksyscall_opts *opts)
10154 {
10155         LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts);
10156         char func_name[128];
10157
10158         if (!OPTS_VALID(opts, bpf_ksyscall_opts))
10159                 return libbpf_err_ptr(-EINVAL);
10160
10161         if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) {
10162                 /* arch_specific_syscall_pfx() should never return NULL here
10163                  * because it is guarded by kernel_supports(). However, since
10164                  * compiler does not know that we have an explicit conditional
10165                  * as well.
10166                  */
10167                 snprintf(func_name, sizeof(func_name), "__%s_sys_%s",
10168                          arch_specific_syscall_pfx() ? : "", syscall_name);
10169         } else {
10170                 snprintf(func_name, sizeof(func_name), "__se_sys_%s", syscall_name);
10171         }
10172
10173         kprobe_opts.retprobe = OPTS_GET(opts, retprobe, false);
10174         kprobe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
10175
10176         return bpf_program__attach_kprobe_opts(prog, func_name, &kprobe_opts);
10177 }
10178
10179 /* Adapted from perf/util/string.c */
10180 static bool glob_match(const char *str, const char *pat)
10181 {
10182         while (*str && *pat && *pat != '*') {
10183                 if (*pat == '?') {      /* Matches any single character */
10184                         str++;
10185                         pat++;
10186                         continue;
10187                 }
10188                 if (*str != *pat)
10189                         return false;
10190                 str++;
10191                 pat++;
10192         }
10193         /* Check wild card */
10194         if (*pat == '*') {
10195                 while (*pat == '*')
10196                         pat++;
10197                 if (!*pat) /* Tail wild card matches all */
10198                         return true;
10199                 while (*str)
10200                         if (glob_match(str++, pat))
10201                                 return true;
10202         }
10203         return !*str && !*pat;
10204 }
10205
10206 struct kprobe_multi_resolve {
10207         const char *pattern;
10208         unsigned long *addrs;
10209         size_t cap;
10210         size_t cnt;
10211 };
10212
10213 static int
10214 resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type,
10215                         const char *sym_name, void *ctx)
10216 {
10217         struct kprobe_multi_resolve *res = ctx;
10218         int err;
10219
10220         if (!glob_match(sym_name, res->pattern))
10221                 return 0;
10222
10223         err = libbpf_ensure_mem((void **) &res->addrs, &res->cap, sizeof(unsigned long),
10224                                 res->cnt + 1);
10225         if (err)
10226                 return err;
10227
10228         res->addrs[res->cnt++] = (unsigned long) sym_addr;
10229         return 0;
10230 }
10231
10232 struct bpf_link *
10233 bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
10234                                       const char *pattern,
10235                                       const struct bpf_kprobe_multi_opts *opts)
10236 {
10237         LIBBPF_OPTS(bpf_link_create_opts, lopts);
10238         struct kprobe_multi_resolve res = {
10239                 .pattern = pattern,
10240         };
10241         struct bpf_link *link = NULL;
10242         char errmsg[STRERR_BUFSIZE];
10243         const unsigned long *addrs;
10244         int err, link_fd, prog_fd;
10245         const __u64 *cookies;
10246         const char **syms;
10247         bool retprobe;
10248         size_t cnt;
10249
10250         if (!OPTS_VALID(opts, bpf_kprobe_multi_opts))
10251                 return libbpf_err_ptr(-EINVAL);
10252
10253         syms    = OPTS_GET(opts, syms, false);
10254         addrs   = OPTS_GET(opts, addrs, false);
10255         cnt     = OPTS_GET(opts, cnt, false);
10256         cookies = OPTS_GET(opts, cookies, false);
10257
10258         if (!pattern && !addrs && !syms)
10259                 return libbpf_err_ptr(-EINVAL);
10260         if (pattern && (addrs || syms || cookies || cnt))
10261                 return libbpf_err_ptr(-EINVAL);
10262         if (!pattern && !cnt)
10263                 return libbpf_err_ptr(-EINVAL);
10264         if (addrs && syms)
10265                 return libbpf_err_ptr(-EINVAL);
10266
10267         if (pattern) {
10268                 err = libbpf_kallsyms_parse(resolve_kprobe_multi_cb, &res);
10269                 if (err)
10270                         goto error;
10271                 if (!res.cnt) {
10272                         err = -ENOENT;
10273                         goto error;
10274                 }
10275                 addrs = res.addrs;
10276                 cnt = res.cnt;
10277         }
10278
10279         retprobe = OPTS_GET(opts, retprobe, false);
10280
10281         lopts.kprobe_multi.syms = syms;
10282         lopts.kprobe_multi.addrs = addrs;
10283         lopts.kprobe_multi.cookies = cookies;
10284         lopts.kprobe_multi.cnt = cnt;
10285         lopts.kprobe_multi.flags = retprobe ? BPF_F_KPROBE_MULTI_RETURN : 0;
10286
10287         link = calloc(1, sizeof(*link));
10288         if (!link) {
10289                 err = -ENOMEM;
10290                 goto error;
10291         }
10292         link->detach = &bpf_link__detach_fd;
10293
10294         prog_fd = bpf_program__fd(prog);
10295         link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, &lopts);
10296         if (link_fd < 0) {
10297                 err = -errno;
10298                 pr_warn("prog '%s': failed to attach: %s\n",
10299                         prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10300                 goto error;
10301         }
10302         link->fd = link_fd;
10303         free(res.addrs);
10304         return link;
10305
10306 error:
10307         free(link);
10308         free(res.addrs);
10309         return libbpf_err_ptr(err);
10310 }
10311
10312 static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link)
10313 {
10314         DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts);
10315         unsigned long offset = 0;
10316         const char *func_name;
10317         char *func;
10318         int n;
10319
10320         *link = NULL;
10321
10322         /* no auto-attach for SEC("kprobe") and SEC("kretprobe") */
10323         if (strcmp(prog->sec_name, "kprobe") == 0 || strcmp(prog->sec_name, "kretprobe") == 0)
10324                 return 0;
10325
10326         opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/");
10327         if (opts.retprobe)
10328                 func_name = prog->sec_name + sizeof("kretprobe/") - 1;
10329         else
10330                 func_name = prog->sec_name + sizeof("kprobe/") - 1;
10331
10332         n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset);
10333         if (n < 1) {
10334                 pr_warn("kprobe name is invalid: %s\n", func_name);
10335                 return -EINVAL;
10336         }
10337         if (opts.retprobe && offset != 0) {
10338                 free(func);
10339                 pr_warn("kretprobes do not support offset specification\n");
10340                 return -EINVAL;
10341         }
10342
10343         opts.offset = offset;
10344         *link = bpf_program__attach_kprobe_opts(prog, func, &opts);
10345         free(func);
10346         return libbpf_get_error(*link);
10347 }
10348
10349 static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link)
10350 {
10351         LIBBPF_OPTS(bpf_ksyscall_opts, opts);
10352         const char *syscall_name;
10353
10354         *link = NULL;
10355
10356         /* no auto-attach for SEC("ksyscall") and SEC("kretsyscall") */
10357         if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0)
10358                 return 0;
10359
10360         opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/");
10361         if (opts.retprobe)
10362                 syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1;
10363         else
10364                 syscall_name = prog->sec_name + sizeof("ksyscall/") - 1;
10365
10366         *link = bpf_program__attach_ksyscall(prog, syscall_name, &opts);
10367         return *link ? 0 : -errno;
10368 }
10369
10370 static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link)
10371 {
10372         LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
10373         const char *spec;
10374         char *pattern;
10375         int n;
10376
10377         *link = NULL;
10378
10379         /* no auto-attach for SEC("kprobe.multi") and SEC("kretprobe.multi") */
10380         if (strcmp(prog->sec_name, "kprobe.multi") == 0 ||
10381             strcmp(prog->sec_name, "kretprobe.multi") == 0)
10382                 return 0;
10383
10384         opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/");
10385         if (opts.retprobe)
10386                 spec = prog->sec_name + sizeof("kretprobe.multi/") - 1;
10387         else
10388                 spec = prog->sec_name + sizeof("kprobe.multi/") - 1;
10389
10390         n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern);
10391         if (n < 1) {
10392                 pr_warn("kprobe multi pattern is invalid: %s\n", pattern);
10393                 return -EINVAL;
10394         }
10395
10396         *link = bpf_program__attach_kprobe_multi_opts(prog, pattern, &opts);
10397         free(pattern);
10398         return libbpf_get_error(*link);
10399 }
10400
10401 static void gen_uprobe_legacy_event_name(char *buf, size_t buf_sz,
10402                                          const char *binary_path, uint64_t offset)
10403 {
10404         int i;
10405
10406         snprintf(buf, buf_sz, "libbpf_%u_%s_0x%zx", getpid(), binary_path, (size_t)offset);
10407
10408         /* sanitize binary_path in the probe name */
10409         for (i = 0; buf[i]; i++) {
10410                 if (!isalnum(buf[i]))
10411                         buf[i] = '_';
10412         }
10413 }
10414
10415 static inline int add_uprobe_event_legacy(const char *probe_name, bool retprobe,
10416                                           const char *binary_path, size_t offset)
10417 {
10418         return append_to_file(tracefs_uprobe_events(), "%c:%s/%s %s:0x%zx",
10419                               retprobe ? 'r' : 'p',
10420                               retprobe ? "uretprobes" : "uprobes",
10421                               probe_name, binary_path, offset);
10422 }
10423
10424 static inline int remove_uprobe_event_legacy(const char *probe_name, bool retprobe)
10425 {
10426         return append_to_file(tracefs_uprobe_events(), "-:%s/%s",
10427                               retprobe ? "uretprobes" : "uprobes", probe_name);
10428 }
10429
10430 static int determine_uprobe_perf_type_legacy(const char *probe_name, bool retprobe)
10431 {
10432         char file[512];
10433
10434         snprintf(file, sizeof(file), "%s/events/%s/%s/id",
10435                  tracefs_path(), retprobe ? "uretprobes" : "uprobes", probe_name);
10436
10437         return parse_uint_from_file(file, "%d\n");
10438 }
10439
10440 static int perf_event_uprobe_open_legacy(const char *probe_name, bool retprobe,
10441                                          const char *binary_path, size_t offset, int pid)
10442 {
10443         const size_t attr_sz = sizeof(struct perf_event_attr);
10444         struct perf_event_attr attr;
10445         int type, pfd, err;
10446
10447         err = add_uprobe_event_legacy(probe_name, retprobe, binary_path, offset);
10448         if (err < 0) {
10449                 pr_warn("failed to add legacy uprobe event for %s:0x%zx: %d\n",
10450                         binary_path, (size_t)offset, err);
10451                 return err;
10452         }
10453         type = determine_uprobe_perf_type_legacy(probe_name, retprobe);
10454         if (type < 0) {
10455                 err = type;
10456                 pr_warn("failed to determine legacy uprobe event id for %s:0x%zx: %d\n",
10457                         binary_path, offset, err);
10458                 goto err_clean_legacy;
10459         }
10460
10461         memset(&attr, 0, attr_sz);
10462         attr.size = attr_sz;
10463         attr.config = type;
10464         attr.type = PERF_TYPE_TRACEPOINT;
10465
10466         pfd = syscall(__NR_perf_event_open, &attr,
10467                       pid < 0 ? -1 : pid, /* pid */
10468                       pid == -1 ? 0 : -1, /* cpu */
10469                       -1 /* group_fd */,  PERF_FLAG_FD_CLOEXEC);
10470         if (pfd < 0) {
10471                 err = -errno;
10472                 pr_warn("legacy uprobe perf_event_open() failed: %d\n", err);
10473                 goto err_clean_legacy;
10474         }
10475         return pfd;
10476
10477 err_clean_legacy:
10478         /* Clear the newly added legacy uprobe_event */
10479         remove_uprobe_event_legacy(probe_name, retprobe);
10480         return err;
10481 }
10482
10483 /* Return next ELF section of sh_type after scn, or first of that type if scn is NULL. */
10484 static Elf_Scn *elf_find_next_scn_by_type(Elf *elf, int sh_type, Elf_Scn *scn)
10485 {
10486         while ((scn = elf_nextscn(elf, scn)) != NULL) {
10487                 GElf_Shdr sh;
10488
10489                 if (!gelf_getshdr(scn, &sh))
10490                         continue;
10491                 if (sh.sh_type == sh_type)
10492                         return scn;
10493         }
10494         return NULL;
10495 }
10496
10497 /* Find offset of function name in object specified by path.  "name" matches
10498  * symbol name or name@@LIB for library functions.
10499  */
10500 static long elf_find_func_offset(const char *binary_path, const char *name)
10501 {
10502         int fd, i, sh_types[2] = { SHT_DYNSYM, SHT_SYMTAB };
10503         bool is_shared_lib, is_name_qualified;
10504         char errmsg[STRERR_BUFSIZE];
10505         long ret = -ENOENT;
10506         size_t name_len;
10507         GElf_Ehdr ehdr;
10508         Elf *elf;
10509
10510         fd = open(binary_path, O_RDONLY | O_CLOEXEC);
10511         if (fd < 0) {
10512                 ret = -errno;
10513                 pr_warn("failed to open %s: %s\n", binary_path,
10514                         libbpf_strerror_r(ret, errmsg, sizeof(errmsg)));
10515                 return ret;
10516         }
10517         elf = elf_begin(fd, ELF_C_READ_MMAP, NULL);
10518         if (!elf) {
10519                 pr_warn("elf: could not read elf from %s: %s\n", binary_path, elf_errmsg(-1));
10520                 close(fd);
10521                 return -LIBBPF_ERRNO__FORMAT;
10522         }
10523         if (!gelf_getehdr(elf, &ehdr)) {
10524                 pr_warn("elf: failed to get ehdr from %s: %s\n", binary_path, elf_errmsg(-1));
10525                 ret = -LIBBPF_ERRNO__FORMAT;
10526                 goto out;
10527         }
10528         /* for shared lib case, we do not need to calculate relative offset */
10529         is_shared_lib = ehdr.e_type == ET_DYN;
10530
10531         name_len = strlen(name);
10532         /* Does name specify "@@LIB"? */
10533         is_name_qualified = strstr(name, "@@") != NULL;
10534
10535         /* Search SHT_DYNSYM, SHT_SYMTAB for symbol.  This search order is used because if
10536          * a binary is stripped, it may only have SHT_DYNSYM, and a fully-statically
10537          * linked binary may not have SHT_DYMSYM, so absence of a section should not be
10538          * reported as a warning/error.
10539          */
10540         for (i = 0; i < ARRAY_SIZE(sh_types); i++) {
10541                 size_t nr_syms, strtabidx, idx;
10542                 Elf_Data *symbols = NULL;
10543                 Elf_Scn *scn = NULL;
10544                 int last_bind = -1;
10545                 const char *sname;
10546                 GElf_Shdr sh;
10547
10548                 scn = elf_find_next_scn_by_type(elf, sh_types[i], NULL);
10549                 if (!scn) {
10550                         pr_debug("elf: failed to find symbol table ELF sections in '%s'\n",
10551                                  binary_path);
10552                         continue;
10553                 }
10554                 if (!gelf_getshdr(scn, &sh))
10555                         continue;
10556                 strtabidx = sh.sh_link;
10557                 symbols = elf_getdata(scn, 0);
10558                 if (!symbols) {
10559                         pr_warn("elf: failed to get symbols for symtab section in '%s': %s\n",
10560                                 binary_path, elf_errmsg(-1));
10561                         ret = -LIBBPF_ERRNO__FORMAT;
10562                         goto out;
10563                 }
10564                 nr_syms = symbols->d_size / sh.sh_entsize;
10565
10566                 for (idx = 0; idx < nr_syms; idx++) {
10567                         int curr_bind;
10568                         GElf_Sym sym;
10569                         Elf_Scn *sym_scn;
10570                         GElf_Shdr sym_sh;
10571
10572                         if (!gelf_getsym(symbols, idx, &sym))
10573                                 continue;
10574
10575                         if (GELF_ST_TYPE(sym.st_info) != STT_FUNC)
10576                                 continue;
10577
10578                         sname = elf_strptr(elf, strtabidx, sym.st_name);
10579                         if (!sname)
10580                                 continue;
10581
10582                         curr_bind = GELF_ST_BIND(sym.st_info);
10583
10584                         /* User can specify func, func@@LIB or func@@LIB_VERSION. */
10585                         if (strncmp(sname, name, name_len) != 0)
10586                                 continue;
10587                         /* ...but we don't want a search for "foo" to match 'foo2" also, so any
10588                          * additional characters in sname should be of the form "@@LIB".
10589                          */
10590                         if (!is_name_qualified && sname[name_len] != '\0' && sname[name_len] != '@')
10591                                 continue;
10592
10593                         if (ret >= 0) {
10594                                 /* handle multiple matches */
10595                                 if (last_bind != STB_WEAK && curr_bind != STB_WEAK) {
10596                                         /* Only accept one non-weak bind. */
10597                                         pr_warn("elf: ambiguous match for '%s', '%s' in '%s'\n",
10598                                                 sname, name, binary_path);
10599                                         ret = -LIBBPF_ERRNO__FORMAT;
10600                                         goto out;
10601                                 } else if (curr_bind == STB_WEAK) {
10602                                         /* already have a non-weak bind, and
10603                                          * this is a weak bind, so ignore.
10604                                          */
10605                                         continue;
10606                                 }
10607                         }
10608
10609                         /* Transform symbol's virtual address (absolute for
10610                          * binaries and relative for shared libs) into file
10611                          * offset, which is what kernel is expecting for
10612                          * uprobe/uretprobe attachment.
10613                          * See Documentation/trace/uprobetracer.rst for more
10614                          * details.
10615                          * This is done by looking up symbol's containing
10616                          * section's header and using it's virtual address
10617                          * (sh_addr) and corresponding file offset (sh_offset)
10618                          * to transform sym.st_value (virtual address) into
10619                          * desired final file offset.
10620                          */
10621                         sym_scn = elf_getscn(elf, sym.st_shndx);
10622                         if (!sym_scn)
10623                                 continue;
10624                         if (!gelf_getshdr(sym_scn, &sym_sh))
10625                                 continue;
10626
10627                         ret = sym.st_value - sym_sh.sh_addr + sym_sh.sh_offset;
10628                         last_bind = curr_bind;
10629                 }
10630                 if (ret > 0)
10631                         break;
10632         }
10633
10634         if (ret > 0) {
10635                 pr_debug("elf: symbol address match for '%s' in '%s': 0x%lx\n", name, binary_path,
10636                          ret);
10637         } else {
10638                 if (ret == 0) {
10639                         pr_warn("elf: '%s' is 0 in symtab for '%s': %s\n", name, binary_path,
10640                                 is_shared_lib ? "should not be 0 in a shared library" :
10641                                                 "try using shared library path instead");
10642                         ret = -ENOENT;
10643                 } else {
10644                         pr_warn("elf: failed to find symbol '%s' in '%s'\n", name, binary_path);
10645                 }
10646         }
10647 out:
10648         elf_end(elf);
10649         close(fd);
10650         return ret;
10651 }
10652
10653 static const char *arch_specific_lib_paths(void)
10654 {
10655         /*
10656          * Based on https://packages.debian.org/sid/libc6.
10657          *
10658          * Assume that the traced program is built for the same architecture
10659          * as libbpf, which should cover the vast majority of cases.
10660          */
10661 #if defined(__x86_64__)
10662         return "/lib/x86_64-linux-gnu";
10663 #elif defined(__i386__)
10664         return "/lib/i386-linux-gnu";
10665 #elif defined(__s390x__)
10666         return "/lib/s390x-linux-gnu";
10667 #elif defined(__s390__)
10668         return "/lib/s390-linux-gnu";
10669 #elif defined(__arm__) && defined(__SOFTFP__)
10670         return "/lib/arm-linux-gnueabi";
10671 #elif defined(__arm__) && !defined(__SOFTFP__)
10672         return "/lib/arm-linux-gnueabihf";
10673 #elif defined(__aarch64__)
10674         return "/lib/aarch64-linux-gnu";
10675 #elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 64
10676         return "/lib/mips64el-linux-gnuabi64";
10677 #elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 32
10678         return "/lib/mipsel-linux-gnu";
10679 #elif defined(__powerpc64__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
10680         return "/lib/powerpc64le-linux-gnu";
10681 #elif defined(__sparc__) && defined(__arch64__)
10682         return "/lib/sparc64-linux-gnu";
10683 #elif defined(__riscv) && __riscv_xlen == 64
10684         return "/lib/riscv64-linux-gnu";
10685 #else
10686         return NULL;
10687 #endif
10688 }
10689
10690 /* Get full path to program/shared library. */
10691 static int resolve_full_path(const char *file, char *result, size_t result_sz)
10692 {
10693         const char *search_paths[3] = {};
10694         int i, perm;
10695
10696         if (str_has_sfx(file, ".so") || strstr(file, ".so.")) {
10697                 search_paths[0] = getenv("LD_LIBRARY_PATH");
10698                 search_paths[1] = "/usr/lib64:/usr/lib";
10699                 search_paths[2] = arch_specific_lib_paths();
10700                 perm = R_OK;
10701         } else {
10702                 search_paths[0] = getenv("PATH");
10703                 search_paths[1] = "/usr/bin:/usr/sbin";
10704                 perm = R_OK | X_OK;
10705         }
10706
10707         for (i = 0; i < ARRAY_SIZE(search_paths); i++) {
10708                 const char *s;
10709
10710                 if (!search_paths[i])
10711                         continue;
10712                 for (s = search_paths[i]; s != NULL; s = strchr(s, ':')) {
10713                         char *next_path;
10714                         int seg_len;
10715
10716                         if (s[0] == ':')
10717                                 s++;
10718                         next_path = strchr(s, ':');
10719                         seg_len = next_path ? next_path - s : strlen(s);
10720                         if (!seg_len)
10721                                 continue;
10722                         snprintf(result, result_sz, "%.*s/%s", seg_len, s, file);
10723                         /* ensure it has required permissions */
10724                         if (faccessat(AT_FDCWD, result, perm, AT_EACCESS) < 0)
10725                                 continue;
10726                         pr_debug("resolved '%s' to '%s'\n", file, result);
10727                         return 0;
10728                 }
10729         }
10730         return -ENOENT;
10731 }
10732
10733 LIBBPF_API struct bpf_link *
10734 bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid,
10735                                 const char *binary_path, size_t func_offset,
10736                                 const struct bpf_uprobe_opts *opts)
10737 {
10738         DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
10739         char errmsg[STRERR_BUFSIZE], *legacy_probe = NULL;
10740         char full_binary_path[PATH_MAX];
10741         struct bpf_link *link;
10742         size_t ref_ctr_off;
10743         int pfd, err;
10744         bool retprobe, legacy;
10745         const char *func_name;
10746
10747         if (!OPTS_VALID(opts, bpf_uprobe_opts))
10748                 return libbpf_err_ptr(-EINVAL);
10749
10750         retprobe = OPTS_GET(opts, retprobe, false);
10751         ref_ctr_off = OPTS_GET(opts, ref_ctr_offset, 0);
10752         pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
10753
10754         if (!binary_path)
10755                 return libbpf_err_ptr(-EINVAL);
10756
10757         if (!strchr(binary_path, '/')) {
10758                 err = resolve_full_path(binary_path, full_binary_path,
10759                                         sizeof(full_binary_path));
10760                 if (err) {
10761                         pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",
10762                                 prog->name, binary_path, err);
10763                         return libbpf_err_ptr(err);
10764                 }
10765                 binary_path = full_binary_path;
10766         }
10767         func_name = OPTS_GET(opts, func_name, NULL);
10768         if (func_name) {
10769                 long sym_off;
10770
10771                 sym_off = elf_find_func_offset(binary_path, func_name);
10772                 if (sym_off < 0)
10773                         return libbpf_err_ptr(sym_off);
10774                 func_offset += sym_off;
10775         }
10776
10777         legacy = determine_uprobe_perf_type() < 0;
10778         if (!legacy) {
10779                 pfd = perf_event_open_probe(true /* uprobe */, retprobe, binary_path,
10780                                             func_offset, pid, ref_ctr_off);
10781         } else {
10782                 char probe_name[PATH_MAX + 64];
10783
10784                 if (ref_ctr_off)
10785                         return libbpf_err_ptr(-EINVAL);
10786
10787                 gen_uprobe_legacy_event_name(probe_name, sizeof(probe_name),
10788                                              binary_path, func_offset);
10789
10790                 legacy_probe = strdup(probe_name);
10791                 if (!legacy_probe)
10792                         return libbpf_err_ptr(-ENOMEM);
10793
10794                 pfd = perf_event_uprobe_open_legacy(legacy_probe, retprobe,
10795                                                     binary_path, func_offset, pid);
10796         }
10797         if (pfd < 0) {
10798                 err = -errno;
10799                 pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n",
10800                         prog->name, retprobe ? "uretprobe" : "uprobe",
10801                         binary_path, func_offset,
10802                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10803                 goto err_out;
10804         }
10805
10806         link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
10807         err = libbpf_get_error(link);
10808         if (err) {
10809                 close(pfd);
10810                 pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n",
10811                         prog->name, retprobe ? "uretprobe" : "uprobe",
10812                         binary_path, func_offset,
10813                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10814                 goto err_clean_legacy;
10815         }
10816         if (legacy) {
10817                 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
10818
10819                 perf_link->legacy_probe_name = legacy_probe;
10820                 perf_link->legacy_is_kprobe = false;
10821                 perf_link->legacy_is_retprobe = retprobe;
10822         }
10823         return link;
10824
10825 err_clean_legacy:
10826         if (legacy)
10827                 remove_uprobe_event_legacy(legacy_probe, retprobe);
10828 err_out:
10829         free(legacy_probe);
10830         return libbpf_err_ptr(err);
10831 }
10832
10833 /* Format of u[ret]probe section definition supporting auto-attach:
10834  * u[ret]probe/binary:function[+offset]
10835  *
10836  * binary can be an absolute/relative path or a filename; the latter is resolved to a
10837  * full binary path via bpf_program__attach_uprobe_opts.
10838  *
10839  * Specifying uprobe+ ensures we carry out strict matching; either "uprobe" must be
10840  * specified (and auto-attach is not possible) or the above format is specified for
10841  * auto-attach.
10842  */
10843 static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link)
10844 {
10845         DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts);
10846         char *probe_type = NULL, *binary_path = NULL, *func_name = NULL;
10847         int n, ret = -EINVAL;
10848         long offset = 0;
10849
10850         *link = NULL;
10851
10852         n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[a-zA-Z0-9_.]+%li",
10853                    &probe_type, &binary_path, &func_name, &offset);
10854         switch (n) {
10855         case 1:
10856                 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */
10857                 ret = 0;
10858                 break;
10859         case 2:
10860                 pr_warn("prog '%s': section '%s' missing ':function[+offset]' specification\n",
10861                         prog->name, prog->sec_name);
10862                 break;
10863         case 3:
10864         case 4:
10865                 opts.retprobe = strcmp(probe_type, "uretprobe") == 0 ||
10866                                 strcmp(probe_type, "uretprobe.s") == 0;
10867                 if (opts.retprobe && offset != 0) {
10868                         pr_warn("prog '%s': uretprobes do not support offset specification\n",
10869                                 prog->name);
10870                         break;
10871                 }
10872                 opts.func_name = func_name;
10873                 *link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts);
10874                 ret = libbpf_get_error(*link);
10875                 break;
10876         default:
10877                 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,
10878                         prog->sec_name);
10879                 break;
10880         }
10881         free(probe_type);
10882         free(binary_path);
10883         free(func_name);
10884
10885         return ret;
10886 }
10887
10888 struct bpf_link *bpf_program__attach_uprobe(const struct bpf_program *prog,
10889                                             bool retprobe, pid_t pid,
10890                                             const char *binary_path,
10891                                             size_t func_offset)
10892 {
10893         DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts, .retprobe = retprobe);
10894
10895         return bpf_program__attach_uprobe_opts(prog, pid, binary_path, func_offset, &opts);
10896 }
10897
10898 struct bpf_link *bpf_program__attach_usdt(const struct bpf_program *prog,
10899                                           pid_t pid, const char *binary_path,
10900                                           const char *usdt_provider, const char *usdt_name,
10901                                           const struct bpf_usdt_opts *opts)
10902 {
10903         char resolved_path[512];
10904         struct bpf_object *obj = prog->obj;
10905         struct bpf_link *link;
10906         __u64 usdt_cookie;
10907         int err;
10908
10909         if (!OPTS_VALID(opts, bpf_uprobe_opts))
10910                 return libbpf_err_ptr(-EINVAL);
10911
10912         if (bpf_program__fd(prog) < 0) {
10913                 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n",
10914                         prog->name);
10915                 return libbpf_err_ptr(-EINVAL);
10916         }
10917
10918         if (!binary_path)
10919                 return libbpf_err_ptr(-EINVAL);
10920
10921         if (!strchr(binary_path, '/')) {
10922                 err = resolve_full_path(binary_path, resolved_path, sizeof(resolved_path));
10923                 if (err) {
10924                         pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",
10925                                 prog->name, binary_path, err);
10926                         return libbpf_err_ptr(err);
10927                 }
10928                 binary_path = resolved_path;
10929         }
10930
10931         /* USDT manager is instantiated lazily on first USDT attach. It will
10932          * be destroyed together with BPF object in bpf_object__close().
10933          */
10934         if (IS_ERR(obj->usdt_man))
10935                 return libbpf_ptr(obj->usdt_man);
10936         if (!obj->usdt_man) {
10937                 obj->usdt_man = usdt_manager_new(obj);
10938                 if (IS_ERR(obj->usdt_man))
10939                         return libbpf_ptr(obj->usdt_man);
10940         }
10941
10942         usdt_cookie = OPTS_GET(opts, usdt_cookie, 0);
10943         link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path,
10944                                         usdt_provider, usdt_name, usdt_cookie);
10945         err = libbpf_get_error(link);
10946         if (err)
10947                 return libbpf_err_ptr(err);
10948         return link;
10949 }
10950
10951 static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link)
10952 {
10953         char *path = NULL, *provider = NULL, *name = NULL;
10954         const char *sec_name;
10955         int n, err;
10956
10957         sec_name = bpf_program__section_name(prog);
10958         if (strcmp(sec_name, "usdt") == 0) {
10959                 /* no auto-attach for just SEC("usdt") */
10960                 *link = NULL;
10961                 return 0;
10962         }
10963
10964         n = sscanf(sec_name, "usdt/%m[^:]:%m[^:]:%m[^:]", &path, &provider, &name);
10965         if (n != 3) {
10966                 pr_warn("invalid section '%s', expected SEC(\"usdt/<path>:<provider>:<name>\")\n",
10967                         sec_name);
10968                 err = -EINVAL;
10969         } else {
10970                 *link = bpf_program__attach_usdt(prog, -1 /* any process */, path,
10971                                                  provider, name, NULL);
10972                 err = libbpf_get_error(*link);
10973         }
10974         free(path);
10975         free(provider);
10976         free(name);
10977         return err;
10978 }
10979
10980 static int determine_tracepoint_id(const char *tp_category,
10981                                    const char *tp_name)
10982 {
10983         char file[PATH_MAX];
10984         int ret;
10985
10986         ret = snprintf(file, sizeof(file), "%s/events/%s/%s/id",
10987                        tracefs_path(), tp_category, tp_name);
10988         if (ret < 0)
10989                 return -errno;
10990         if (ret >= sizeof(file)) {
10991                 pr_debug("tracepoint %s/%s path is too long\n",
10992                          tp_category, tp_name);
10993                 return -E2BIG;
10994         }
10995         return parse_uint_from_file(file, "%d\n");
10996 }
10997
10998 static int perf_event_open_tracepoint(const char *tp_category,
10999                                       const char *tp_name)
11000 {
11001         const size_t attr_sz = sizeof(struct perf_event_attr);
11002         struct perf_event_attr attr;
11003         char errmsg[STRERR_BUFSIZE];
11004         int tp_id, pfd, err;
11005
11006         tp_id = determine_tracepoint_id(tp_category, tp_name);
11007         if (tp_id < 0) {
11008                 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n",
11009                         tp_category, tp_name,
11010                         libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg)));
11011                 return tp_id;
11012         }
11013
11014         memset(&attr, 0, attr_sz);
11015         attr.type = PERF_TYPE_TRACEPOINT;
11016         attr.size = attr_sz;
11017         attr.config = tp_id;
11018
11019         pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */,
11020                       -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
11021         if (pfd < 0) {
11022                 err = -errno;
11023                 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n",
11024                         tp_category, tp_name,
11025                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11026                 return err;
11027         }
11028         return pfd;
11029 }
11030
11031 struct bpf_link *bpf_program__attach_tracepoint_opts(const struct bpf_program *prog,
11032                                                      const char *tp_category,
11033                                                      const char *tp_name,
11034                                                      const struct bpf_tracepoint_opts *opts)
11035 {
11036         DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
11037         char errmsg[STRERR_BUFSIZE];
11038         struct bpf_link *link;
11039         int pfd, err;
11040
11041         if (!OPTS_VALID(opts, bpf_tracepoint_opts))
11042                 return libbpf_err_ptr(-EINVAL);
11043
11044         pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
11045
11046         pfd = perf_event_open_tracepoint(tp_category, tp_name);
11047         if (pfd < 0) {
11048                 pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n",
11049                         prog->name, tp_category, tp_name,
11050                         libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
11051                 return libbpf_err_ptr(pfd);
11052         }
11053         link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
11054         err = libbpf_get_error(link);
11055         if (err) {
11056                 close(pfd);
11057                 pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n",
11058                         prog->name, tp_category, tp_name,
11059                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11060                 return libbpf_err_ptr(err);
11061         }
11062         return link;
11063 }
11064
11065 struct bpf_link *bpf_program__attach_tracepoint(const struct bpf_program *prog,
11066                                                 const char *tp_category,
11067                                                 const char *tp_name)
11068 {
11069         return bpf_program__attach_tracepoint_opts(prog, tp_category, tp_name, NULL);
11070 }
11071
11072 static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11073 {
11074         char *sec_name, *tp_cat, *tp_name;
11075
11076         *link = NULL;
11077
11078         /* no auto-attach for SEC("tp") or SEC("tracepoint") */
11079         if (strcmp(prog->sec_name, "tp") == 0 || strcmp(prog->sec_name, "tracepoint") == 0)
11080                 return 0;
11081
11082         sec_name = strdup(prog->sec_name);
11083         if (!sec_name)
11084                 return -ENOMEM;
11085
11086         /* extract "tp/<category>/<name>" or "tracepoint/<category>/<name>" */
11087         if (str_has_pfx(prog->sec_name, "tp/"))
11088                 tp_cat = sec_name + sizeof("tp/") - 1;
11089         else
11090                 tp_cat = sec_name + sizeof("tracepoint/") - 1;
11091         tp_name = strchr(tp_cat, '/');
11092         if (!tp_name) {
11093                 free(sec_name);
11094                 return -EINVAL;
11095         }
11096         *tp_name = '\0';
11097         tp_name++;
11098
11099         *link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name);
11100         free(sec_name);
11101         return libbpf_get_error(*link);
11102 }
11103
11104 struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *prog,
11105                                                     const char *tp_name)
11106 {
11107         char errmsg[STRERR_BUFSIZE];
11108         struct bpf_link *link;
11109         int prog_fd, pfd;
11110
11111         prog_fd = bpf_program__fd(prog);
11112         if (prog_fd < 0) {
11113                 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
11114                 return libbpf_err_ptr(-EINVAL);
11115         }
11116
11117         link = calloc(1, sizeof(*link));
11118         if (!link)
11119                 return libbpf_err_ptr(-ENOMEM);
11120         link->detach = &bpf_link__detach_fd;
11121
11122         pfd = bpf_raw_tracepoint_open(tp_name, prog_fd);
11123         if (pfd < 0) {
11124                 pfd = -errno;
11125                 free(link);
11126                 pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n",
11127                         prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
11128                 return libbpf_err_ptr(pfd);
11129         }
11130         link->fd = pfd;
11131         return link;
11132 }
11133
11134 static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11135 {
11136         static const char *const prefixes[] = {
11137                 "raw_tp",
11138                 "raw_tracepoint",
11139                 "raw_tp.w",
11140                 "raw_tracepoint.w",
11141         };
11142         size_t i;
11143         const char *tp_name = NULL;
11144
11145         *link = NULL;
11146
11147         for (i = 0; i < ARRAY_SIZE(prefixes); i++) {
11148                 size_t pfx_len;
11149
11150                 if (!str_has_pfx(prog->sec_name, prefixes[i]))
11151                         continue;
11152
11153                 pfx_len = strlen(prefixes[i]);
11154                 /* no auto-attach case of, e.g., SEC("raw_tp") */
11155                 if (prog->sec_name[pfx_len] == '\0')
11156                         return 0;
11157
11158                 if (prog->sec_name[pfx_len] != '/')
11159                         continue;
11160
11161                 tp_name = prog->sec_name + pfx_len + 1;
11162                 break;
11163         }
11164
11165         if (!tp_name) {
11166                 pr_warn("prog '%s': invalid section name '%s'\n",
11167                         prog->name, prog->sec_name);
11168                 return -EINVAL;
11169         }
11170
11171         *link = bpf_program__attach_raw_tracepoint(prog, tp_name);
11172         return libbpf_get_error(*link);
11173 }
11174
11175 /* Common logic for all BPF program types that attach to a btf_id */
11176 static struct bpf_link *bpf_program__attach_btf_id(const struct bpf_program *prog,
11177                                                    const struct bpf_trace_opts *opts)
11178 {
11179         LIBBPF_OPTS(bpf_link_create_opts, link_opts);
11180         char errmsg[STRERR_BUFSIZE];
11181         struct bpf_link *link;
11182         int prog_fd, pfd;
11183
11184         if (!OPTS_VALID(opts, bpf_trace_opts))
11185                 return libbpf_err_ptr(-EINVAL);
11186
11187         prog_fd = bpf_program__fd(prog);
11188         if (prog_fd < 0) {
11189                 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
11190                 return libbpf_err_ptr(-EINVAL);
11191         }
11192
11193         link = calloc(1, sizeof(*link));
11194         if (!link)
11195                 return libbpf_err_ptr(-ENOMEM);
11196         link->detach = &bpf_link__detach_fd;
11197
11198         /* libbpf is smart enough to redirect to BPF_RAW_TRACEPOINT_OPEN on old kernels */
11199         link_opts.tracing.cookie = OPTS_GET(opts, cookie, 0);
11200         pfd = bpf_link_create(prog_fd, 0, bpf_program__expected_attach_type(prog), &link_opts);
11201         if (pfd < 0) {
11202                 pfd = -errno;
11203                 free(link);
11204                 pr_warn("prog '%s': failed to attach: %s\n",
11205                         prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
11206                 return libbpf_err_ptr(pfd);
11207         }
11208         link->fd = pfd;
11209         return link;
11210 }
11211
11212 struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog)
11213 {
11214         return bpf_program__attach_btf_id(prog, NULL);
11215 }
11216
11217 struct bpf_link *bpf_program__attach_trace_opts(const struct bpf_program *prog,
11218                                                 const struct bpf_trace_opts *opts)
11219 {
11220         return bpf_program__attach_btf_id(prog, opts);
11221 }
11222
11223 struct bpf_link *bpf_program__attach_lsm(const struct bpf_program *prog)
11224 {
11225         return bpf_program__attach_btf_id(prog, NULL);
11226 }
11227
11228 static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11229 {
11230         *link = bpf_program__attach_trace(prog);
11231         return libbpf_get_error(*link);
11232 }
11233
11234 static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11235 {
11236         *link = bpf_program__attach_lsm(prog);
11237         return libbpf_get_error(*link);
11238 }
11239
11240 static struct bpf_link *
11241 bpf_program__attach_fd(const struct bpf_program *prog, int target_fd, int btf_id,
11242                        const char *target_name)
11243 {
11244         DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts,
11245                             .target_btf_id = btf_id);
11246         enum bpf_attach_type attach_type;
11247         char errmsg[STRERR_BUFSIZE];
11248         struct bpf_link *link;
11249         int prog_fd, link_fd;
11250
11251         prog_fd = bpf_program__fd(prog);
11252         if (prog_fd < 0) {
11253                 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
11254                 return libbpf_err_ptr(-EINVAL);
11255         }
11256
11257         link = calloc(1, sizeof(*link));
11258         if (!link)
11259                 return libbpf_err_ptr(-ENOMEM);
11260         link->detach = &bpf_link__detach_fd;
11261
11262         attach_type = bpf_program__expected_attach_type(prog);
11263         link_fd = bpf_link_create(prog_fd, target_fd, attach_type, &opts);
11264         if (link_fd < 0) {
11265                 link_fd = -errno;
11266                 free(link);
11267                 pr_warn("prog '%s': failed to attach to %s: %s\n",
11268                         prog->name, target_name,
11269                         libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
11270                 return libbpf_err_ptr(link_fd);
11271         }
11272         link->fd = link_fd;
11273         return link;
11274 }
11275
11276 struct bpf_link *
11277 bpf_program__attach_cgroup(const struct bpf_program *prog, int cgroup_fd)
11278 {
11279         return bpf_program__attach_fd(prog, cgroup_fd, 0, "cgroup");
11280 }
11281
11282 struct bpf_link *
11283 bpf_program__attach_netns(const struct bpf_program *prog, int netns_fd)
11284 {
11285         return bpf_program__attach_fd(prog, netns_fd, 0, "netns");
11286 }
11287
11288 struct bpf_link *bpf_program__attach_xdp(const struct bpf_program *prog, int ifindex)
11289 {
11290         /* target_fd/target_ifindex use the same field in LINK_CREATE */
11291         return bpf_program__attach_fd(prog, ifindex, 0, "xdp");
11292 }
11293
11294 struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog,
11295                                               int target_fd,
11296                                               const char *attach_func_name)
11297 {
11298         int btf_id;
11299
11300         if (!!target_fd != !!attach_func_name) {
11301                 pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n",
11302                         prog->name);
11303                 return libbpf_err_ptr(-EINVAL);
11304         }
11305
11306         if (prog->type != BPF_PROG_TYPE_EXT) {
11307                 pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace",
11308                         prog->name);
11309                 return libbpf_err_ptr(-EINVAL);
11310         }
11311
11312         if (target_fd) {
11313                 btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd);
11314                 if (btf_id < 0)
11315                         return libbpf_err_ptr(btf_id);
11316
11317                 return bpf_program__attach_fd(prog, target_fd, btf_id, "freplace");
11318         } else {
11319                 /* no target, so use raw_tracepoint_open for compatibility
11320                  * with old kernels
11321                  */
11322                 return bpf_program__attach_trace(prog);
11323         }
11324 }
11325
11326 struct bpf_link *
11327 bpf_program__attach_iter(const struct bpf_program *prog,
11328                          const struct bpf_iter_attach_opts *opts)
11329 {
11330         DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);
11331         char errmsg[STRERR_BUFSIZE];
11332         struct bpf_link *link;
11333         int prog_fd, link_fd;
11334         __u32 target_fd = 0;
11335
11336         if (!OPTS_VALID(opts, bpf_iter_attach_opts))
11337                 return libbpf_err_ptr(-EINVAL);
11338
11339         link_create_opts.iter_info = OPTS_GET(opts, link_info, (void *)0);
11340         link_create_opts.iter_info_len = OPTS_GET(opts, link_info_len, 0);
11341
11342         prog_fd = bpf_program__fd(prog);
11343         if (prog_fd < 0) {
11344                 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
11345                 return libbpf_err_ptr(-EINVAL);
11346         }
11347
11348         link = calloc(1, sizeof(*link));
11349         if (!link)
11350                 return libbpf_err_ptr(-ENOMEM);
11351         link->detach = &bpf_link__detach_fd;
11352
11353         link_fd = bpf_link_create(prog_fd, target_fd, BPF_TRACE_ITER,
11354                                   &link_create_opts);
11355         if (link_fd < 0) {
11356                 link_fd = -errno;
11357                 free(link);
11358                 pr_warn("prog '%s': failed to attach to iterator: %s\n",
11359                         prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
11360                 return libbpf_err_ptr(link_fd);
11361         }
11362         link->fd = link_fd;
11363         return link;
11364 }
11365
11366 static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11367 {
11368         *link = bpf_program__attach_iter(prog, NULL);
11369         return libbpf_get_error(*link);
11370 }
11371
11372 struct bpf_link *bpf_program__attach(const struct bpf_program *prog)
11373 {
11374         struct bpf_link *link = NULL;
11375         int err;
11376
11377         if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
11378                 return libbpf_err_ptr(-EOPNOTSUPP);
11379
11380         err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link);
11381         if (err)
11382                 return libbpf_err_ptr(err);
11383
11384         /* When calling bpf_program__attach() explicitly, auto-attach support
11385          * is expected to work, so NULL returned link is considered an error.
11386          * This is different for skeleton's attach, see comment in
11387          * bpf_object__attach_skeleton().
11388          */
11389         if (!link)
11390                 return libbpf_err_ptr(-EOPNOTSUPP);
11391
11392         return link;
11393 }
11394
11395 static int bpf_link__detach_struct_ops(struct bpf_link *link)
11396 {
11397         __u32 zero = 0;
11398
11399         if (bpf_map_delete_elem(link->fd, &zero))
11400                 return -errno;
11401
11402         return 0;
11403 }
11404
11405 struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map)
11406 {
11407         struct bpf_struct_ops *st_ops;
11408         struct bpf_link *link;
11409         __u32 i, zero = 0;
11410         int err;
11411
11412         if (!bpf_map__is_struct_ops(map) || map->fd == -1)
11413                 return libbpf_err_ptr(-EINVAL);
11414
11415         link = calloc(1, sizeof(*link));
11416         if (!link)
11417                 return libbpf_err_ptr(-EINVAL);
11418
11419         st_ops = map->st_ops;
11420         for (i = 0; i < btf_vlen(st_ops->type); i++) {
11421                 struct bpf_program *prog = st_ops->progs[i];
11422                 void *kern_data;
11423                 int prog_fd;
11424
11425                 if (!prog)
11426                         continue;
11427
11428                 prog_fd = bpf_program__fd(prog);
11429                 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i];
11430                 *(unsigned long *)kern_data = prog_fd;
11431         }
11432
11433         err = bpf_map_update_elem(map->fd, &zero, st_ops->kern_vdata, 0);
11434         if (err) {
11435                 err = -errno;
11436                 free(link);
11437                 return libbpf_err_ptr(err);
11438         }
11439
11440         link->detach = bpf_link__detach_struct_ops;
11441         link->fd = map->fd;
11442
11443         return link;
11444 }
11445
11446 typedef enum bpf_perf_event_ret (*bpf_perf_event_print_t)(struct perf_event_header *hdr,
11447                                                           void *private_data);
11448
11449 static enum bpf_perf_event_ret
11450 perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
11451                        void **copy_mem, size_t *copy_size,
11452                        bpf_perf_event_print_t fn, void *private_data)
11453 {
11454         struct perf_event_mmap_page *header = mmap_mem;
11455         __u64 data_head = ring_buffer_read_head(header);
11456         __u64 data_tail = header->data_tail;
11457         void *base = ((__u8 *)header) + page_size;
11458         int ret = LIBBPF_PERF_EVENT_CONT;
11459         struct perf_event_header *ehdr;
11460         size_t ehdr_size;
11461
11462         while (data_head != data_tail) {
11463                 ehdr = base + (data_tail & (mmap_size - 1));
11464                 ehdr_size = ehdr->size;
11465
11466                 if (((void *)ehdr) + ehdr_size > base + mmap_size) {
11467                         void *copy_start = ehdr;
11468                         size_t len_first = base + mmap_size - copy_start;
11469                         size_t len_secnd = ehdr_size - len_first;
11470
11471                         if (*copy_size < ehdr_size) {
11472                                 free(*copy_mem);
11473                                 *copy_mem = malloc(ehdr_size);
11474                                 if (!*copy_mem) {
11475                                         *copy_size = 0;
11476                                         ret = LIBBPF_PERF_EVENT_ERROR;
11477                                         break;
11478                                 }
11479                                 *copy_size = ehdr_size;
11480                         }
11481
11482                         memcpy(*copy_mem, copy_start, len_first);
11483                         memcpy(*copy_mem + len_first, base, len_secnd);
11484                         ehdr = *copy_mem;
11485                 }
11486
11487                 ret = fn(ehdr, private_data);
11488                 data_tail += ehdr_size;
11489                 if (ret != LIBBPF_PERF_EVENT_CONT)
11490                         break;
11491         }
11492
11493         ring_buffer_write_tail(header, data_tail);
11494         return libbpf_err(ret);
11495 }
11496
11497 struct perf_buffer;
11498
11499 struct perf_buffer_params {
11500         struct perf_event_attr *attr;
11501         /* if event_cb is specified, it takes precendence */
11502         perf_buffer_event_fn event_cb;
11503         /* sample_cb and lost_cb are higher-level common-case callbacks */
11504         perf_buffer_sample_fn sample_cb;
11505         perf_buffer_lost_fn lost_cb;
11506         void *ctx;
11507         int cpu_cnt;
11508         int *cpus;
11509         int *map_keys;
11510 };
11511
11512 struct perf_cpu_buf {
11513         struct perf_buffer *pb;
11514         void *base; /* mmap()'ed memory */
11515         void *buf; /* for reconstructing segmented data */
11516         size_t buf_size;
11517         int fd;
11518         int cpu;
11519         int map_key;
11520 };
11521
11522 struct perf_buffer {
11523         perf_buffer_event_fn event_cb;
11524         perf_buffer_sample_fn sample_cb;
11525         perf_buffer_lost_fn lost_cb;
11526         void *ctx; /* passed into callbacks */
11527
11528         size_t page_size;
11529         size_t mmap_size;
11530         struct perf_cpu_buf **cpu_bufs;
11531         struct epoll_event *events;
11532         int cpu_cnt; /* number of allocated CPU buffers */
11533         int epoll_fd; /* perf event FD */
11534         int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */
11535 };
11536
11537 static void perf_buffer__free_cpu_buf(struct perf_buffer *pb,
11538                                       struct perf_cpu_buf *cpu_buf)
11539 {
11540         if (!cpu_buf)
11541                 return;
11542         if (cpu_buf->base &&
11543             munmap(cpu_buf->base, pb->mmap_size + pb->page_size))
11544                 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu);
11545         if (cpu_buf->fd >= 0) {
11546                 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0);
11547                 close(cpu_buf->fd);
11548         }
11549         free(cpu_buf->buf);
11550         free(cpu_buf);
11551 }
11552
11553 void perf_buffer__free(struct perf_buffer *pb)
11554 {
11555         int i;
11556
11557         if (IS_ERR_OR_NULL(pb))
11558                 return;
11559         if (pb->cpu_bufs) {
11560                 for (i = 0; i < pb->cpu_cnt; i++) {
11561                         struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
11562
11563                         if (!cpu_buf)
11564                                 continue;
11565
11566                         bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);
11567                         perf_buffer__free_cpu_buf(pb, cpu_buf);
11568                 }
11569                 free(pb->cpu_bufs);
11570         }
11571         if (pb->epoll_fd >= 0)
11572                 close(pb->epoll_fd);
11573         free(pb->events);
11574         free(pb);
11575 }
11576
11577 static struct perf_cpu_buf *
11578 perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr,
11579                           int cpu, int map_key)
11580 {
11581         struct perf_cpu_buf *cpu_buf;
11582         char msg[STRERR_BUFSIZE];
11583         int err;
11584
11585         cpu_buf = calloc(1, sizeof(*cpu_buf));
11586         if (!cpu_buf)
11587                 return ERR_PTR(-ENOMEM);
11588
11589         cpu_buf->pb = pb;
11590         cpu_buf->cpu = cpu;
11591         cpu_buf->map_key = map_key;
11592
11593         cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu,
11594                               -1, PERF_FLAG_FD_CLOEXEC);
11595         if (cpu_buf->fd < 0) {
11596                 err = -errno;
11597                 pr_warn("failed to open perf buffer event on cpu #%d: %s\n",
11598                         cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
11599                 goto error;
11600         }
11601
11602         cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,
11603                              PROT_READ | PROT_WRITE, MAP_SHARED,
11604                              cpu_buf->fd, 0);
11605         if (cpu_buf->base == MAP_FAILED) {
11606                 cpu_buf->base = NULL;
11607                 err = -errno;
11608                 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n",
11609                         cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
11610                 goto error;
11611         }
11612
11613         if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
11614                 err = -errno;
11615                 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n",
11616                         cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
11617                 goto error;
11618         }
11619
11620         return cpu_buf;
11621
11622 error:
11623         perf_buffer__free_cpu_buf(pb, cpu_buf);
11624         return (struct perf_cpu_buf *)ERR_PTR(err);
11625 }
11626
11627 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
11628                                               struct perf_buffer_params *p);
11629
11630 struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt,
11631                                      perf_buffer_sample_fn sample_cb,
11632                                      perf_buffer_lost_fn lost_cb,
11633                                      void *ctx,
11634                                      const struct perf_buffer_opts *opts)
11635 {
11636         const size_t attr_sz = sizeof(struct perf_event_attr);
11637         struct perf_buffer_params p = {};
11638         struct perf_event_attr attr;
11639
11640         if (!OPTS_VALID(opts, perf_buffer_opts))
11641                 return libbpf_err_ptr(-EINVAL);
11642
11643         memset(&attr, 0, attr_sz);
11644         attr.size = attr_sz;
11645         attr.config = PERF_COUNT_SW_BPF_OUTPUT;
11646         attr.type = PERF_TYPE_SOFTWARE;
11647         attr.sample_type = PERF_SAMPLE_RAW;
11648         attr.sample_period = 1;
11649         attr.wakeup_events = 1;
11650
11651         p.attr = &attr;
11652         p.sample_cb = sample_cb;
11653         p.lost_cb = lost_cb;
11654         p.ctx = ctx;
11655
11656         return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p));
11657 }
11658
11659 struct perf_buffer *perf_buffer__new_raw(int map_fd, size_t page_cnt,
11660                                          struct perf_event_attr *attr,
11661                                          perf_buffer_event_fn event_cb, void *ctx,
11662                                          const struct perf_buffer_raw_opts *opts)
11663 {
11664         struct perf_buffer_params p = {};
11665
11666         if (!attr)
11667                 return libbpf_err_ptr(-EINVAL);
11668
11669         if (!OPTS_VALID(opts, perf_buffer_raw_opts))
11670                 return libbpf_err_ptr(-EINVAL);
11671
11672         p.attr = attr;
11673         p.event_cb = event_cb;
11674         p.ctx = ctx;
11675         p.cpu_cnt = OPTS_GET(opts, cpu_cnt, 0);
11676         p.cpus = OPTS_GET(opts, cpus, NULL);
11677         p.map_keys = OPTS_GET(opts, map_keys, NULL);
11678
11679         return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p));
11680 }
11681
11682 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
11683                                               struct perf_buffer_params *p)
11684 {
11685         const char *online_cpus_file = "/sys/devices/system/cpu/online";
11686         struct bpf_map_info map;
11687         char msg[STRERR_BUFSIZE];
11688         struct perf_buffer *pb;
11689         bool *online = NULL;
11690         __u32 map_info_len;
11691         int err, i, j, n;
11692
11693         if (page_cnt == 0 || (page_cnt & (page_cnt - 1))) {
11694                 pr_warn("page count should be power of two, but is %zu\n",
11695                         page_cnt);
11696                 return ERR_PTR(-EINVAL);
11697         }
11698
11699         /* best-effort sanity checks */
11700         memset(&map, 0, sizeof(map));
11701         map_info_len = sizeof(map);
11702         err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len);
11703         if (err) {
11704                 err = -errno;
11705                 /* if BPF_OBJ_GET_INFO_BY_FD is supported, will return
11706                  * -EBADFD, -EFAULT, or -E2BIG on real error
11707                  */
11708                 if (err != -EINVAL) {
11709                         pr_warn("failed to get map info for map FD %d: %s\n",
11710                                 map_fd, libbpf_strerror_r(err, msg, sizeof(msg)));
11711                         return ERR_PTR(err);
11712                 }
11713                 pr_debug("failed to get map info for FD %d; API not supported? Ignoring...\n",
11714                          map_fd);
11715         } else {
11716                 if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) {
11717                         pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n",
11718                                 map.name);
11719                         return ERR_PTR(-EINVAL);
11720                 }
11721         }
11722
11723         pb = calloc(1, sizeof(*pb));
11724         if (!pb)
11725                 return ERR_PTR(-ENOMEM);
11726
11727         pb->event_cb = p->event_cb;
11728         pb->sample_cb = p->sample_cb;
11729         pb->lost_cb = p->lost_cb;
11730         pb->ctx = p->ctx;
11731
11732         pb->page_size = getpagesize();
11733         pb->mmap_size = pb->page_size * page_cnt;
11734         pb->map_fd = map_fd;
11735
11736         pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
11737         if (pb->epoll_fd < 0) {
11738                 err = -errno;
11739                 pr_warn("failed to create epoll instance: %s\n",
11740                         libbpf_strerror_r(err, msg, sizeof(msg)));
11741                 goto error;
11742         }
11743
11744         if (p->cpu_cnt > 0) {
11745                 pb->cpu_cnt = p->cpu_cnt;
11746         } else {
11747                 pb->cpu_cnt = libbpf_num_possible_cpus();
11748                 if (pb->cpu_cnt < 0) {
11749                         err = pb->cpu_cnt;
11750                         goto error;
11751                 }
11752                 if (map.max_entries && map.max_entries < pb->cpu_cnt)
11753                         pb->cpu_cnt = map.max_entries;
11754         }
11755
11756         pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));
11757         if (!pb->events) {
11758                 err = -ENOMEM;
11759                 pr_warn("failed to allocate events: out of memory\n");
11760                 goto error;
11761         }
11762         pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));
11763         if (!pb->cpu_bufs) {
11764                 err = -ENOMEM;
11765                 pr_warn("failed to allocate buffers: out of memory\n");
11766                 goto error;
11767         }
11768
11769         err = parse_cpu_mask_file(online_cpus_file, &online, &n);
11770         if (err) {
11771                 pr_warn("failed to get online CPU mask: %d\n", err);
11772                 goto error;
11773         }
11774
11775         for (i = 0, j = 0; i < pb->cpu_cnt; i++) {
11776                 struct perf_cpu_buf *cpu_buf;
11777                 int cpu, map_key;
11778
11779                 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i;
11780                 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i;
11781
11782                 /* in case user didn't explicitly requested particular CPUs to
11783                  * be attached to, skip offline/not present CPUs
11784                  */
11785                 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu]))
11786                         continue;
11787
11788                 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);
11789                 if (IS_ERR(cpu_buf)) {
11790                         err = PTR_ERR(cpu_buf);
11791                         goto error;
11792                 }
11793
11794                 pb->cpu_bufs[j] = cpu_buf;
11795
11796                 err = bpf_map_update_elem(pb->map_fd, &map_key,
11797                                           &cpu_buf->fd, 0);
11798                 if (err) {
11799                         err = -errno;
11800                         pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",
11801                                 cpu, map_key, cpu_buf->fd,
11802                                 libbpf_strerror_r(err, msg, sizeof(msg)));
11803                         goto error;
11804                 }
11805
11806                 pb->events[j].events = EPOLLIN;
11807                 pb->events[j].data.ptr = cpu_buf;
11808                 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,
11809                               &pb->events[j]) < 0) {
11810                         err = -errno;
11811                         pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n",
11812                                 cpu, cpu_buf->fd,
11813                                 libbpf_strerror_r(err, msg, sizeof(msg)));
11814                         goto error;
11815                 }
11816                 j++;
11817         }
11818         pb->cpu_cnt = j;
11819         free(online);
11820
11821         return pb;
11822
11823 error:
11824         free(online);
11825         if (pb)
11826                 perf_buffer__free(pb);
11827         return ERR_PTR(err);
11828 }
11829
11830 struct perf_sample_raw {
11831         struct perf_event_header header;
11832         uint32_t size;
11833         char data[];
11834 };
11835
11836 struct perf_sample_lost {
11837         struct perf_event_header header;
11838         uint64_t id;
11839         uint64_t lost;
11840         uint64_t sample_id;
11841 };
11842
11843 static enum bpf_perf_event_ret
11844 perf_buffer__process_record(struct perf_event_header *e, void *ctx)
11845 {
11846         struct perf_cpu_buf *cpu_buf = ctx;
11847         struct perf_buffer *pb = cpu_buf->pb;
11848         void *data = e;
11849
11850         /* user wants full control over parsing perf event */
11851         if (pb->event_cb)
11852                 return pb->event_cb(pb->ctx, cpu_buf->cpu, e);
11853
11854         switch (e->type) {
11855         case PERF_RECORD_SAMPLE: {
11856                 struct perf_sample_raw *s = data;
11857
11858                 if (pb->sample_cb)
11859                         pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);
11860                 break;
11861         }
11862         case PERF_RECORD_LOST: {
11863                 struct perf_sample_lost *s = data;
11864
11865                 if (pb->lost_cb)
11866                         pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);
11867                 break;
11868         }
11869         default:
11870                 pr_warn("unknown perf sample type %d\n", e->type);
11871                 return LIBBPF_PERF_EVENT_ERROR;
11872         }
11873         return LIBBPF_PERF_EVENT_CONT;
11874 }
11875
11876 static int perf_buffer__process_records(struct perf_buffer *pb,
11877                                         struct perf_cpu_buf *cpu_buf)
11878 {
11879         enum bpf_perf_event_ret ret;
11880
11881         ret = perf_event_read_simple(cpu_buf->base, pb->mmap_size,
11882                                      pb->page_size, &cpu_buf->buf,
11883                                      &cpu_buf->buf_size,
11884                                      perf_buffer__process_record, cpu_buf);
11885         if (ret != LIBBPF_PERF_EVENT_CONT)
11886                 return ret;
11887         return 0;
11888 }
11889
11890 int perf_buffer__epoll_fd(const struct perf_buffer *pb)
11891 {
11892         return pb->epoll_fd;
11893 }
11894
11895 int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms)
11896 {
11897         int i, cnt, err;
11898
11899         cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);
11900         if (cnt < 0)
11901                 return -errno;
11902
11903         for (i = 0; i < cnt; i++) {
11904                 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;
11905
11906                 err = perf_buffer__process_records(pb, cpu_buf);
11907                 if (err) {
11908                         pr_warn("error while processing records: %d\n", err);
11909                         return libbpf_err(err);
11910                 }
11911         }
11912         return cnt;
11913 }
11914
11915 /* Return number of PERF_EVENT_ARRAY map slots set up by this perf_buffer
11916  * manager.
11917  */
11918 size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb)
11919 {
11920         return pb->cpu_cnt;
11921 }
11922
11923 /*
11924  * Return perf_event FD of a ring buffer in *buf_idx* slot of
11925  * PERF_EVENT_ARRAY BPF map. This FD can be polled for new data using
11926  * select()/poll()/epoll() Linux syscalls.
11927  */
11928 int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx)
11929 {
11930         struct perf_cpu_buf *cpu_buf;
11931
11932         if (buf_idx >= pb->cpu_cnt)
11933                 return libbpf_err(-EINVAL);
11934
11935         cpu_buf = pb->cpu_bufs[buf_idx];
11936         if (!cpu_buf)
11937                 return libbpf_err(-ENOENT);
11938
11939         return cpu_buf->fd;
11940 }
11941
11942 int perf_buffer__buffer(struct perf_buffer *pb, int buf_idx, void **buf, size_t *buf_size)
11943 {
11944         struct perf_cpu_buf *cpu_buf;
11945
11946         if (buf_idx >= pb->cpu_cnt)
11947                 return libbpf_err(-EINVAL);
11948
11949         cpu_buf = pb->cpu_bufs[buf_idx];
11950         if (!cpu_buf)
11951                 return libbpf_err(-ENOENT);
11952
11953         *buf = cpu_buf->base;
11954         *buf_size = pb->mmap_size;
11955         return 0;
11956 }
11957
11958 /*
11959  * Consume data from perf ring buffer corresponding to slot *buf_idx* in
11960  * PERF_EVENT_ARRAY BPF map without waiting/polling. If there is no data to
11961  * consume, do nothing and return success.
11962  * Returns:
11963  *   - 0 on success;
11964  *   - <0 on failure.
11965  */
11966 int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx)
11967 {
11968         struct perf_cpu_buf *cpu_buf;
11969
11970         if (buf_idx >= pb->cpu_cnt)
11971                 return libbpf_err(-EINVAL);
11972
11973         cpu_buf = pb->cpu_bufs[buf_idx];
11974         if (!cpu_buf)
11975                 return libbpf_err(-ENOENT);
11976
11977         return perf_buffer__process_records(pb, cpu_buf);
11978 }
11979
11980 int perf_buffer__consume(struct perf_buffer *pb)
11981 {
11982         int i, err;
11983
11984         for (i = 0; i < pb->cpu_cnt; i++) {
11985                 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
11986
11987                 if (!cpu_buf)
11988                         continue;
11989
11990                 err = perf_buffer__process_records(pb, cpu_buf);
11991                 if (err) {
11992                         pr_warn("perf_buffer: failed to process records in buffer #%d: %d\n", i, err);
11993                         return libbpf_err(err);
11994                 }
11995         }
11996         return 0;
11997 }
11998
11999 int bpf_program__set_attach_target(struct bpf_program *prog,
12000                                    int attach_prog_fd,
12001                                    const char *attach_func_name)
12002 {
12003         int btf_obj_fd = 0, btf_id = 0, err;
12004
12005         if (!prog || attach_prog_fd < 0)
12006                 return libbpf_err(-EINVAL);
12007
12008         if (prog->obj->loaded)
12009                 return libbpf_err(-EINVAL);
12010
12011         if (attach_prog_fd && !attach_func_name) {
12012                 /* remember attach_prog_fd and let bpf_program__load() find
12013                  * BTF ID during the program load
12014                  */
12015                 prog->attach_prog_fd = attach_prog_fd;
12016                 return 0;
12017         }
12018
12019         if (attach_prog_fd) {
12020                 btf_id = libbpf_find_prog_btf_id(attach_func_name,
12021                                                  attach_prog_fd);
12022                 if (btf_id < 0)
12023                         return libbpf_err(btf_id);
12024         } else {
12025                 if (!attach_func_name)
12026                         return libbpf_err(-EINVAL);
12027
12028                 /* load btf_vmlinux, if not yet */
12029                 err = bpf_object__load_vmlinux_btf(prog->obj, true);
12030                 if (err)
12031                         return libbpf_err(err);
12032                 err = find_kernel_btf_id(prog->obj, attach_func_name,
12033                                          prog->expected_attach_type,
12034                                          &btf_obj_fd, &btf_id);
12035                 if (err)
12036                         return libbpf_err(err);
12037         }
12038
12039         prog->attach_btf_id = btf_id;
12040         prog->attach_btf_obj_fd = btf_obj_fd;
12041         prog->attach_prog_fd = attach_prog_fd;
12042         return 0;
12043 }
12044
12045 int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz)
12046 {
12047         int err = 0, n, len, start, end = -1;
12048         bool *tmp;
12049
12050         *mask = NULL;
12051         *mask_sz = 0;
12052
12053         /* Each sub string separated by ',' has format \d+-\d+ or \d+ */
12054         while (*s) {
12055                 if (*s == ',' || *s == '\n') {
12056                         s++;
12057                         continue;
12058                 }
12059                 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len);
12060                 if (n <= 0 || n > 2) {
12061                         pr_warn("Failed to get CPU range %s: %d\n", s, n);
12062                         err = -EINVAL;
12063                         goto cleanup;
12064                 } else if (n == 1) {
12065                         end = start;
12066                 }
12067                 if (start < 0 || start > end) {
12068                         pr_warn("Invalid CPU range [%d,%d] in %s\n",
12069                                 start, end, s);
12070                         err = -EINVAL;
12071                         goto cleanup;
12072                 }
12073                 tmp = realloc(*mask, end + 1);
12074                 if (!tmp) {
12075                         err = -ENOMEM;
12076                         goto cleanup;
12077                 }
12078                 *mask = tmp;
12079                 memset(tmp + *mask_sz, 0, start - *mask_sz);
12080                 memset(tmp + start, 1, end - start + 1);
12081                 *mask_sz = end + 1;
12082                 s += len;
12083         }
12084         if (!*mask_sz) {
12085                 pr_warn("Empty CPU range\n");
12086                 return -EINVAL;
12087         }
12088         return 0;
12089 cleanup:
12090         free(*mask);
12091         *mask = NULL;
12092         return err;
12093 }
12094
12095 int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz)
12096 {
12097         int fd, err = 0, len;
12098         char buf[128];
12099
12100         fd = open(fcpu, O_RDONLY | O_CLOEXEC);
12101         if (fd < 0) {
12102                 err = -errno;
12103                 pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err);
12104                 return err;
12105         }
12106         len = read(fd, buf, sizeof(buf));
12107         close(fd);
12108         if (len <= 0) {
12109                 err = len ? -errno : -EINVAL;
12110                 pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err);
12111                 return err;
12112         }
12113         if (len >= sizeof(buf)) {
12114                 pr_warn("CPU mask is too big in file %s\n", fcpu);
12115                 return -E2BIG;
12116         }
12117         buf[len] = '\0';
12118
12119         return parse_cpu_mask_str(buf, mask, mask_sz);
12120 }
12121
12122 int libbpf_num_possible_cpus(void)
12123 {
12124         static const char *fcpu = "/sys/devices/system/cpu/possible";
12125         static int cpus;
12126         int err, n, i, tmp_cpus;
12127         bool *mask;
12128
12129         tmp_cpus = READ_ONCE(cpus);
12130         if (tmp_cpus > 0)
12131                 return tmp_cpus;
12132
12133         err = parse_cpu_mask_file(fcpu, &mask, &n);
12134         if (err)
12135                 return libbpf_err(err);
12136
12137         tmp_cpus = 0;
12138         for (i = 0; i < n; i++) {
12139                 if (mask[i])
12140                         tmp_cpus++;
12141         }
12142         free(mask);
12143
12144         WRITE_ONCE(cpus, tmp_cpus);
12145         return tmp_cpus;
12146 }
12147
12148 static int populate_skeleton_maps(const struct bpf_object *obj,
12149                                   struct bpf_map_skeleton *maps,
12150                                   size_t map_cnt)
12151 {
12152         int i;
12153
12154         for (i = 0; i < map_cnt; i++) {
12155                 struct bpf_map **map = maps[i].map;
12156                 const char *name = maps[i].name;
12157                 void **mmaped = maps[i].mmaped;
12158
12159                 *map = bpf_object__find_map_by_name(obj, name);
12160                 if (!*map) {
12161                         pr_warn("failed to find skeleton map '%s'\n", name);
12162                         return -ESRCH;
12163                 }
12164
12165                 /* externs shouldn't be pre-setup from user code */
12166                 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG)
12167                         *mmaped = (*map)->mmaped;
12168         }
12169         return 0;
12170 }
12171
12172 static int populate_skeleton_progs(const struct bpf_object *obj,
12173                                    struct bpf_prog_skeleton *progs,
12174                                    size_t prog_cnt)
12175 {
12176         int i;
12177
12178         for (i = 0; i < prog_cnt; i++) {
12179                 struct bpf_program **prog = progs[i].prog;
12180                 const char *name = progs[i].name;
12181
12182                 *prog = bpf_object__find_program_by_name(obj, name);
12183                 if (!*prog) {
12184                         pr_warn("failed to find skeleton program '%s'\n", name);
12185                         return -ESRCH;
12186                 }
12187         }
12188         return 0;
12189 }
12190
12191 int bpf_object__open_skeleton(struct bpf_object_skeleton *s,
12192                               const struct bpf_object_open_opts *opts)
12193 {
12194         DECLARE_LIBBPF_OPTS(bpf_object_open_opts, skel_opts,
12195                 .object_name = s->name,
12196         );
12197         struct bpf_object *obj;
12198         int err;
12199
12200         /* Attempt to preserve opts->object_name, unless overriden by user
12201          * explicitly. Overwriting object name for skeletons is discouraged,
12202          * as it breaks global data maps, because they contain object name
12203          * prefix as their own map name prefix. When skeleton is generated,
12204          * bpftool is making an assumption that this name will stay the same.
12205          */
12206         if (opts) {
12207                 memcpy(&skel_opts, opts, sizeof(*opts));
12208                 if (!opts->object_name)
12209                         skel_opts.object_name = s->name;
12210         }
12211
12212         obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts);
12213         err = libbpf_get_error(obj);
12214         if (err) {
12215                 pr_warn("failed to initialize skeleton BPF object '%s': %d\n",
12216                         s->name, err);
12217                 return libbpf_err(err);
12218         }
12219
12220         *s->obj = obj;
12221         err = populate_skeleton_maps(obj, s->maps, s->map_cnt);
12222         if (err) {
12223                 pr_warn("failed to populate skeleton maps for '%s': %d\n", s->name, err);
12224                 return libbpf_err(err);
12225         }
12226
12227         err = populate_skeleton_progs(obj, s->progs, s->prog_cnt);
12228         if (err) {
12229                 pr_warn("failed to populate skeleton progs for '%s': %d\n", s->name, err);
12230                 return libbpf_err(err);
12231         }
12232
12233         return 0;
12234 }
12235
12236 int bpf_object__open_subskeleton(struct bpf_object_subskeleton *s)
12237 {
12238         int err, len, var_idx, i;
12239         const char *var_name;
12240         const struct bpf_map *map;
12241         struct btf *btf;
12242         __u32 map_type_id;
12243         const struct btf_type *map_type, *var_type;
12244         const struct bpf_var_skeleton *var_skel;
12245         struct btf_var_secinfo *var;
12246
12247         if (!s->obj)
12248                 return libbpf_err(-EINVAL);
12249
12250         btf = bpf_object__btf(s->obj);
12251         if (!btf) {
12252                 pr_warn("subskeletons require BTF at runtime (object %s)\n",
12253                         bpf_object__name(s->obj));
12254                 return libbpf_err(-errno);
12255         }
12256
12257         err = populate_skeleton_maps(s->obj, s->maps, s->map_cnt);
12258         if (err) {
12259                 pr_warn("failed to populate subskeleton maps: %d\n", err);
12260                 return libbpf_err(err);
12261         }
12262
12263         err = populate_skeleton_progs(s->obj, s->progs, s->prog_cnt);
12264         if (err) {
12265                 pr_warn("failed to populate subskeleton maps: %d\n", err);
12266                 return libbpf_err(err);
12267         }
12268
12269         for (var_idx = 0; var_idx < s->var_cnt; var_idx++) {
12270                 var_skel = &s->vars[var_idx];
12271                 map = *var_skel->map;
12272                 map_type_id = bpf_map__btf_value_type_id(map);
12273                 map_type = btf__type_by_id(btf, map_type_id);
12274
12275                 if (!btf_is_datasec(map_type)) {
12276                         pr_warn("type for map '%1$s' is not a datasec: %2$s",
12277                                 bpf_map__name(map),
12278                                 __btf_kind_str(btf_kind(map_type)));
12279                         return libbpf_err(-EINVAL);
12280                 }
12281
12282                 len = btf_vlen(map_type);
12283                 var = btf_var_secinfos(map_type);
12284                 for (i = 0; i < len; i++, var++) {
12285                         var_type = btf__type_by_id(btf, var->type);
12286                         var_name = btf__name_by_offset(btf, var_type->name_off);
12287                         if (strcmp(var_name, var_skel->name) == 0) {
12288                                 *var_skel->addr = map->mmaped + var->offset;
12289                                 break;
12290                         }
12291                 }
12292         }
12293         return 0;
12294 }
12295
12296 void bpf_object__destroy_subskeleton(struct bpf_object_subskeleton *s)
12297 {
12298         if (!s)
12299                 return;
12300         free(s->maps);
12301         free(s->progs);
12302         free(s->vars);
12303         free(s);
12304 }
12305
12306 int bpf_object__load_skeleton(struct bpf_object_skeleton *s)
12307 {
12308         int i, err;
12309
12310         err = bpf_object__load(*s->obj);
12311         if (err) {
12312                 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err);
12313                 return libbpf_err(err);
12314         }
12315
12316         for (i = 0; i < s->map_cnt; i++) {
12317                 struct bpf_map *map = *s->maps[i].map;
12318                 size_t mmap_sz = bpf_map_mmap_sz(map);
12319                 int prot, map_fd = bpf_map__fd(map);
12320                 void **mmaped = s->maps[i].mmaped;
12321
12322                 if (!mmaped)
12323                         continue;
12324
12325                 if (!(map->def.map_flags & BPF_F_MMAPABLE)) {
12326                         *mmaped = NULL;
12327                         continue;
12328                 }
12329
12330                 if (map->def.map_flags & BPF_F_RDONLY_PROG)
12331                         prot = PROT_READ;
12332                 else
12333                         prot = PROT_READ | PROT_WRITE;
12334
12335                 /* Remap anonymous mmap()-ed "map initialization image" as
12336                  * a BPF map-backed mmap()-ed memory, but preserving the same
12337                  * memory address. This will cause kernel to change process'
12338                  * page table to point to a different piece of kernel memory,
12339                  * but from userspace point of view memory address (and its
12340                  * contents, being identical at this point) will stay the
12341                  * same. This mapping will be released by bpf_object__close()
12342                  * as per normal clean up procedure, so we don't need to worry
12343                  * about it from skeleton's clean up perspective.
12344                  */
12345                 *mmaped = mmap(map->mmaped, mmap_sz, prot,
12346                                 MAP_SHARED | MAP_FIXED, map_fd, 0);
12347                 if (*mmaped == MAP_FAILED) {
12348                         err = -errno;
12349                         *mmaped = NULL;
12350                         pr_warn("failed to re-mmap() map '%s': %d\n",
12351                                  bpf_map__name(map), err);
12352                         return libbpf_err(err);
12353                 }
12354         }
12355
12356         return 0;
12357 }
12358
12359 int bpf_object__attach_skeleton(struct bpf_object_skeleton *s)
12360 {
12361         int i, err;
12362
12363         for (i = 0; i < s->prog_cnt; i++) {
12364                 struct bpf_program *prog = *s->progs[i].prog;
12365                 struct bpf_link **link = s->progs[i].link;
12366
12367                 if (!prog->autoload || !prog->autoattach)
12368                         continue;
12369
12370                 /* auto-attaching not supported for this program */
12371                 if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
12372                         continue;
12373
12374                 /* if user already set the link manually, don't attempt auto-attach */
12375                 if (*link)
12376                         continue;
12377
12378                 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link);
12379                 if (err) {
12380                         pr_warn("prog '%s': failed to auto-attach: %d\n",
12381                                 bpf_program__name(prog), err);
12382                         return libbpf_err(err);
12383                 }
12384
12385                 /* It's possible that for some SEC() definitions auto-attach
12386                  * is supported in some cases (e.g., if definition completely
12387                  * specifies target information), but is not in other cases.
12388                  * SEC("uprobe") is one such case. If user specified target
12389                  * binary and function name, such BPF program can be
12390                  * auto-attached. But if not, it shouldn't trigger skeleton's
12391                  * attach to fail. It should just be skipped.
12392                  * attach_fn signals such case with returning 0 (no error) and
12393                  * setting link to NULL.
12394                  */
12395         }
12396
12397         return 0;
12398 }
12399
12400 void bpf_object__detach_skeleton(struct bpf_object_skeleton *s)
12401 {
12402         int i;
12403
12404         for (i = 0; i < s->prog_cnt; i++) {
12405                 struct bpf_link **link = s->progs[i].link;
12406
12407                 bpf_link__destroy(*link);
12408                 *link = NULL;
12409         }
12410 }
12411
12412 void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s)
12413 {
12414         if (!s)
12415                 return;
12416
12417         if (s->progs)
12418                 bpf_object__detach_skeleton(s);
12419         if (s->obj)
12420                 bpf_object__close(*s->obj);
12421         free(s->maps);
12422         free(s->progs);
12423         free(s);
12424 }