tools/lib/bpf/libbpf.c

   1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
   2
   3 /*
   4  * Common eBPF ELF object loading operations.
   5  *
   6  * Copyright (C) 2013-2015 Alexei Starovoitov <[email protected]>
   7  * Copyright (C) 2015 Wang Nan <[email protected]>
   8  * Copyright (C) 2015 Huawei Inc.
   9  * Copyright (C) 2017 Nicira, Inc.
  10  * Copyright (C) 2019 Isovalent, Inc.
  11  */
  12
  13 #ifndef _GNU_SOURCE
  14 #define _GNU_SOURCE
  15 #endif
  16 #include <stdlib.h>
  17 #include <stdio.h>
  18 #include <stdarg.h>
  19 #include <libgen.h>
  20 #include <inttypes.h>
  21 #include <limits.h>
  22 #include <string.h>
  23 #include <unistd.h>
  24 #include <endian.h>
  25 #include <fcntl.h>
  26 #include <errno.h>
  27 #include <ctype.h>
  28 #include <asm/unistd.h>
  29 #include <linux/err.h>
  30 #include <linux/kernel.h>
  31 #include <linux/bpf.h>
  32 #include <linux/btf.h>
  33 #include <linux/filter.h>
  34 #include <linux/limits.h>
  35 #include <linux/perf_event.h>
  36 #include <linux/ring_buffer.h>
  37 #include <linux/version.h>
  38 #include <sys/epoll.h>
  39 #include <sys/ioctl.h>
  40 #include <sys/mman.h>
  41 #include <sys/stat.h>
  42 #include <sys/types.h>
  43 #include <sys/vfs.h>
  44 #include <sys/utsname.h>
  45 #include <sys/resource.h>
  46 #include <libelf.h>
  47 #include <gelf.h>
  48 #include <zlib.h>
  49
  50 #include "libbpf.h"
  51 #include "bpf.h"
  52 #include "btf.h"
  53 #include "str_error.h"
  54 #include "libbpf_internal.h"
  55 #include "hashmap.h"
  56 #include "bpf_gen_internal.h"
  57
  58 #ifndef BPF_FS_MAGIC
  59 #define BPF_FS_MAGIC            0xcafe4a11
  60 #endif
  61
  62 #define BPF_INSN_SZ (sizeof(struct bpf_insn))
  63
  64 /* vsprintf() in __base_pr() uses nonliteral format string. It may break
  65  * compilation if user enables corresponding warning. Disable it explicitly.
  66  */
  67 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
  68
  69 #define __printf(a, b)  __attribute__((format(printf, a, b)))
  70
  71 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj);
  72 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog);
  73
  74 static const char * const attach_type_name[] = {
  75         [BPF_CGROUP_INET_INGRESS]       = "cgroup_inet_ingress",
  76         [BPF_CGROUP_INET_EGRESS]        = "cgroup_inet_egress",
  77         [BPF_CGROUP_INET_SOCK_CREATE]   = "cgroup_inet_sock_create",
  78         [BPF_CGROUP_INET_SOCK_RELEASE]  = "cgroup_inet_sock_release",
  79         [BPF_CGROUP_SOCK_OPS]           = "cgroup_sock_ops",
  80         [BPF_CGROUP_DEVICE]             = "cgroup_device",
  81         [BPF_CGROUP_INET4_BIND]         = "cgroup_inet4_bind",
  82         [BPF_CGROUP_INET6_BIND]         = "cgroup_inet6_bind",
  83         [BPF_CGROUP_INET4_CONNECT]      = "cgroup_inet4_connect",
  84         [BPF_CGROUP_INET6_CONNECT]      = "cgroup_inet6_connect",
  85         [BPF_CGROUP_INET4_POST_BIND]    = "cgroup_inet4_post_bind",
  86         [BPF_CGROUP_INET6_POST_BIND]    = "cgroup_inet6_post_bind",
  87         [BPF_CGROUP_INET4_GETPEERNAME]  = "cgroup_inet4_getpeername",
  88         [BPF_CGROUP_INET6_GETPEERNAME]  = "cgroup_inet6_getpeername",
  89         [BPF_CGROUP_INET4_GETSOCKNAME]  = "cgroup_inet4_getsockname",
  90         [BPF_CGROUP_INET6_GETSOCKNAME]  = "cgroup_inet6_getsockname",
  91         [BPF_CGROUP_UDP4_SENDMSG]       = "cgroup_udp4_sendmsg",
  92         [BPF_CGROUP_UDP6_SENDMSG]       = "cgroup_udp6_sendmsg",
  93         [BPF_CGROUP_SYSCTL]             = "cgroup_sysctl",
  94         [BPF_CGROUP_UDP4_RECVMSG]       = "cgroup_udp4_recvmsg",
  95         [BPF_CGROUP_UDP6_RECVMSG]       = "cgroup_udp6_recvmsg",
  96         [BPF_CGROUP_GETSOCKOPT]         = "cgroup_getsockopt",
  97         [BPF_CGROUP_SETSOCKOPT]         = "cgroup_setsockopt",
  98         [BPF_SK_SKB_STREAM_PARSER]      = "sk_skb_stream_parser",
  99         [BPF_SK_SKB_STREAM_VERDICT]     = "sk_skb_stream_verdict",
 100         [BPF_SK_SKB_VERDICT]            = "sk_skb_verdict",
 101         [BPF_SK_MSG_VERDICT]            = "sk_msg_verdict",
 102         [BPF_LIRC_MODE2]                = "lirc_mode2",
 103         [BPF_FLOW_DISSECTOR]            = "flow_dissector",
 104         [BPF_TRACE_RAW_TP]              = "trace_raw_tp",
 105         [BPF_TRACE_FENTRY]              = "trace_fentry",
 106         [BPF_TRACE_FEXIT]               = "trace_fexit",
 107         [BPF_MODIFY_RETURN]             = "modify_return",
 108         [BPF_LSM_MAC]                   = "lsm_mac",
 109         [BPF_LSM_CGROUP]                = "lsm_cgroup",
 110         [BPF_SK_LOOKUP]                 = "sk_lookup",
 111         [BPF_TRACE_ITER]                = "trace_iter",
 112         [BPF_XDP_DEVMAP]                = "xdp_devmap",
 113         [BPF_XDP_CPUMAP]                = "xdp_cpumap",
 114         [BPF_XDP]                       = "xdp",
 115         [BPF_SK_REUSEPORT_SELECT]       = "sk_reuseport_select",
 116         [BPF_SK_REUSEPORT_SELECT_OR_MIGRATE]    = "sk_reuseport_select_or_migrate",
 117         [BPF_PERF_EVENT]                = "perf_event",
 118         [BPF_TRACE_KPROBE_MULTI]        = "trace_kprobe_multi",
 119 };
 120
 121 static const char * const link_type_name[] = {
 122         [BPF_LINK_TYPE_UNSPEC]                  = "unspec",
 123         [BPF_LINK_TYPE_RAW_TRACEPOINT]          = "raw_tracepoint",
 124         [BPF_LINK_TYPE_TRACING]                 = "tracing",
 125         [BPF_LINK_TYPE_CGROUP]                  = "cgroup",
 126         [BPF_LINK_TYPE_ITER]                    = "iter",
 127         [BPF_LINK_TYPE_NETNS]                   = "netns",
 128         [BPF_LINK_TYPE_XDP]                     = "xdp",
 129         [BPF_LINK_TYPE_PERF_EVENT]              = "perf_event",
 130         [BPF_LINK_TYPE_KPROBE_MULTI]            = "kprobe_multi",
 131         [BPF_LINK_TYPE_STRUCT_OPS]              = "struct_ops",
 132 };
 133
 134 static const char * const map_type_name[] = {
 135         [BPF_MAP_TYPE_UNSPEC]                   = "unspec",
 136         [BPF_MAP_TYPE_HASH]                     = "hash",
 137         [BPF_MAP_TYPE_ARRAY]                    = "array",
 138         [BPF_MAP_TYPE_PROG_ARRAY]               = "prog_array",
 139         [BPF_MAP_TYPE_PERF_EVENT_ARRAY]         = "perf_event_array",
 140         [BPF_MAP_TYPE_PERCPU_HASH]              = "percpu_hash",
 141         [BPF_MAP_TYPE_PERCPU_ARRAY]             = "percpu_array",
 142         [BPF_MAP_TYPE_STACK_TRACE]              = "stack_trace",
 143         [BPF_MAP_TYPE_CGROUP_ARRAY]             = "cgroup_array",
 144         [BPF_MAP_TYPE_LRU_HASH]                 = "lru_hash",
 145         [BPF_MAP_TYPE_LRU_PERCPU_HASH]          = "lru_percpu_hash",
 146         [BPF_MAP_TYPE_LPM_TRIE]                 = "lpm_trie",
 147         [BPF_MAP_TYPE_ARRAY_OF_MAPS]            = "array_of_maps",
 148         [BPF_MAP_TYPE_HASH_OF_MAPS]             = "hash_of_maps",
 149         [BPF_MAP_TYPE_DEVMAP]                   = "devmap",
 150         [BPF_MAP_TYPE_DEVMAP_HASH]              = "devmap_hash",
 151         [BPF_MAP_TYPE_SOCKMAP]                  = "sockmap",
 152         [BPF_MAP_TYPE_CPUMAP]                   = "cpumap",
 153         [BPF_MAP_TYPE_XSKMAP]                   = "xskmap",
 154         [BPF_MAP_TYPE_SOCKHASH]                 = "sockhash",
 155         [BPF_MAP_TYPE_CGROUP_STORAGE]           = "cgroup_storage",
 156         [BPF_MAP_TYPE_REUSEPORT_SOCKARRAY]      = "reuseport_sockarray",
 157         [BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE]    = "percpu_cgroup_storage",
 158         [BPF_MAP_TYPE_QUEUE]                    = "queue",
 159         [BPF_MAP_TYPE_STACK]                    = "stack",
 160         [BPF_MAP_TYPE_SK_STORAGE]               = "sk_storage",
 161         [BPF_MAP_TYPE_STRUCT_OPS]               = "struct_ops",
 162         [BPF_MAP_TYPE_RINGBUF]                  = "ringbuf",
 163         [BPF_MAP_TYPE_INODE_STORAGE]            = "inode_storage",
 164         [BPF_MAP_TYPE_TASK_STORAGE]             = "task_storage",
 165         [BPF_MAP_TYPE_BLOOM_FILTER]             = "bloom_filter",
 166         [BPF_MAP_TYPE_USER_RINGBUF]             = "user_ringbuf",
 167         [BPF_MAP_TYPE_CGRP_STORAGE]             = "cgrp_storage",
 168 };
 169
 170 static const char * const prog_type_name[] = {
 171         [BPF_PROG_TYPE_UNSPEC]                  = "unspec",
 172         [BPF_PROG_TYPE_SOCKET_FILTER]           = "socket_filter",
 173         [BPF_PROG_TYPE_KPROBE]                  = "kprobe",
 174         [BPF_PROG_TYPE_SCHED_CLS]               = "sched_cls",
 175         [BPF_PROG_TYPE_SCHED_ACT]               = "sched_act",
 176         [BPF_PROG_TYPE_TRACEPOINT]              = "tracepoint",
 177         [BPF_PROG_TYPE_XDP]                     = "xdp",
 178         [BPF_PROG_TYPE_PERF_EVENT]              = "perf_event",
 179         [BPF_PROG_TYPE_CGROUP_SKB]              = "cgroup_skb",
 180         [BPF_PROG_TYPE_CGROUP_SOCK]             = "cgroup_sock",
 181         [BPF_PROG_TYPE_LWT_IN]                  = "lwt_in",
 182         [BPF_PROG_TYPE_LWT_OUT]                 = "lwt_out",
 183         [BPF_PROG_TYPE_LWT_XMIT]                = "lwt_xmit",
 184         [BPF_PROG_TYPE_SOCK_OPS]                = "sock_ops",
 185         [BPF_PROG_TYPE_SK_SKB]                  = "sk_skb",
 186         [BPF_PROG_TYPE_CGROUP_DEVICE]           = "cgroup_device",
 187         [BPF_PROG_TYPE_SK_MSG]                  = "sk_msg",
 188         [BPF_PROG_TYPE_RAW_TRACEPOINT]          = "raw_tracepoint",
 189         [BPF_PROG_TYPE_CGROUP_SOCK_ADDR]        = "cgroup_sock_addr",
 190         [BPF_PROG_TYPE_LWT_SEG6LOCAL]           = "lwt_seg6local",
 191         [BPF_PROG_TYPE_LIRC_MODE2]              = "lirc_mode2",
 192         [BPF_PROG_TYPE_SK_REUSEPORT]            = "sk_reuseport",
 193         [BPF_PROG_TYPE_FLOW_DISSECTOR]          = "flow_dissector",
 194         [BPF_PROG_TYPE_CGROUP_SYSCTL]           = "cgroup_sysctl",
 195         [BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE] = "raw_tracepoint_writable",
 196         [BPF_PROG_TYPE_CGROUP_SOCKOPT]          = "cgroup_sockopt",
 197         [BPF_PROG_TYPE_TRACING]                 = "tracing",
 198         [BPF_PROG_TYPE_STRUCT_OPS]              = "struct_ops",
 199         [BPF_PROG_TYPE_EXT]                     = "ext",
 200         [BPF_PROG_TYPE_LSM]                     = "lsm",
 201         [BPF_PROG_TYPE_SK_LOOKUP]               = "sk_lookup",
 202         [BPF_PROG_TYPE_SYSCALL]                 = "syscall",
 203 };
 204
 205 static int __base_pr(enum libbpf_print_level level, const char *format,
 206                      va_list args)
 207 {
 208         if (level == LIBBPF_DEBUG)
 209                 return 0;
 210
 211         return vfprintf(stderr, format, args);
 212 }
 213
 214 static libbpf_print_fn_t __libbpf_pr = __base_pr;
 215
 216 libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn)
 217 {
 218         libbpf_print_fn_t old_print_fn = __libbpf_pr;
 219
 220         __libbpf_pr = fn;
 221         return old_print_fn;
 222 }
 223
 224 __printf(2, 3)
 225 void libbpf_print(enum libbpf_print_level level, const char *format, ...)
 226 {
 227         va_list args;
 228         int old_errno;
 229
 230         if (!__libbpf_pr)
 231                 return;
 232
 233         old_errno = errno;
 234
 235         va_start(args, format);
 236         __libbpf_pr(level, format, args);
 237         va_end(args);
 238
 239         errno = old_errno;
 240 }
 241
 242 static void pr_perm_msg(int err)
 243 {
 244         struct rlimit limit;
 245         char buf[100];
 246
 247         if (err != -EPERM || geteuid() != 0)
 248                 return;
 249
 250         err = getrlimit(RLIMIT_MEMLOCK, &limit);
 251         if (err)
 252                 return;
 253
 254         if (limit.rlim_cur == RLIM_INFINITY)
 255                 return;
 256
 257         if (limit.rlim_cur < 1024)
 258                 snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur);
 259         else if (limit.rlim_cur < 1024*1024)
 260                 snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024);
 261         else
 262                 snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024));
 263
 264         pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n",
 265                 buf);
 266 }
 267
 268 #define STRERR_BUFSIZE  128
 269
 270 /* Copied from tools/perf/util/util.h */
 271 #ifndef zfree
 272 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
 273 #endif
 274
 275 #ifndef zclose
 276 # define zclose(fd) ({                  \
 277         int ___err = 0;                 \
 278         if ((fd) >= 0)                  \
 279                 ___err = close((fd));   \
 280         fd = -1;                        \
 281         ___err; })
 282 #endif
 283
 284 static inline __u64 ptr_to_u64(const void *ptr)
 285 {
 286         return (__u64) (unsigned long) ptr;
 287 }
 288
 289 int libbpf_set_strict_mode(enum libbpf_strict_mode mode)
 290 {
 291         /* as of v1.0 libbpf_set_strict_mode() is a no-op */
 292         return 0;
 293 }
 294
 295 __u32 libbpf_major_version(void)
 296 {
 297         return LIBBPF_MAJOR_VERSION;
 298 }
 299
 300 __u32 libbpf_minor_version(void)
 301 {
 302         return LIBBPF_MINOR_VERSION;
 303 }
 304
 305 const char *libbpf_version_string(void)
 306 {
 307 #define __S(X) #X
 308 #define _S(X) __S(X)
 309         return  "v" _S(LIBBPF_MAJOR_VERSION) "." _S(LIBBPF_MINOR_VERSION);
 310 #undef _S
 311 #undef __S
 312 }
 313
 314 enum reloc_type {
 315         RELO_LD64,
 316         RELO_CALL,
 317         RELO_DATA,
 318         RELO_EXTERN_VAR,
 319         RELO_EXTERN_FUNC,
 320         RELO_SUBPROG_ADDR,
 321         RELO_CORE,
 322 };
 323
 324 struct reloc_desc {
 325         enum reloc_type type;
 326         int insn_idx;
 327         union {
 328                 const struct bpf_core_relo *core_relo; /* used when type == RELO_CORE */
 329                 struct {
 330                         int map_idx;
 331                         int sym_off;
 332                 };
 333         };
 334 };
 335
 336 /* stored as sec_def->cookie for all libbpf-supported SEC()s */
 337 enum sec_def_flags {
 338         SEC_NONE = 0,
 339         /* expected_attach_type is optional, if kernel doesn't support that */
 340         SEC_EXP_ATTACH_OPT = 1,
 341         /* legacy, only used by libbpf_get_type_names() and
 342          * libbpf_attach_type_by_name(), not used by libbpf itself at all.
 343          * This used to be associated with cgroup (and few other) BPF programs
 344          * that were attachable through BPF_PROG_ATTACH command. Pretty
 345          * meaningless nowadays, though.
 346          */
 347         SEC_ATTACHABLE = 2,
 348         SEC_ATTACHABLE_OPT = SEC_ATTACHABLE | SEC_EXP_ATTACH_OPT,
 349         /* attachment target is specified through BTF ID in either kernel or
 350          * other BPF program's BTF object
 351          */
 352         SEC_ATTACH_BTF = 4,
 353         /* BPF program type allows sleeping/blocking in kernel */
 354         SEC_SLEEPABLE = 8,
 355         /* BPF program support non-linear XDP buffer */
 356         SEC_XDP_FRAGS = 16,
 357 };
 358
 359 struct bpf_sec_def {
 360         char *sec;
 361         enum bpf_prog_type prog_type;
 362         enum bpf_attach_type expected_attach_type;
 363         long cookie;
 364         int handler_id;
 365
 366         libbpf_prog_setup_fn_t prog_setup_fn;
 367         libbpf_prog_prepare_load_fn_t prog_prepare_load_fn;
 368         libbpf_prog_attach_fn_t prog_attach_fn;
 369 };
 370
 371 /*
 372  * bpf_prog should be a better name but it has been used in
 373  * linux/filter.h.
 374  */
 375 struct bpf_program {
 376         char *name;
 377         char *sec_name;
 378         size_t sec_idx;
 379         const struct bpf_sec_def *sec_def;
 380         /* this program's instruction offset (in number of instructions)
 381          * within its containing ELF section
 382          */
 383         size_t sec_insn_off;
 384         /* number of original instructions in ELF section belonging to this
 385          * program, not taking into account subprogram instructions possible
 386          * appended later during relocation
 387          */
 388         size_t sec_insn_cnt;
 389         /* Offset (in number of instructions) of the start of instruction
 390          * belonging to this BPF program  within its containing main BPF
 391          * program. For the entry-point (main) BPF program, this is always
 392          * zero. For a sub-program, this gets reset before each of main BPF
 393          * programs are processed and relocated and is used to determined
 394          * whether sub-program was already appended to the main program, and
 395          * if yes, at which instruction offset.
 396          */
 397         size_t sub_insn_off;
 398
 399         /* instructions that belong to BPF program; insns[0] is located at
 400          * sec_insn_off instruction within its ELF section in ELF file, so
 401          * when mapping ELF file instruction index to the local instruction,
 402          * one needs to subtract sec_insn_off; and vice versa.
 403          */
 404         struct bpf_insn *insns;
 405         /* actual number of instruction in this BPF program's image; for
 406          * entry-point BPF programs this includes the size of main program
 407          * itself plus all the used sub-programs, appended at the end
 408          */
 409         size_t insns_cnt;
 410
 411         struct reloc_desc *reloc_desc;
 412         int nr_reloc;
 413
 414         /* BPF verifier log settings */
 415         char *log_buf;
 416         size_t log_size;
 417         __u32 log_level;
 418
 419         struct bpf_object *obj;
 420
 421         int fd;
 422         bool autoload;
 423         bool autoattach;
 424         bool mark_btf_static;
 425         enum bpf_prog_type type;
 426         enum bpf_attach_type expected_attach_type;
 427
 428         int prog_ifindex;
 429         __u32 attach_btf_obj_fd;
 430         __u32 attach_btf_id;
 431         __u32 attach_prog_fd;
 432
 433         void *func_info;
 434         __u32 func_info_rec_size;
 435         __u32 func_info_cnt;
 436
 437         void *line_info;
 438         __u32 line_info_rec_size;
 439         __u32 line_info_cnt;
 440         __u32 prog_flags;
 441 };
 442
 443 struct bpf_struct_ops {
 444         const char *tname;
 445         const struct btf_type *type;
 446         struct bpf_program **progs;
 447         __u32 *kern_func_off;
 448         /* e.g. struct tcp_congestion_ops in bpf_prog's btf format */
 449         void *data;
 450         /* e.g. struct bpf_struct_ops_tcp_congestion_ops in
 451          *      btf_vmlinux's format.
 452          * struct bpf_struct_ops_tcp_congestion_ops {
 453          *      [... some other kernel fields ...]
 454          *      struct tcp_congestion_ops data;
 455          * }
 456          * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)
 457          * bpf_map__init_kern_struct_ops() will populate the "kern_vdata"
 458          * from "data".
 459          */
 460         void *kern_vdata;
 461         __u32 type_id;
 462 };
 463
 464 #define DATA_SEC ".data"
 465 #define BSS_SEC ".bss"
 466 #define RODATA_SEC ".rodata"
 467 #define KCONFIG_SEC ".kconfig"
 468 #define KSYMS_SEC ".ksyms"
 469 #define STRUCT_OPS_SEC ".struct_ops"
 470
 471 enum libbpf_map_type {
 472         LIBBPF_MAP_UNSPEC,
 473         LIBBPF_MAP_DATA,
 474         LIBBPF_MAP_BSS,
 475         LIBBPF_MAP_RODATA,
 476         LIBBPF_MAP_KCONFIG,
 477 };
 478
 479 struct bpf_map_def {
 480         unsigned int type;
 481         unsigned int key_size;
 482         unsigned int value_size;
 483         unsigned int max_entries;
 484         unsigned int map_flags;
 485 };
 486
 487 struct bpf_map {
 488         struct bpf_object *obj;
 489         char *name;
 490         /* real_name is defined for special internal maps (.rodata*,
 491          * .data*, .bss, .kconfig) and preserves their original ELF section
 492          * name. This is important to be able to find corresponding BTF
 493          * DATASEC information.
 494          */
 495         char *real_name;
 496         int fd;
 497         int sec_idx;
 498         size_t sec_offset;
 499         int map_ifindex;
 500         int inner_map_fd;
 501         struct bpf_map_def def;
 502         __u32 numa_node;
 503         __u32 btf_var_idx;
 504         __u32 btf_key_type_id;
 505         __u32 btf_value_type_id;
 506         __u32 btf_vmlinux_value_type_id;
 507         enum libbpf_map_type libbpf_type;
 508         void *mmaped;
 509         struct bpf_struct_ops *st_ops;
 510         struct bpf_map *inner_map;
 511         void **init_slots;
 512         int init_slots_sz;
 513         char *pin_path;
 514         bool pinned;
 515         bool reused;
 516         bool autocreate;
 517         __u64 map_extra;
 518 };
 519
 520 enum extern_type {
 521         EXT_UNKNOWN,
 522         EXT_KCFG,
 523         EXT_KSYM,
 524 };
 525
 526 enum kcfg_type {
 527         KCFG_UNKNOWN,
 528         KCFG_CHAR,
 529         KCFG_BOOL,
 530         KCFG_INT,
 531         KCFG_TRISTATE,
 532         KCFG_CHAR_ARR,
 533 };
 534
 535 struct extern_desc {
 536         enum extern_type type;
 537         int sym_idx;
 538         int btf_id;
 539         int sec_btf_id;
 540         const char *name;
 541         bool is_set;
 542         bool is_weak;
 543         union {
 544                 struct {
 545                         enum kcfg_type type;
 546                         int sz;
 547                         int align;
 548                         int data_off;
 549                         bool is_signed;
 550                 } kcfg;
 551                 struct {
 552                         unsigned long long addr;
 553
 554                         /* target btf_id of the corresponding kernel var. */
 555                         int kernel_btf_obj_fd;
 556                         int kernel_btf_id;
 557
 558                         /* local btf_id of the ksym extern's type. */
 559                         __u32 type_id;
 560                         /* BTF fd index to be patched in for insn->off, this is
 561                          * 0 for vmlinux BTF, index in obj->fd_array for module
 562                          * BTF
 563                          */
 564                         __s16 btf_fd_idx;
 565                 } ksym;
 566         };
 567 };
 568
 569 struct module_btf {
 570         struct btf *btf;
 571         char *name;
 572         __u32 id;
 573         int fd;
 574         int fd_array_idx;
 575 };
 576
 577 enum sec_type {
 578         SEC_UNUSED = 0,
 579         SEC_RELO,
 580         SEC_BSS,
 581         SEC_DATA,
 582         SEC_RODATA,
 583 };
 584
 585 struct elf_sec_desc {
 586         enum sec_type sec_type;
 587         Elf64_Shdr *shdr;
 588         Elf_Data *data;
 589 };
 590
 591 struct elf_state {
 592         int fd;
 593         const void *obj_buf;
 594         size_t obj_buf_sz;
 595         Elf *elf;
 596         Elf64_Ehdr *ehdr;
 597         Elf_Data *symbols;
 598         Elf_Data *st_ops_data;
 599         size_t shstrndx; /* section index for section name strings */
 600         size_t strtabidx;
 601         struct elf_sec_desc *secs;
 602         size_t sec_cnt;
 603         int btf_maps_shndx;
 604         __u32 btf_maps_sec_btf_id;
 605         int text_shndx;
 606         int symbols_shndx;
 607         int st_ops_shndx;
 608 };
 609
 610 struct usdt_manager;
 611
 612 struct bpf_object {
 613         char name[BPF_OBJ_NAME_LEN];
 614         char license[64];
 615         __u32 kern_version;
 616
 617         struct bpf_program *programs;
 618         size_t nr_programs;
 619         struct bpf_map *maps;
 620         size_t nr_maps;
 621         size_t maps_cap;
 622
 623         char *kconfig;
 624         struct extern_desc *externs;
 625         int nr_extern;
 626         int kconfig_map_idx;
 627
 628         bool loaded;
 629         bool has_subcalls;
 630         bool has_rodata;
 631
 632         struct bpf_gen *gen_loader;
 633
 634         /* Information when doing ELF related work. Only valid if efile.elf is not NULL */
 635         struct elf_state efile;
 636
 637         struct btf *btf;
 638         struct btf_ext *btf_ext;
 639
 640         /* Parse and load BTF vmlinux if any of the programs in the object need
 641          * it at load time.
 642          */
 643         struct btf *btf_vmlinux;
 644         /* Path to the custom BTF to be used for BPF CO-RE relocations as an
 645          * override for vmlinux BTF.
 646          */
 647         char *btf_custom_path;
 648         /* vmlinux BTF override for CO-RE relocations */
 649         struct btf *btf_vmlinux_override;
 650         /* Lazily initialized kernel module BTFs */
 651         struct module_btf *btf_modules;
 652         bool btf_modules_loaded;
 653         size_t btf_module_cnt;
 654         size_t btf_module_cap;
 655
 656         /* optional log settings passed to BPF_BTF_LOAD and BPF_PROG_LOAD commands */
 657         char *log_buf;
 658         size_t log_size;
 659         __u32 log_level;
 660
 661         int *fd_array;
 662         size_t fd_array_cap;
 663         size_t fd_array_cnt;
 664
 665         struct usdt_manager *usdt_man;
 666
 667         char path[];
 668 };
 669
 670 static const char *elf_sym_str(const struct bpf_object *obj, size_t off);
 671 static const char *elf_sec_str(const struct bpf_object *obj, size_t off);
 672 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx);
 673 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name);
 674 static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn);
 675 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn);
 676 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn);
 677 static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx);
 678 static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx);
 679
 680 void bpf_program__unload(struct bpf_program *prog)
 681 {
 682         if (!prog)
 683                 return;
 684
 685         zclose(prog->fd);
 686
 687         zfree(&prog->func_info);
 688         zfree(&prog->line_info);
 689 }
 690
 691 static void bpf_program__exit(struct bpf_program *prog)
 692 {
 693         if (!prog)
 694                 return;
 695
 696         bpf_program__unload(prog);
 697         zfree(&prog->name);
 698         zfree(&prog->sec_name);
 699         zfree(&prog->insns);
 700         zfree(&prog->reloc_desc);
 701
 702         prog->nr_reloc = 0;
 703         prog->insns_cnt = 0;
 704         prog->sec_idx = -1;
 705 }
 706
 707 static bool insn_is_subprog_call(const struct bpf_insn *insn)
 708 {
 709         return BPF_CLASS(insn->code) == BPF_JMP &&
 710                BPF_OP(insn->code) == BPF_CALL &&
 711                BPF_SRC(insn->code) == BPF_K &&
 712                insn->src_reg == BPF_PSEUDO_CALL &&
 713                insn->dst_reg == 0 &&
 714                insn->off == 0;
 715 }
 716
 717 static bool is_call_insn(const struct bpf_insn *insn)
 718 {
 719         return insn->code == (BPF_JMP | BPF_CALL);
 720 }
 721
 722 static bool insn_is_pseudo_func(struct bpf_insn *insn)
 723 {
 724         return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
 725 }
 726
 727 static int
 728 bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog,
 729                       const char *name, size_t sec_idx, const char *sec_name,
 730                       size_t sec_off, void *insn_data, size_t insn_data_sz)
 731 {
 732         if (insn_data_sz == 0 || insn_data_sz % BPF_INSN_SZ || sec_off % BPF_INSN_SZ) {
 733                 pr_warn("sec '%s': corrupted program '%s', offset %zu, size %zu\n",
 734                         sec_name, name, sec_off, insn_data_sz);
 735                 return -EINVAL;
 736         }
 737
 738         memset(prog, 0, sizeof(*prog));
 739         prog->obj = obj;
 740
 741         prog->sec_idx = sec_idx;
 742         prog->sec_insn_off = sec_off / BPF_INSN_SZ;
 743         prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ;
 744         /* insns_cnt can later be increased by appending used subprograms */
 745         prog->insns_cnt = prog->sec_insn_cnt;
 746
 747         prog->type = BPF_PROG_TYPE_UNSPEC;
 748         prog->fd = -1;
 749
 750         /* libbpf's convention for SEC("?abc...") is that it's just like
 751          * SEC("abc...") but the corresponding bpf_program starts out with
 752          * autoload set to false.
 753          */
 754         if (sec_name[0] == '?') {
 755                 prog->autoload = false;
 756                 /* from now on forget there was ? in section name */
 757                 sec_name++;
 758         } else {
 759                 prog->autoload = true;
 760         }
 761
 762         prog->autoattach = true;
 763
 764         /* inherit object's log_level */
 765         prog->log_level = obj->log_level;
 766
 767         prog->sec_name = strdup(sec_name);
 768         if (!prog->sec_name)
 769                 goto errout;
 770
 771         prog->name = strdup(name);
 772         if (!prog->name)
 773                 goto errout;
 774
 775         prog->insns = malloc(insn_data_sz);
 776         if (!prog->insns)
 777                 goto errout;
 778         memcpy(prog->insns, insn_data, insn_data_sz);
 779
 780         return 0;
 781 errout:
 782         pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name);
 783         bpf_program__exit(prog);
 784         return -ENOMEM;
 785 }
 786
 787 static int
 788 bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data,
 789                          const char *sec_name, int sec_idx)
 790 {
 791         Elf_Data *symbols = obj->efile.symbols;
 792         struct bpf_program *prog, *progs;
 793         void *data = sec_data->d_buf;
 794         size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms;
 795         int nr_progs, err, i;
 796         const char *name;
 797         Elf64_Sym *sym;
 798
 799         progs = obj->programs;
 800         nr_progs = obj->nr_programs;
 801         nr_syms = symbols->d_size / sizeof(Elf64_Sym);
 802         sec_off = 0;
 803
 804         for (i = 0; i < nr_syms; i++) {
 805                 sym = elf_sym_by_idx(obj, i);
 806
 807                 if (sym->st_shndx != sec_idx)
 808                         continue;
 809                 if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC)
 810                         continue;
 811
 812                 prog_sz = sym->st_size;
 813                 sec_off = sym->st_value;
 814
 815                 name = elf_sym_str(obj, sym->st_name);
 816                 if (!name) {
 817                         pr_warn("sec '%s': failed to get symbol name for offset %zu\n",
 818                                 sec_name, sec_off);
 819                         return -LIBBPF_ERRNO__FORMAT;
 820                 }
 821
 822                 if (sec_off + prog_sz > sec_sz) {
 823                         pr_warn("sec '%s': program at offset %zu crosses section boundary\n",
 824                                 sec_name, sec_off);
 825                         return -LIBBPF_ERRNO__FORMAT;
 826                 }
 827
 828                 if (sec_idx != obj->efile.text_shndx && ELF64_ST_BIND(sym->st_info) == STB_LOCAL) {
 829                         pr_warn("sec '%s': program '%s' is static and not supported\n", sec_name, name);
 830                         return -ENOTSUP;
 831                 }
 832
 833                 pr_debug("sec '%s': found program '%s' at insn offset %zu (%zu bytes), code size %zu insns (%zu bytes)\n",
 834                          sec_name, name, sec_off / BPF_INSN_SZ, sec_off, prog_sz / BPF_INSN_SZ, prog_sz);
 835
 836                 progs = libbpf_reallocarray(progs, nr_progs + 1, sizeof(*progs));
 837                 if (!progs) {
 838                         /*
 839                          * In this case the original obj->programs
 840                          * is still valid, so don't need special treat for
 841                          * bpf_close_object().
 842                          */
 843                         pr_warn("sec '%s': failed to alloc memory for new program '%s'\n",
 844                                 sec_name, name);
 845                         return -ENOMEM;
 846                 }
 847                 obj->programs = progs;
 848
 849                 prog = &progs[nr_progs];
 850
 851                 err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name,
 852                                             sec_off, data + sec_off, prog_sz);
 853                 if (err)
 854                         return err;
 855
 856                 /* if function is a global/weak symbol, but has restricted
 857                  * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF FUNC
 858                  * as static to enable more permissive BPF verification mode
 859                  * with more outside context available to BPF verifier
 860                  */
 861                 if (ELF64_ST_BIND(sym->st_info) != STB_LOCAL
 862                     && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN
 863                         || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL))
 864                         prog->mark_btf_static = true;
 865
 866                 nr_progs++;
 867                 obj->nr_programs = nr_progs;
 868         }
 869
 870         return 0;
 871 }
 872
 873 __u32 get_kernel_version(void)
 874 {
 875         /* On Ubuntu LINUX_VERSION_CODE doesn't correspond to info.release,
 876          * but Ubuntu provides /proc/version_signature file, as described at
 877          * https://ubuntu.com/kernel, with an example contents below, which we
 878          * can use to get a proper LINUX_VERSION_CODE.
 879          *
 880          *   Ubuntu 5.4.0-12.15-generic 5.4.8
 881          *
 882          * In the above, 5.4.8 is what kernel is actually expecting, while
 883          * uname() call will return 5.4.0 in info.release.
 884          */
 885         const char *ubuntu_kver_file = "/proc/version_signature";
 886         __u32 major, minor, patch;
 887         struct utsname info;
 888
 889         if (faccessat(AT_FDCWD, ubuntu_kver_file, R_OK, AT_EACCESS) == 0) {
 890                 FILE *f;
 891
 892                 f = fopen(ubuntu_kver_file, "r");
 893                 if (f) {
 894                         if (fscanf(f, "%*s %*s %d.%d.%d\n", &major, &minor, &patch) == 3) {
 895                                 fclose(f);
 896                                 return KERNEL_VERSION(major, minor, patch);
 897                         }
 898                         fclose(f);
 899                 }
 900                 /* something went wrong, fall back to uname() approach */
 901         }
 902
 903         uname(&info);
 904         if (sscanf(info.release, "%u.%u.%u", &major, &minor, &patch) != 3)
 905                 return 0;
 906         return KERNEL_VERSION(major, minor, patch);
 907 }
 908
 909 static const struct btf_member *
 910 find_member_by_offset(const struct btf_type *t, __u32 bit_offset)
 911 {
 912         struct btf_member *m;
 913         int i;
 914
 915         for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
 916                 if (btf_member_bit_offset(t, i) == bit_offset)
 917                         return m;
 918         }
 919
 920         return NULL;
 921 }
 922
 923 static const struct btf_member *
 924 find_member_by_name(const struct btf *btf, const struct btf_type *t,
 925                     const char *name)
 926 {
 927         struct btf_member *m;
 928         int i;
 929
 930         for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
 931                 if (!strcmp(btf__name_by_offset(btf, m->name_off), name))
 932                         return m;
 933         }
 934
 935         return NULL;
 936 }
 937
 938 #define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_"
 939 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
 940                                    const char *name, __u32 kind);
 941
 942 static int
 943 find_struct_ops_kern_types(const struct btf *btf, const char *tname,
 944                            const struct btf_type **type, __u32 *type_id,
 945                            const struct btf_type **vtype, __u32 *vtype_id,
 946                            const struct btf_member **data_member)
 947 {
 948         const struct btf_type *kern_type, *kern_vtype;
 949         const struct btf_member *kern_data_member;
 950         __s32 kern_vtype_id, kern_type_id;
 951         __u32 i;
 952
 953         kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT);
 954         if (kern_type_id < 0) {
 955                 pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n",
 956                         tname);
 957                 return kern_type_id;
 958         }
 959         kern_type = btf__type_by_id(btf, kern_type_id);
 960
 961         /* Find the corresponding "map_value" type that will be used
 962          * in map_update(BPF_MAP_TYPE_STRUCT_OPS).  For example,
 963          * find "struct bpf_struct_ops_tcp_congestion_ops" from the
 964          * btf_vmlinux.
 965          */
 966         kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX,
 967                                                 tname, BTF_KIND_STRUCT);
 968         if (kern_vtype_id < 0) {
 969                 pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n",
 970                         STRUCT_OPS_VALUE_PREFIX, tname);
 971                 return kern_vtype_id;
 972         }
 973         kern_vtype = btf__type_by_id(btf, kern_vtype_id);
 974
 975         /* Find "struct tcp_congestion_ops" from
 976          * struct bpf_struct_ops_tcp_congestion_ops {
 977          *      [ ... ]
 978          *      struct tcp_congestion_ops data;
 979          * }
 980          */
 981         kern_data_member = btf_members(kern_vtype);
 982         for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) {
 983                 if (kern_data_member->type == kern_type_id)
 984                         break;
 985         }
 986         if (i == btf_vlen(kern_vtype)) {
 987                 pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n",
 988                         tname, STRUCT_OPS_VALUE_PREFIX, tname);
 989                 return -EINVAL;
 990         }
 991
 992         *type = kern_type;
 993         *type_id = kern_type_id;
 994         *vtype = kern_vtype;
 995         *vtype_id = kern_vtype_id;
 996         *data_member = kern_data_member;
 997
 998         return 0;
 999 }
1000
1001 static bool bpf_map__is_struct_ops(const struct bpf_map *map)
1002 {
1003         return map->def.type == BPF_MAP_TYPE_STRUCT_OPS;
1004 }
1005
1006 /* Init the map's fields that depend on kern_btf */
1007 static int bpf_map__init_kern_struct_ops(struct bpf_map *map,
1008                                          const struct btf *btf,
1009                                          const struct btf *kern_btf)
1010 {
1011         const struct btf_member *member, *kern_member, *kern_data_member;
1012         const struct btf_type *type, *kern_type, *kern_vtype;
1013         __u32 i, kern_type_id, kern_vtype_id, kern_data_off;
1014         struct bpf_struct_ops *st_ops;
1015         void *data, *kern_data;
1016         const char *tname;
1017         int err;
1018
1019         st_ops = map->st_ops;
1020         type = st_ops->type;
1021         tname = st_ops->tname;
1022         err = find_struct_ops_kern_types(kern_btf, tname,
1023                                          &kern_type, &kern_type_id,
1024                                          &kern_vtype, &kern_vtype_id,
1025                                          &kern_data_member);
1026         if (err)
1027                 return err;
1028
1029         pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n",
1030                  map->name, st_ops->type_id, kern_type_id, kern_vtype_id);
1031
1032         map->def.value_size = kern_vtype->size;
1033         map->btf_vmlinux_value_type_id = kern_vtype_id;
1034
1035         st_ops->kern_vdata = calloc(1, kern_vtype->size);
1036         if (!st_ops->kern_vdata)
1037                 return -ENOMEM;
1038
1039         data = st_ops->data;
1040         kern_data_off = kern_data_member->offset / 8;
1041         kern_data = st_ops->kern_vdata + kern_data_off;
1042
1043         member = btf_members(type);
1044         for (i = 0; i < btf_vlen(type); i++, member++) {
1045                 const struct btf_type *mtype, *kern_mtype;
1046                 __u32 mtype_id, kern_mtype_id;
1047                 void *mdata, *kern_mdata;
1048                 __s64 msize, kern_msize;
1049                 __u32 moff, kern_moff;
1050                 __u32 kern_member_idx;
1051                 const char *mname;
1052
1053                 mname = btf__name_by_offset(btf, member->name_off);
1054                 kern_member = find_member_by_name(kern_btf, kern_type, mname);
1055                 if (!kern_member) {
1056                         pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n",
1057                                 map->name, mname);
1058                         return -ENOTSUP;
1059                 }
1060
1061                 kern_member_idx = kern_member - btf_members(kern_type);
1062                 if (btf_member_bitfield_size(type, i) ||
1063                     btf_member_bitfield_size(kern_type, kern_member_idx)) {
1064                         pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n",
1065                                 map->name, mname);
1066                         return -ENOTSUP;
1067                 }
1068
1069                 moff = member->offset / 8;
1070                 kern_moff = kern_member->offset / 8;
1071
1072                 mdata = data + moff;
1073                 kern_mdata = kern_data + kern_moff;
1074
1075                 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);
1076                 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type,
1077                                                     &kern_mtype_id);
1078                 if (BTF_INFO_KIND(mtype->info) !=
1079                     BTF_INFO_KIND(kern_mtype->info)) {
1080                         pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n",
1081                                 map->name, mname, BTF_INFO_KIND(mtype->info),
1082                                 BTF_INFO_KIND(kern_mtype->info));
1083                         return -ENOTSUP;
1084                 }
1085
1086                 if (btf_is_ptr(mtype)) {
1087                         struct bpf_program *prog;
1088
1089                         prog = st_ops->progs[i];
1090                         if (!prog)
1091                                 continue;
1092
1093                         kern_mtype = skip_mods_and_typedefs(kern_btf,
1094                                                             kern_mtype->type,
1095                                                             &kern_mtype_id);
1096
1097                         /* mtype->type must be a func_proto which was
1098                          * guaranteed in bpf_object__collect_st_ops_relos(),
1099                          * so only check kern_mtype for func_proto here.
1100                          */
1101                         if (!btf_is_func_proto(kern_mtype)) {
1102                                 pr_warn("struct_ops init_kern %s: kernel member %s is not a func ptr\n",
1103                                         map->name, mname);
1104                                 return -ENOTSUP;
1105                         }
1106
1107                         prog->attach_btf_id = kern_type_id;
1108                         prog->expected_attach_type = kern_member_idx;
1109
1110                         st_ops->kern_func_off[i] = kern_data_off + kern_moff;
1111
1112                         pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n",
1113                                  map->name, mname, prog->name, moff,
1114                                  kern_moff);
1115
1116                         continue;
1117                 }
1118
1119                 msize = btf__resolve_size(btf, mtype_id);
1120                 kern_msize = btf__resolve_size(kern_btf, kern_mtype_id);
1121                 if (msize < 0 || kern_msize < 0 || msize != kern_msize) {
1122                         pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n",
1123                                 map->name, mname, (ssize_t)msize,
1124                                 (ssize_t)kern_msize);
1125                         return -ENOTSUP;
1126                 }
1127
1128                 pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n",
1129                          map->name, mname, (unsigned int)msize,
1130                          moff, kern_moff);
1131                 memcpy(kern_mdata, mdata, msize);
1132         }
1133
1134         return 0;
1135 }
1136
1137 static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj)
1138 {
1139         struct bpf_map *map;
1140         size_t i;
1141         int err;
1142
1143         for (i = 0; i < obj->nr_maps; i++) {
1144                 map = &obj->maps[i];
1145
1146                 if (!bpf_map__is_struct_ops(map))
1147                         continue;
1148
1149                 err = bpf_map__init_kern_struct_ops(map, obj->btf,
1150                                                     obj->btf_vmlinux);
1151                 if (err)
1152                         return err;
1153         }
1154
1155         return 0;
1156 }
1157
1158 static int bpf_object__init_struct_ops_maps(struct bpf_object *obj)
1159 {
1160         const struct btf_type *type, *datasec;
1161         const struct btf_var_secinfo *vsi;
1162         struct bpf_struct_ops *st_ops;
1163         const char *tname, *var_name;
1164         __s32 type_id, datasec_id;
1165         const struct btf *btf;
1166         struct bpf_map *map;
1167         __u32 i;
1168
1169         if (obj->efile.st_ops_shndx == -1)
1170                 return 0;
1171
1172         btf = obj->btf;
1173         datasec_id = btf__find_by_name_kind(btf, STRUCT_OPS_SEC,
1174                                             BTF_KIND_DATASEC);
1175         if (datasec_id < 0) {
1176                 pr_warn("struct_ops init: DATASEC %s not found\n",
1177                         STRUCT_OPS_SEC);
1178                 return -EINVAL;
1179         }
1180
1181         datasec = btf__type_by_id(btf, datasec_id);
1182         vsi = btf_var_secinfos(datasec);
1183         for (i = 0; i < btf_vlen(datasec); i++, vsi++) {
1184                 type = btf__type_by_id(obj->btf, vsi->type);
1185                 var_name = btf__name_by_offset(obj->btf, type->name_off);
1186
1187                 type_id = btf__resolve_type(obj->btf, vsi->type);
1188                 if (type_id < 0) {
1189                         pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n",
1190                                 vsi->type, STRUCT_OPS_SEC);
1191                         return -EINVAL;
1192                 }
1193
1194                 type = btf__type_by_id(obj->btf, type_id);
1195                 tname = btf__name_by_offset(obj->btf, type->name_off);
1196                 if (!tname[0]) {
1197                         pr_warn("struct_ops init: anonymous type is not supported\n");
1198                         return -ENOTSUP;
1199                 }
1200                 if (!btf_is_struct(type)) {
1201                         pr_warn("struct_ops init: %s is not a struct\n", tname);
1202                         return -EINVAL;
1203                 }
1204
1205                 map = bpf_object__add_map(obj);
1206                 if (IS_ERR(map))
1207                         return PTR_ERR(map);
1208
1209                 map->sec_idx = obj->efile.st_ops_shndx;
1210                 map->sec_offset = vsi->offset;
1211                 map->name = strdup(var_name);
1212                 if (!map->name)
1213                         return -ENOMEM;
1214
1215                 map->def.type = BPF_MAP_TYPE_STRUCT_OPS;
1216                 map->def.key_size = sizeof(int);
1217                 map->def.value_size = type->size;
1218                 map->def.max_entries = 1;
1219
1220                 map->st_ops = calloc(1, sizeof(*map->st_ops));
1221                 if (!map->st_ops)
1222                         return -ENOMEM;
1223                 st_ops = map->st_ops;
1224                 st_ops->data = malloc(type->size);
1225                 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs));
1226                 st_ops->kern_func_off = malloc(btf_vlen(type) *
1227                                                sizeof(*st_ops->kern_func_off));
1228                 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off)
1229                         return -ENOMEM;
1230
1231                 if (vsi->offset + type->size > obj->efile.st_ops_data->d_size) {
1232                         pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n",
1233                                 var_name, STRUCT_OPS_SEC);
1234                         return -EINVAL;
1235                 }
1236
1237                 memcpy(st_ops->data,
1238                        obj->efile.st_ops_data->d_buf + vsi->offset,
1239                        type->size);
1240                 st_ops->tname = tname;
1241                 st_ops->type = type;
1242                 st_ops->type_id = type_id;
1243
1244                 pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n",
1245                          tname, type_id, var_name, vsi->offset);
1246         }
1247
1248         return 0;
1249 }
1250
1251 static struct bpf_object *bpf_object__new(const char *path,
1252                                           const void *obj_buf,
1253                                           size_t obj_buf_sz,
1254                                           const char *obj_name)
1255 {
1256         struct bpf_object *obj;
1257         char *end;
1258
1259         obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
1260         if (!obj) {
1261                 pr_warn("alloc memory failed for %s\n", path);
1262                 return ERR_PTR(-ENOMEM);
1263         }
1264
1265         strcpy(obj->path, path);
1266         if (obj_name) {
1267                 libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name));
1268         } else {
1269                 /* Using basename() GNU version which doesn't modify arg. */
1270                 libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name));
1271                 end = strchr(obj->name, '.');
1272                 if (end)
1273                         *end = 0;
1274         }
1275
1276         obj->efile.fd = -1;
1277         /*
1278          * Caller of this function should also call
1279          * bpf_object__elf_finish() after data collection to return
1280          * obj_buf to user. If not, we should duplicate the buffer to
1281          * avoid user freeing them before elf finish.
1282          */
1283         obj->efile.obj_buf = obj_buf;
1284         obj->efile.obj_buf_sz = obj_buf_sz;
1285         obj->efile.btf_maps_shndx = -1;
1286         obj->efile.st_ops_shndx = -1;
1287         obj->kconfig_map_idx = -1;
1288
1289         obj->kern_version = get_kernel_version();
1290         obj->loaded = false;
1291
1292         return obj;
1293 }
1294
1295 static void bpf_object__elf_finish(struct bpf_object *obj)
1296 {
1297         if (!obj->efile.elf)
1298                 return;
1299
1300         elf_end(obj->efile.elf);
1301         obj->efile.elf = NULL;
1302         obj->efile.symbols = NULL;
1303         obj->efile.st_ops_data = NULL;
1304
1305         zfree(&obj->efile.secs);
1306         obj->efile.sec_cnt = 0;
1307         zclose(obj->efile.fd);
1308         obj->efile.obj_buf = NULL;
1309         obj->efile.obj_buf_sz = 0;
1310 }
1311
1312 static int bpf_object__elf_init(struct bpf_object *obj)
1313 {
1314         Elf64_Ehdr *ehdr;
1315         int err = 0;
1316         Elf *elf;
1317
1318         if (obj->efile.elf) {
1319                 pr_warn("elf: init internal error\n");
1320                 return -LIBBPF_ERRNO__LIBELF;
1321         }
1322
1323         if (obj->efile.obj_buf_sz > 0) {
1324                 /* obj_buf should have been validated by bpf_object__open_mem(). */
1325                 elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz);
1326         } else {
1327                 obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC);
1328                 if (obj->efile.fd < 0) {
1329                         char errmsg[STRERR_BUFSIZE], *cp;
1330
1331                         err = -errno;
1332                         cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
1333                         pr_warn("elf: failed to open %s: %s\n", obj->path, cp);
1334                         return err;
1335                 }
1336
1337                 elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL);
1338         }
1339
1340         if (!elf) {
1341                 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1));
1342                 err = -LIBBPF_ERRNO__LIBELF;
1343                 goto errout;
1344         }
1345
1346         obj->efile.elf = elf;
1347
1348         if (elf_kind(elf) != ELF_K_ELF) {
1349                 err = -LIBBPF_ERRNO__FORMAT;
1350                 pr_warn("elf: '%s' is not a proper ELF object\n", obj->path);
1351                 goto errout;
1352         }
1353
1354         if (gelf_getclass(elf) != ELFCLASS64) {
1355                 err = -LIBBPF_ERRNO__FORMAT;
1356                 pr_warn("elf: '%s' is not a 64-bit ELF object\n", obj->path);
1357                 goto errout;
1358         }
1359
1360         obj->efile.ehdr = ehdr = elf64_getehdr(elf);
1361         if (!obj->efile.ehdr) {
1362                 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1));
1363                 err = -LIBBPF_ERRNO__FORMAT;
1364                 goto errout;
1365         }
1366
1367         if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) {
1368                 pr_warn("elf: failed to get section names section index for %s: %s\n",
1369                         obj->path, elf_errmsg(-1));
1370                 err = -LIBBPF_ERRNO__FORMAT;
1371                 goto errout;
1372         }
1373
1374         /* Elf is corrupted/truncated, avoid calling elf_strptr. */
1375         if (!elf_rawdata(elf_getscn(elf, obj->efile.shstrndx), NULL)) {
1376                 pr_warn("elf: failed to get section names strings from %s: %s\n",
1377                         obj->path, elf_errmsg(-1));
1378                 err = -LIBBPF_ERRNO__FORMAT;
1379                 goto errout;
1380         }
1381
1382         /* Old LLVM set e_machine to EM_NONE */
1383         if (ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF)) {
1384                 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path);
1385                 err = -LIBBPF_ERRNO__FORMAT;
1386                 goto errout;
1387         }
1388
1389         return 0;
1390 errout:
1391         bpf_object__elf_finish(obj);
1392         return err;
1393 }
1394
1395 static int bpf_object__check_endianness(struct bpf_object *obj)
1396 {
1397 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1398         if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
1399                 return 0;
1400 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
1401         if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
1402                 return 0;
1403 #else
1404 # error "Unrecognized __BYTE_ORDER__"
1405 #endif
1406         pr_warn("elf: endianness mismatch in %s.\n", obj->path);
1407         return -LIBBPF_ERRNO__ENDIAN;
1408 }
1409
1410 static int
1411 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size)
1412 {
1413         if (!data) {
1414                 pr_warn("invalid license section in %s\n", obj->path);
1415                 return -LIBBPF_ERRNO__FORMAT;
1416         }
1417         /* libbpf_strlcpy() only copies first N - 1 bytes, so size + 1 won't
1418          * go over allowed ELF data section buffer
1419          */
1420         libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license)));
1421         pr_debug("license of %s is %s\n", obj->path, obj->license);
1422         return 0;
1423 }
1424
1425 static int
1426 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size)
1427 {
1428         __u32 kver;
1429
1430         if (!data || size != sizeof(kver)) {
1431                 pr_warn("invalid kver section in %s\n", obj->path);
1432                 return -LIBBPF_ERRNO__FORMAT;
1433         }
1434         memcpy(&kver, data, sizeof(kver));
1435         obj->kern_version = kver;
1436         pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
1437         return 0;
1438 }
1439
1440 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type)
1441 {
1442         if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
1443             type == BPF_MAP_TYPE_HASH_OF_MAPS)
1444                 return true;
1445         return false;
1446 }
1447
1448 static int find_elf_sec_sz(const struct bpf_object *obj, const char *name, __u32 *size)
1449 {
1450         Elf_Data *data;
1451         Elf_Scn *scn;
1452
1453         if (!name)
1454                 return -EINVAL;
1455
1456         scn = elf_sec_by_name(obj, name);
1457         data = elf_sec_data(obj, scn);
1458         if (data) {
1459                 *size = data->d_size;
1460                 return 0; /* found it */
1461         }
1462
1463         return -ENOENT;
1464 }
1465
1466 static Elf64_Sym *find_elf_var_sym(const struct bpf_object *obj, const char *name)
1467 {
1468         Elf_Data *symbols = obj->efile.symbols;
1469         const char *sname;
1470         size_t si;
1471
1472         for (si = 0; si < symbols->d_size / sizeof(Elf64_Sym); si++) {
1473                 Elf64_Sym *sym = elf_sym_by_idx(obj, si);
1474
1475                 if (ELF64_ST_TYPE(sym->st_info) != STT_OBJECT)
1476                         continue;
1477
1478                 if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL &&
1479                     ELF64_ST_BIND(sym->st_info) != STB_WEAK)
1480                         continue;
1481
1482                 sname = elf_sym_str(obj, sym->st_name);
1483                 if (!sname) {
1484                         pr_warn("failed to get sym name string for var %s\n", name);
1485                         return ERR_PTR(-EIO);
1486                 }
1487                 if (strcmp(name, sname) == 0)
1488                         return sym;
1489         }
1490
1491         return ERR_PTR(-ENOENT);
1492 }
1493
1494 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj)
1495 {
1496         struct bpf_map *map;
1497         int err;
1498
1499         err = libbpf_ensure_mem((void **)&obj->maps, &obj->maps_cap,
1500                                 sizeof(*obj->maps), obj->nr_maps + 1);
1501         if (err)
1502                 return ERR_PTR(err);
1503
1504         map = &obj->maps[obj->nr_maps++];
1505         map->obj = obj;
1506         map->fd = -1;
1507         map->inner_map_fd = -1;
1508         map->autocreate = true;
1509
1510         return map;
1511 }
1512
1513 static size_t bpf_map_mmap_sz(const struct bpf_map *map)
1514 {
1515         long page_sz = sysconf(_SC_PAGE_SIZE);
1516         size_t map_sz;
1517
1518         map_sz = (size_t)roundup(map->def.value_size, 8) * map->def.max_entries;
1519         map_sz = roundup(map_sz, page_sz);
1520         return map_sz;
1521 }
1522
1523 static char *internal_map_name(struct bpf_object *obj, const char *real_name)
1524 {
1525         char map_name[BPF_OBJ_NAME_LEN], *p;
1526         int pfx_len, sfx_len = max((size_t)7, strlen(real_name));
1527
1528         /* This is one of the more confusing parts of libbpf for various
1529          * reasons, some of which are historical. The original idea for naming
1530          * internal names was to include as much of BPF object name prefix as
1531          * possible, so that it can be distinguished from similar internal
1532          * maps of a different BPF object.
1533          * As an example, let's say we have bpf_object named 'my_object_name'
1534          * and internal map corresponding to '.rodata' ELF section. The final
1535          * map name advertised to user and to the kernel will be
1536          * 'my_objec.rodata', taking first 8 characters of object name and
1537          * entire 7 characters of '.rodata'.
1538          * Somewhat confusingly, if internal map ELF section name is shorter
1539          * than 7 characters, e.g., '.bss', we still reserve 7 characters
1540          * for the suffix, even though we only have 4 actual characters, and
1541          * resulting map will be called 'my_objec.bss', not even using all 15
1542          * characters allowed by the kernel. Oh well, at least the truncated
1543          * object name is somewhat consistent in this case. But if the map
1544          * name is '.kconfig', we'll still have entirety of '.kconfig' added
1545          * (8 chars) and thus will be left with only first 7 characters of the
1546          * object name ('my_obje'). Happy guessing, user, that the final map
1547          * name will be "my_obje.kconfig".
1548          * Now, with libbpf starting to support arbitrarily named .rodata.*
1549          * and .data.* data sections, it's possible that ELF section name is
1550          * longer than allowed 15 chars, so we now need to be careful to take
1551          * only up to 15 first characters of ELF name, taking no BPF object
1552          * name characters at all. So '.rodata.abracadabra' will result in
1553          * '.rodata.abracad' kernel and user-visible name.
1554          * We need to keep this convoluted logic intact for .data, .bss and
1555          * .rodata maps, but for new custom .data.custom and .rodata.custom
1556          * maps we use their ELF names as is, not prepending bpf_object name
1557          * in front. We still need to truncate them to 15 characters for the
1558          * kernel. Full name can be recovered for such maps by using DATASEC
1559          * BTF type associated with such map's value type, though.
1560          */
1561         if (sfx_len >= BPF_OBJ_NAME_LEN)
1562                 sfx_len = BPF_OBJ_NAME_LEN - 1;
1563
1564         /* if there are two or more dots in map name, it's a custom dot map */
1565         if (strchr(real_name + 1, '.') != NULL)
1566                 pfx_len = 0;
1567         else
1568                 pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name));
1569
1570         snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,
1571                  sfx_len, real_name);
1572
1573         /* sanitise map name to characters allowed by kernel */
1574         for (p = map_name; *p && p < map_name + sizeof(map_name); p++)
1575                 if (!isalnum(*p) && *p != '_' && *p != '.')
1576                         *p = '_';
1577
1578         return strdup(map_name);
1579 }
1580
1581 static int
1582 map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map);
1583
1584 /* Internal BPF map is mmap()'able only if at least one of corresponding
1585  * DATASEC's VARs are to be exposed through BPF skeleton. I.e., it's a GLOBAL
1586  * variable and it's not marked as __hidden (which turns it into, effectively,
1587  * a STATIC variable).
1588  */
1589 static bool map_is_mmapable(struct bpf_object *obj, struct bpf_map *map)
1590 {
1591         const struct btf_type *t, *vt;
1592         struct btf_var_secinfo *vsi;
1593         int i, n;
1594
1595         if (!map->btf_value_type_id)
1596                 return false;
1597
1598         t = btf__type_by_id(obj->btf, map->btf_value_type_id);
1599         if (!btf_is_datasec(t))
1600                 return false;
1601
1602         vsi = btf_var_secinfos(t);
1603         for (i = 0, n = btf_vlen(t); i < n; i++, vsi++) {
1604                 vt = btf__type_by_id(obj->btf, vsi->type);
1605                 if (!btf_is_var(vt))
1606                         continue;
1607
1608                 if (btf_var(vt)->linkage != BTF_VAR_STATIC)
1609                         return true;
1610         }
1611
1612         return false;
1613 }
1614
1615 static int
1616 bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
1617                               const char *real_name, int sec_idx, void *data, size_t data_sz)
1618 {
1619         struct bpf_map_def *def;
1620         struct bpf_map *map;
1621         int err;
1622
1623         map = bpf_object__add_map(obj);
1624         if (IS_ERR(map))
1625                 return PTR_ERR(map);
1626
1627         map->libbpf_type = type;
1628         map->sec_idx = sec_idx;
1629         map->sec_offset = 0;
1630         map->real_name = strdup(real_name);
1631         map->name = internal_map_name(obj, real_name);
1632         if (!map->real_name || !map->name) {
1633                 zfree(&map->real_name);
1634                 zfree(&map->name);
1635                 return -ENOMEM;
1636         }
1637
1638         def = &map->def;
1639         def->type = BPF_MAP_TYPE_ARRAY;
1640         def->key_size = sizeof(int);
1641         def->value_size = data_sz;
1642         def->max_entries = 1;
1643         def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG
1644                          ? BPF_F_RDONLY_PROG : 0;
1645
1646         /* failures are fine because of maps like .rodata.str1.1 */
1647         (void) map_fill_btf_type_info(obj, map);
1648
1649         if (map_is_mmapable(obj, map))
1650                 def->map_flags |= BPF_F_MMAPABLE;
1651
1652         pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n",
1653                  map->name, map->sec_idx, map->sec_offset, def->map_flags);
1654
1655         map->mmaped = mmap(NULL, bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE,
1656                            MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1657         if (map->mmaped == MAP_FAILED) {
1658                 err = -errno;
1659                 map->mmaped = NULL;
1660                 pr_warn("failed to alloc map '%s' content buffer: %d\n",
1661                         map->name, err);
1662                 zfree(&map->real_name);
1663                 zfree(&map->name);
1664                 return err;
1665         }
1666
1667         if (data)
1668                 memcpy(map->mmaped, data, data_sz);
1669
1670         pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
1671         return 0;
1672 }
1673
1674 static int bpf_object__init_global_data_maps(struct bpf_object *obj)
1675 {
1676         struct elf_sec_desc *sec_desc;
1677         const char *sec_name;
1678         int err = 0, sec_idx;
1679
1680         /*
1681          * Populate obj->maps with libbpf internal maps.
1682          */
1683         for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) {
1684                 sec_desc = &obj->efile.secs[sec_idx];
1685
1686                 /* Skip recognized sections with size 0. */
1687                 if (!sec_desc->data || sec_desc->data->d_size == 0)
1688                         continue;
1689
1690                 switch (sec_desc->sec_type) {
1691                 case SEC_DATA:
1692                         sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
1693                         err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA,
1694                                                             sec_name, sec_idx,
1695                                                             sec_desc->data->d_buf,
1696                                                             sec_desc->data->d_size);
1697                         break;
1698                 case SEC_RODATA:
1699                         obj->has_rodata = true;
1700                         sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
1701                         err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA,
1702                                                             sec_name, sec_idx,
1703                                                             sec_desc->data->d_buf,
1704                                                             sec_desc->data->d_size);
1705                         break;
1706                 case SEC_BSS:
1707                         sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
1708                         err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS,
1709                                                             sec_name, sec_idx,
1710                                                             NULL,
1711                                                             sec_desc->data->d_size);
1712                         break;
1713                 default:
1714                         /* skip */
1715                         break;
1716                 }
1717                 if (err)
1718                         return err;
1719         }
1720         return 0;
1721 }
1722
1723
1724 static struct extern_desc *find_extern_by_name(const struct bpf_object *obj,
1725                                                const void *name)
1726 {
1727         int i;
1728
1729         for (i = 0; i < obj->nr_extern; i++) {
1730                 if (strcmp(obj->externs[i].name, name) == 0)
1731                         return &obj->externs[i];
1732         }
1733         return NULL;
1734 }
1735
1736 static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val,
1737                               char value)
1738 {
1739         switch (ext->kcfg.type) {
1740         case KCFG_BOOL:
1741                 if (value == 'm') {
1742                         pr_warn("extern (kcfg) '%s': value '%c' implies tristate or char type\n",
1743                                 ext->name, value);
1744                         return -EINVAL;
1745                 }
1746                 *(bool *)ext_val = value == 'y' ? true : false;
1747                 break;
1748         case KCFG_TRISTATE:
1749                 if (value == 'y')
1750                         *(enum libbpf_tristate *)ext_val = TRI_YES;
1751                 else if (value == 'm')
1752                         *(enum libbpf_tristate *)ext_val = TRI_MODULE;
1753                 else /* value == 'n' */
1754                         *(enum libbpf_tristate *)ext_val = TRI_NO;
1755                 break;
1756         case KCFG_CHAR:
1757                 *(char *)ext_val = value;
1758                 break;
1759         case KCFG_UNKNOWN:
1760         case KCFG_INT:
1761         case KCFG_CHAR_ARR:
1762         default:
1763                 pr_warn("extern (kcfg) '%s': value '%c' implies bool, tristate, or char type\n",
1764                         ext->name, value);
1765                 return -EINVAL;
1766         }
1767         ext->is_set = true;
1768         return 0;
1769 }
1770
1771 static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val,
1772                               const char *value)
1773 {
1774         size_t len;
1775
1776         if (ext->kcfg.type != KCFG_CHAR_ARR) {
1777                 pr_warn("extern (kcfg) '%s': value '%s' implies char array type\n",
1778                         ext->name, value);
1779                 return -EINVAL;
1780         }
1781
1782         len = strlen(value);
1783         if (value[len - 1] != '"') {
1784                 pr_warn("extern (kcfg) '%s': invalid string config '%s'\n",
1785                         ext->name, value);
1786                 return -EINVAL;
1787         }
1788
1789         /* strip quotes */
1790         len -= 2;
1791         if (len >= ext->kcfg.sz) {
1792                 pr_warn("extern (kcfg) '%s': long string '%s' of (%zu bytes) truncated to %d bytes\n",
1793                         ext->name, value, len, ext->kcfg.sz - 1);
1794                 len = ext->kcfg.sz - 1;
1795         }
1796         memcpy(ext_val, value + 1, len);
1797         ext_val[len] = '\0';
1798         ext->is_set = true;
1799         return 0;
1800 }
1801
1802 static int parse_u64(const char *value, __u64 *res)
1803 {
1804         char *value_end;
1805         int err;
1806
1807         errno = 0;
1808         *res = strtoull(value, &value_end, 0);
1809         if (errno) {
1810                 err = -errno;
1811                 pr_warn("failed to parse '%s' as integer: %d\n", value, err);
1812                 return err;
1813         }
1814         if (*value_end) {
1815                 pr_warn("failed to parse '%s' as integer completely\n", value);
1816                 return -EINVAL;
1817         }
1818         return 0;
1819 }
1820
1821 static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v)
1822 {
1823         int bit_sz = ext->kcfg.sz * 8;
1824
1825         if (ext->kcfg.sz == 8)
1826                 return true;
1827
1828         /* Validate that value stored in u64 fits in integer of `ext->sz`
1829          * bytes size without any loss of information. If the target integer
1830          * is signed, we rely on the following limits of integer type of
1831          * Y bits and subsequent transformation:
1832          *
1833          *     -2^(Y-1) <= X           <= 2^(Y-1) - 1
1834          *            0 <= X + 2^(Y-1) <= 2^Y - 1
1835          *            0 <= X + 2^(Y-1) <  2^Y
1836          *
1837          *  For unsigned target integer, check that all the (64 - Y) bits are
1838          *  zero.
1839          */
1840         if (ext->kcfg.is_signed)
1841                 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);
1842         else
1843                 return (v >> bit_sz) == 0;
1844 }
1845
1846 static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val,
1847                               __u64 value)
1848 {
1849         if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR &&
1850             ext->kcfg.type != KCFG_BOOL) {
1851                 pr_warn("extern (kcfg) '%s': value '%llu' implies integer, char, or boolean type\n",
1852                         ext->name, (unsigned long long)value);
1853                 return -EINVAL;
1854         }
1855         if (ext->kcfg.type == KCFG_BOOL && value > 1) {
1856                 pr_warn("extern (kcfg) '%s': value '%llu' isn't boolean compatible\n",
1857                         ext->name, (unsigned long long)value);
1858                 return -EINVAL;
1859
1860         }
1861         if (!is_kcfg_value_in_range(ext, value)) {
1862                 pr_warn("extern (kcfg) '%s': value '%llu' doesn't fit in %d bytes\n",
1863                         ext->name, (unsigned long long)value, ext->kcfg.sz);
1864                 return -ERANGE;
1865         }
1866         switch (ext->kcfg.sz) {
1867         case 1:
1868                 *(__u8 *)ext_val = value;
1869                 break;
1870         case 2:
1871                 *(__u16 *)ext_val = value;
1872                 break;
1873         case 4:
1874                 *(__u32 *)ext_val = value;
1875                 break;
1876         case 8:
1877                 *(__u64 *)ext_val = value;
1878                 break;
1879         default:
1880                 return -EINVAL;
1881         }
1882         ext->is_set = true;
1883         return 0;
1884 }
1885
1886 static int bpf_object__process_kconfig_line(struct bpf_object *obj,
1887                                             char *buf, void *data)
1888 {
1889         struct extern_desc *ext;
1890         char *sep, *value;
1891         int len, err = 0;
1892         void *ext_val;
1893         __u64 num;
1894
1895         if (!str_has_pfx(buf, "CONFIG_"))
1896                 return 0;
1897
1898         sep = strchr(buf, '=');
1899         if (!sep) {
1900                 pr_warn("failed to parse '%s': no separator\n", buf);
1901                 return -EINVAL;
1902         }
1903
1904         /* Trim ending '\n' */
1905         len = strlen(buf);
1906         if (buf[len - 1] == '\n')
1907                 buf[len - 1] = '\0';
1908         /* Split on '=' and ensure that a value is present. */
1909         *sep = '\0';
1910         if (!sep[1]) {
1911                 *sep = '=';
1912                 pr_warn("failed to parse '%s': no value\n", buf);
1913                 return -EINVAL;
1914         }
1915
1916         ext = find_extern_by_name(obj, buf);
1917         if (!ext || ext->is_set)
1918                 return 0;
1919
1920         ext_val = data + ext->kcfg.data_off;
1921         value = sep + 1;
1922
1923         switch (*value) {
1924         case 'y': case 'n': case 'm':
1925                 err = set_kcfg_value_tri(ext, ext_val, *value);
1926                 break;
1927         case '"':
1928                 err = set_kcfg_value_str(ext, ext_val, value);
1929                 break;
1930         default:
1931                 /* assume integer */
1932                 err = parse_u64(value, &num);
1933                 if (err) {
1934                         pr_warn("extern (kcfg) '%s': value '%s' isn't a valid integer\n", ext->name, value);
1935                         return err;
1936                 }
1937                 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) {
1938                         pr_warn("extern (kcfg) '%s': value '%s' implies integer type\n", ext->name, value);
1939                         return -EINVAL;
1940                 }
1941                 err = set_kcfg_value_num(ext, ext_val, num);
1942                 break;
1943         }
1944         if (err)
1945                 return err;
1946         pr_debug("extern (kcfg) '%s': set to %s\n", ext->name, value);
1947         return 0;
1948 }
1949
1950 static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data)
1951 {
1952         char buf[PATH_MAX];
1953         struct utsname uts;
1954         int len, err = 0;
1955         gzFile file;
1956
1957         uname(&uts);
1958         len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release);
1959         if (len < 0)
1960                 return -EINVAL;
1961         else if (len >= PATH_MAX)
1962                 return -ENAMETOOLONG;
1963
1964         /* gzopen also accepts uncompressed files. */
1965         file = gzopen(buf, "r");
1966         if (!file)
1967                 file = gzopen("/proc/config.gz", "r");
1968
1969         if (!file) {
1970                 pr_warn("failed to open system Kconfig\n");
1971                 return -ENOENT;
1972         }
1973
1974         while (gzgets(file, buf, sizeof(buf))) {
1975                 err = bpf_object__process_kconfig_line(obj, buf, data);
1976                 if (err) {
1977                         pr_warn("error parsing system Kconfig line '%s': %d\n",
1978                                 buf, err);
1979                         goto out;
1980                 }
1981         }
1982
1983 out:
1984         gzclose(file);
1985         return err;
1986 }
1987
1988 static int bpf_object__read_kconfig_mem(struct bpf_object *obj,
1989                                         const char *config, void *data)
1990 {
1991         char buf[PATH_MAX];
1992         int err = 0;
1993         FILE *file;
1994
1995         file = fmemopen((void *)config, strlen(config), "r");
1996         if (!file) {
1997                 err = -errno;
1998                 pr_warn("failed to open in-memory Kconfig: %d\n", err);
1999                 return err;
2000         }
2001
2002         while (fgets(buf, sizeof(buf), file)) {
2003                 err = bpf_object__process_kconfig_line(obj, buf, data);
2004                 if (err) {
2005                         pr_warn("error parsing in-memory Kconfig line '%s': %d\n",
2006                                 buf, err);
2007                         break;
2008                 }
2009         }
2010
2011         fclose(file);
2012         return err;
2013 }
2014
2015 static int bpf_object__init_kconfig_map(struct bpf_object *obj)
2016 {
2017         struct extern_desc *last_ext = NULL, *ext;
2018         size_t map_sz;
2019         int i, err;
2020
2021         for (i = 0; i < obj->nr_extern; i++) {
2022                 ext = &obj->externs[i];
2023                 if (ext->type == EXT_KCFG)
2024                         last_ext = ext;
2025         }
2026
2027         if (!last_ext)
2028                 return 0;
2029
2030         map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz;
2031         err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG,
2032                                             ".kconfig", obj->efile.symbols_shndx,
2033                                             NULL, map_sz);
2034         if (err)
2035                 return err;
2036
2037         obj->kconfig_map_idx = obj->nr_maps - 1;
2038
2039         return 0;
2040 }
2041
2042 const struct btf_type *
2043 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id)
2044 {
2045         const struct btf_type *t = btf__type_by_id(btf, id);
2046
2047         if (res_id)
2048                 *res_id = id;
2049
2050         while (btf_is_mod(t) || btf_is_typedef(t)) {
2051                 if (res_id)
2052                         *res_id = t->type;
2053                 t = btf__type_by_id(btf, t->type);
2054         }
2055
2056         return t;
2057 }
2058
2059 static const struct btf_type *
2060 resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id)
2061 {
2062         const struct btf_type *t;
2063
2064         t = skip_mods_and_typedefs(btf, id, NULL);
2065         if (!btf_is_ptr(t))
2066                 return NULL;
2067
2068         t = skip_mods_and_typedefs(btf, t->type, res_id);
2069
2070         return btf_is_func_proto(t) ? t : NULL;
2071 }
2072
2073 static const char *__btf_kind_str(__u16 kind)
2074 {
2075         switch (kind) {
2076         case BTF_KIND_UNKN: return "void";
2077         case BTF_KIND_INT: return "int";
2078         case BTF_KIND_PTR: return "ptr";
2079         case BTF_KIND_ARRAY: return "array";
2080         case BTF_KIND_STRUCT: return "struct";
2081         case BTF_KIND_UNION: return "union";
2082         case BTF_KIND_ENUM: return "enum";
2083         case BTF_KIND_FWD: return "fwd";
2084         case BTF_KIND_TYPEDEF: return "typedef";
2085         case BTF_KIND_VOLATILE: return "volatile";
2086         case BTF_KIND_CONST: return "const";
2087         case BTF_KIND_RESTRICT: return "restrict";
2088         case BTF_KIND_FUNC: return "func";
2089         case BTF_KIND_FUNC_PROTO: return "func_proto";
2090         case BTF_KIND_VAR: return "var";
2091         case BTF_KIND_DATASEC: return "datasec";
2092         case BTF_KIND_FLOAT: return "float";
2093         case BTF_KIND_DECL_TAG: return "decl_tag";
2094         case BTF_KIND_TYPE_TAG: return "type_tag";
2095         case BTF_KIND_ENUM64: return "enum64";
2096         default: return "unknown";
2097         }
2098 }
2099
2100 const char *btf_kind_str(const struct btf_type *t)
2101 {
2102         return __btf_kind_str(btf_kind(t));
2103 }
2104
2105 /*
2106  * Fetch integer attribute of BTF map definition. Such attributes are
2107  * represented using a pointer to an array, in which dimensionality of array
2108  * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY];
2109  * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF
2110  * type definition, while using only sizeof(void *) space in ELF data section.
2111  */
2112 static bool get_map_field_int(const char *map_name, const struct btf *btf,
2113                               const struct btf_member *m, __u32 *res)
2114 {
2115         const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
2116         const char *name = btf__name_by_offset(btf, m->name_off);
2117         const struct btf_array *arr_info;
2118         const struct btf_type *arr_t;
2119
2120         if (!btf_is_ptr(t)) {
2121                 pr_warn("map '%s': attr '%s': expected PTR, got %s.\n",
2122                         map_name, name, btf_kind_str(t));
2123                 return false;
2124         }
2125
2126         arr_t = btf__type_by_id(btf, t->type);
2127         if (!arr_t) {
2128                 pr_warn("map '%s': attr '%s': type [%u] not found.\n",
2129                         map_name, name, t->type);
2130                 return false;
2131         }
2132         if (!btf_is_array(arr_t)) {
2133                 pr_warn("map '%s': attr '%s': expected ARRAY, got %s.\n",
2134                         map_name, name, btf_kind_str(arr_t));
2135                 return false;
2136         }
2137         arr_info = btf_array(arr_t);
2138         *res = arr_info->nelems;
2139         return true;
2140 }
2141
2142 static int pathname_concat(char *buf, size_t buf_sz, const char *path, const char *name)
2143 {
2144         int len;
2145
2146         len = snprintf(buf, buf_sz, "%s/%s", path, name);
2147         if (len < 0)
2148                 return -EINVAL;
2149         if (len >= buf_sz)
2150                 return -ENAMETOOLONG;
2151
2152         return 0;
2153 }
2154
2155 static int build_map_pin_path(struct bpf_map *map, const char *path)
2156 {
2157         char buf[PATH_MAX];
2158         int err;
2159
2160         if (!path)
2161                 path = "/sys/fs/bpf";
2162
2163         err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map));
2164         if (err)
2165                 return err;
2166
2167         return bpf_map__set_pin_path(map, buf);
2168 }
2169
2170 /* should match definition in bpf_helpers.h */
2171 enum libbpf_pin_type {
2172         LIBBPF_PIN_NONE,
2173         /* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */
2174         LIBBPF_PIN_BY_NAME,
2175 };
2176
2177 int parse_btf_map_def(const char *map_name, struct btf *btf,
2178                       const struct btf_type *def_t, bool strict,
2179                       struct btf_map_def *map_def, struct btf_map_def *inner_def)
2180 {
2181         const struct btf_type *t;
2182         const struct btf_member *m;
2183         bool is_inner = inner_def == NULL;
2184         int vlen, i;
2185
2186         vlen = btf_vlen(def_t);
2187         m = btf_members(def_t);
2188         for (i = 0; i < vlen; i++, m++) {
2189                 const char *name = btf__name_by_offset(btf, m->name_off);
2190
2191                 if (!name) {
2192                         pr_warn("map '%s': invalid field #%d.\n", map_name, i);
2193                         return -EINVAL;
2194                 }
2195                 if (strcmp(name, "type") == 0) {
2196                         if (!get_map_field_int(map_name, btf, m, &map_def->map_type))
2197                                 return -EINVAL;
2198                         map_def->parts |= MAP_DEF_MAP_TYPE;
2199                 } else if (strcmp(name, "max_entries") == 0) {
2200                         if (!get_map_field_int(map_name, btf, m, &map_def->max_entries))
2201                                 return -EINVAL;
2202                         map_def->parts |= MAP_DEF_MAX_ENTRIES;
2203                 } else if (strcmp(name, "map_flags") == 0) {
2204                         if (!get_map_field_int(map_name, btf, m, &map_def->map_flags))
2205                                 return -EINVAL;
2206                         map_def->parts |= MAP_DEF_MAP_FLAGS;
2207                 } else if (strcmp(name, "numa_node") == 0) {
2208                         if (!get_map_field_int(map_name, btf, m, &map_def->numa_node))
2209                                 return -EINVAL;
2210                         map_def->parts |= MAP_DEF_NUMA_NODE;
2211                 } else if (strcmp(name, "key_size") == 0) {
2212                         __u32 sz;
2213
2214                         if (!get_map_field_int(map_name, btf, m, &sz))
2215                                 return -EINVAL;
2216                         if (map_def->key_size && map_def->key_size != sz) {
2217                                 pr_warn("map '%s': conflicting key size %u != %u.\n",
2218                                         map_name, map_def->key_size, sz);
2219                                 return -EINVAL;
2220                         }
2221                         map_def->key_size = sz;
2222                         map_def->parts |= MAP_DEF_KEY_SIZE;
2223                 } else if (strcmp(name, "key") == 0) {
2224                         __s64 sz;
2225
2226                         t = btf__type_by_id(btf, m->type);
2227                         if (!t) {
2228                                 pr_warn("map '%s': key type [%d] not found.\n",
2229                                         map_name, m->type);
2230                                 return -EINVAL;
2231                         }
2232                         if (!btf_is_ptr(t)) {
2233                                 pr_warn("map '%s': key spec is not PTR: %s.\n",
2234                                         map_name, btf_kind_str(t));
2235                                 return -EINVAL;
2236                         }
2237                         sz = btf__resolve_size(btf, t->type);
2238                         if (sz < 0) {
2239                                 pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n",
2240                                         map_name, t->type, (ssize_t)sz);
2241                                 return sz;
2242                         }
2243                         if (map_def->key_size && map_def->key_size != sz) {
2244                                 pr_warn("map '%s': conflicting key size %u != %zd.\n",
2245                                         map_name, map_def->key_size, (ssize_t)sz);
2246                                 return -EINVAL;
2247                         }
2248                         map_def->key_size = sz;
2249                         map_def->key_type_id = t->type;
2250                         map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE;
2251                 } else if (strcmp(name, "value_size") == 0) {
2252                         __u32 sz;
2253
2254                         if (!get_map_field_int(map_name, btf, m, &sz))
2255                                 return -EINVAL;
2256                         if (map_def->value_size && map_def->value_size != sz) {
2257                                 pr_warn("map '%s': conflicting value size %u != %u.\n",
2258                                         map_name, map_def->value_size, sz);
2259                                 return -EINVAL;
2260                         }
2261                         map_def->value_size = sz;
2262                         map_def->parts |= MAP_DEF_VALUE_SIZE;
2263                 } else if (strcmp(name, "value") == 0) {
2264                         __s64 sz;
2265
2266                         t = btf__type_by_id(btf, m->type);
2267                         if (!t) {
2268                                 pr_warn("map '%s': value type [%d] not found.\n",
2269                                         map_name, m->type);
2270                                 return -EINVAL;
2271                         }
2272                         if (!btf_is_ptr(t)) {
2273                                 pr_warn("map '%s': value spec is not PTR: %s.\n",
2274                                         map_name, btf_kind_str(t));
2275                                 return -EINVAL;
2276                         }
2277                         sz = btf__resolve_size(btf, t->type);
2278                         if (sz < 0) {
2279                                 pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n",
2280                                         map_name, t->type, (ssize_t)sz);
2281                                 return sz;
2282                         }
2283                         if (map_def->value_size && map_def->value_size != sz) {
2284                                 pr_warn("map '%s': conflicting value size %u != %zd.\n",
2285                                         map_name, map_def->value_size, (ssize_t)sz);
2286                                 return -EINVAL;
2287                         }
2288                         map_def->value_size = sz;
2289                         map_def->value_type_id = t->type;
2290                         map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE;
2291                 }
2292                 else if (strcmp(name, "values") == 0) {
2293                         bool is_map_in_map = bpf_map_type__is_map_in_map(map_def->map_type);
2294                         bool is_prog_array = map_def->map_type == BPF_MAP_TYPE_PROG_ARRAY;
2295                         const char *desc = is_map_in_map ? "map-in-map inner" : "prog-array value";
2296                         char inner_map_name[128];
2297                         int err;
2298
2299                         if (is_inner) {
2300                                 pr_warn("map '%s': multi-level inner maps not supported.\n",
2301                                         map_name);
2302                                 return -ENOTSUP;
2303                         }
2304                         if (i != vlen - 1) {
2305                                 pr_warn("map '%s': '%s' member should be last.\n",
2306                                         map_name, name);
2307                                 return -EINVAL;
2308                         }
2309                         if (!is_map_in_map && !is_prog_array) {
2310                                 pr_warn("map '%s': should be map-in-map or prog-array.\n",
2311                                         map_name);
2312                                 return -ENOTSUP;
2313                         }
2314                         if (map_def->value_size && map_def->value_size != 4) {
2315                                 pr_warn("map '%s': conflicting value size %u != 4.\n",
2316                                         map_name, map_def->value_size);
2317                                 return -EINVAL;
2318                         }
2319                         map_def->value_size = 4;
2320                         t = btf__type_by_id(btf, m->type);
2321                         if (!t) {
2322                                 pr_warn("map '%s': %s type [%d] not found.\n",
2323                                         map_name, desc, m->type);
2324                                 return -EINVAL;
2325                         }
2326                         if (!btf_is_array(t) || btf_array(t)->nelems) {
2327                                 pr_warn("map '%s': %s spec is not a zero-sized array.\n",
2328                                         map_name, desc);
2329                                 return -EINVAL;
2330                         }
2331                         t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL);
2332                         if (!btf_is_ptr(t)) {
2333                                 pr_warn("map '%s': %s def is of unexpected kind %s.\n",
2334                                         map_name, desc, btf_kind_str(t));
2335                                 return -EINVAL;
2336                         }
2337                         t = skip_mods_and_typedefs(btf, t->type, NULL);
2338                         if (is_prog_array) {
2339                                 if (!btf_is_func_proto(t)) {
2340                                         pr_warn("map '%s': prog-array value def is of unexpected kind %s.\n",
2341                                                 map_name, btf_kind_str(t));
2342                                         return -EINVAL;
2343                                 }
2344                                 continue;
2345                         }
2346                         if (!btf_is_struct(t)) {
2347                                 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
2348                                         map_name, btf_kind_str(t));
2349                                 return -EINVAL;
2350                         }
2351
2352                         snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", map_name);
2353                         err = parse_btf_map_def(inner_map_name, btf, t, strict, inner_def, NULL);
2354                         if (err)
2355                                 return err;
2356
2357                         map_def->parts |= MAP_DEF_INNER_MAP;
2358                 } else if (strcmp(name, "pinning") == 0) {
2359                         __u32 val;
2360
2361                         if (is_inner) {
2362                                 pr_warn("map '%s': inner def can't be pinned.\n", map_name);
2363                                 return -EINVAL;
2364                         }
2365                         if (!get_map_field_int(map_name, btf, m, &val))
2366                                 return -EINVAL;
2367                         if (val != LIBBPF_PIN_NONE && val != LIBBPF_PIN_BY_NAME) {
2368                                 pr_warn("map '%s': invalid pinning value %u.\n",
2369                                         map_name, val);
2370                                 return -EINVAL;
2371                         }
2372                         map_def->pinning = val;
2373                         map_def->parts |= MAP_DEF_PINNING;
2374                 } else if (strcmp(name, "map_extra") == 0) {
2375                         __u32 map_extra;
2376
2377                         if (!get_map_field_int(map_name, btf, m, &map_extra))
2378                                 return -EINVAL;
2379                         map_def->map_extra = map_extra;
2380                         map_def->parts |= MAP_DEF_MAP_EXTRA;
2381                 } else {
2382                         if (strict) {
2383                                 pr_warn("map '%s': unknown field '%s'.\n", map_name, name);
2384                                 return -ENOTSUP;
2385                         }
2386                         pr_debug("map '%s': ignoring unknown field '%s'.\n", map_name, name);
2387                 }
2388         }
2389
2390         if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) {
2391                 pr_warn("map '%s': map type isn't specified.\n", map_name);
2392                 return -EINVAL;
2393         }
2394
2395         return 0;
2396 }
2397
2398 static size_t adjust_ringbuf_sz(size_t sz)
2399 {
2400         __u32 page_sz = sysconf(_SC_PAGE_SIZE);
2401         __u32 mul;
2402
2403         /* if user forgot to set any size, make sure they see error */
2404         if (sz == 0)
2405                 return 0;
2406         /* Kernel expects BPF_MAP_TYPE_RINGBUF's max_entries to be
2407          * a power-of-2 multiple of kernel's page size. If user diligently
2408          * satisified these conditions, pass the size through.
2409          */
2410         if ((sz % page_sz) == 0 && is_pow_of_2(sz / page_sz))
2411                 return sz;
2412
2413         /* Otherwise find closest (page_sz * power_of_2) product bigger than
2414          * user-set size to satisfy both user size request and kernel
2415          * requirements and substitute correct max_entries for map creation.
2416          */
2417         for (mul = 1; mul <= UINT_MAX / page_sz; mul <<= 1) {
2418                 if (mul * page_sz > sz)
2419                         return mul * page_sz;
2420         }
2421
2422         /* if it's impossible to satisfy the conditions (i.e., user size is
2423          * very close to UINT_MAX but is not a power-of-2 multiple of
2424          * page_size) then just return original size and let kernel reject it
2425          */
2426         return sz;
2427 }
2428
2429 static bool map_is_ringbuf(const struct bpf_map *map)
2430 {
2431         return map->def.type == BPF_MAP_TYPE_RINGBUF ||
2432                map->def.type == BPF_MAP_TYPE_USER_RINGBUF;
2433 }
2434
2435 static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def)
2436 {
2437         map->def.type = def->map_type;
2438         map->def.key_size = def->key_size;
2439         map->def.value_size = def->value_size;
2440         map->def.max_entries = def->max_entries;
2441         map->def.map_flags = def->map_flags;
2442         map->map_extra = def->map_extra;
2443
2444         map->numa_node = def->numa_node;
2445         map->btf_key_type_id = def->key_type_id;
2446         map->btf_value_type_id = def->value_type_id;
2447
2448         /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
2449         if (map_is_ringbuf(map))
2450                 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
2451
2452         if (def->parts & MAP_DEF_MAP_TYPE)
2453                 pr_debug("map '%s': found type = %u.\n", map->name, def->map_type);
2454
2455         if (def->parts & MAP_DEF_KEY_TYPE)
2456                 pr_debug("map '%s': found key [%u], sz = %u.\n",
2457                          map->name, def->key_type_id, def->key_size);
2458         else if (def->parts & MAP_DEF_KEY_SIZE)
2459                 pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size);
2460
2461         if (def->parts & MAP_DEF_VALUE_TYPE)
2462                 pr_debug("map '%s': found value [%u], sz = %u.\n",
2463                          map->name, def->value_type_id, def->value_size);
2464         else if (def->parts & MAP_DEF_VALUE_SIZE)
2465                 pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size);
2466
2467         if (def->parts & MAP_DEF_MAX_ENTRIES)
2468                 pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries);
2469         if (def->parts & MAP_DEF_MAP_FLAGS)
2470                 pr_debug("map '%s': found map_flags = 0x%x.\n", map->name, def->map_flags);
2471         if (def->parts & MAP_DEF_MAP_EXTRA)
2472                 pr_debug("map '%s': found map_extra = 0x%llx.\n", map->name,
2473                          (unsigned long long)def->map_extra);
2474         if (def->parts & MAP_DEF_PINNING)
2475                 pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning);
2476         if (def->parts & MAP_DEF_NUMA_NODE)
2477                 pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node);
2478
2479         if (def->parts & MAP_DEF_INNER_MAP)
2480                 pr_debug("map '%s': found inner map definition.\n", map->name);
2481 }
2482
2483 static const char *btf_var_linkage_str(__u32 linkage)
2484 {
2485         switch (linkage) {
2486         case BTF_VAR_STATIC: return "static";
2487         case BTF_VAR_GLOBAL_ALLOCATED: return "global";
2488         case BTF_VAR_GLOBAL_EXTERN: return "extern";
2489         default: return "unknown";
2490         }
2491 }
2492
2493 static int bpf_object__init_user_btf_map(struct bpf_object *obj,
2494                                          const struct btf_type *sec,
2495                                          int var_idx, int sec_idx,
2496                                          const Elf_Data *data, bool strict,
2497                                          const char *pin_root_path)
2498 {
2499         struct btf_map_def map_def = {}, inner_def = {};
2500         const struct btf_type *var, *def;
2501         const struct btf_var_secinfo *vi;
2502         const struct btf_var *var_extra;
2503         const char *map_name;
2504         struct bpf_map *map;
2505         int err;
2506
2507         vi = btf_var_secinfos(sec) + var_idx;
2508         var = btf__type_by_id(obj->btf, vi->type);
2509         var_extra = btf_var(var);
2510         map_name = btf__name_by_offset(obj->btf, var->name_off);
2511
2512         if (map_name == NULL || map_name[0] == '\0') {
2513                 pr_warn("map #%d: empty name.\n", var_idx);
2514                 return -EINVAL;
2515         }
2516         if ((__u64)vi->offset + vi->size > data->d_size) {
2517                 pr_warn("map '%s' BTF data is corrupted.\n", map_name);
2518                 return -EINVAL;
2519         }
2520         if (!btf_is_var(var)) {
2521                 pr_warn("map '%s': unexpected var kind %s.\n",
2522                         map_name, btf_kind_str(var));
2523                 return -EINVAL;
2524         }
2525         if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) {
2526                 pr_warn("map '%s': unsupported map linkage %s.\n",
2527                         map_name, btf_var_linkage_str(var_extra->linkage));
2528                 return -EOPNOTSUPP;
2529         }
2530
2531         def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
2532         if (!btf_is_struct(def)) {
2533                 pr_warn("map '%s': unexpected def kind %s.\n",
2534                         map_name, btf_kind_str(var));
2535                 return -EINVAL;
2536         }
2537         if (def->size > vi->size) {
2538                 pr_warn("map '%s': invalid def size.\n", map_name);
2539                 return -EINVAL;
2540         }
2541
2542         map = bpf_object__add_map(obj);
2543         if (IS_ERR(map))
2544                 return PTR_ERR(map);
2545         map->name = strdup(map_name);
2546         if (!map->name) {
2547                 pr_warn("map '%s': failed to alloc map name.\n", map_name);
2548                 return -ENOMEM;
2549         }
2550         map->libbpf_type = LIBBPF_MAP_UNSPEC;
2551         map->def.type = BPF_MAP_TYPE_UNSPEC;
2552         map->sec_idx = sec_idx;
2553         map->sec_offset = vi->offset;
2554         map->btf_var_idx = var_idx;
2555         pr_debug("map '%s': at sec_idx %d, offset %zu.\n",
2556                  map_name, map->sec_idx, map->sec_offset);
2557
2558         err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def);
2559         if (err)
2560                 return err;
2561
2562         fill_map_from_def(map, &map_def);
2563
2564         if (map_def.pinning == LIBBPF_PIN_BY_NAME) {
2565                 err = build_map_pin_path(map, pin_root_path);
2566                 if (err) {
2567                         pr_warn("map '%s': couldn't build pin path.\n", map->name);
2568                         return err;
2569                 }
2570         }
2571
2572         if (map_def.parts & MAP_DEF_INNER_MAP) {
2573                 map->inner_map = calloc(1, sizeof(*map->inner_map));
2574                 if (!map->inner_map)
2575                         return -ENOMEM;
2576                 map->inner_map->fd = -1;
2577                 map->inner_map->sec_idx = sec_idx;
2578                 map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1);
2579                 if (!map->inner_map->name)
2580                         return -ENOMEM;
2581                 sprintf(map->inner_map->name, "%s.inner", map_name);
2582
2583                 fill_map_from_def(map->inner_map, &inner_def);
2584         }
2585
2586         err = map_fill_btf_type_info(obj, map);
2587         if (err)
2588                 return err;
2589
2590         return 0;
2591 }
2592
2593 static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict,
2594                                           const char *pin_root_path)
2595 {
2596         const struct btf_type *sec = NULL;
2597         int nr_types, i, vlen, err;
2598         const struct btf_type *t;
2599         const char *name;
2600         Elf_Data *data;
2601         Elf_Scn *scn;
2602
2603         if (obj->efile.btf_maps_shndx < 0)
2604                 return 0;
2605
2606         scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx);
2607         data = elf_sec_data(obj, scn);
2608         if (!scn || !data) {
2609                 pr_warn("elf: failed to get %s map definitions for %s\n",
2610                         MAPS_ELF_SEC, obj->path);
2611                 return -EINVAL;
2612         }
2613
2614         nr_types = btf__type_cnt(obj->btf);
2615         for (i = 1; i < nr_types; i++) {
2616                 t = btf__type_by_id(obj->btf, i);
2617                 if (!btf_is_datasec(t))
2618                         continue;
2619                 name = btf__name_by_offset(obj->btf, t->name_off);
2620                 if (strcmp(name, MAPS_ELF_SEC) == 0) {
2621                         sec = t;
2622                         obj->efile.btf_maps_sec_btf_id = i;
2623                         break;
2624                 }
2625         }
2626
2627         if (!sec) {
2628                 pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC);
2629                 return -ENOENT;
2630         }
2631
2632         vlen = btf_vlen(sec);
2633         for (i = 0; i < vlen; i++) {
2634                 err = bpf_object__init_user_btf_map(obj, sec, i,
2635                                                     obj->efile.btf_maps_shndx,
2636                                                     data, strict,
2637                                                     pin_root_path);
2638                 if (err)
2639                         return err;
2640         }
2641
2642         return 0;
2643 }
2644
2645 static int bpf_object__init_maps(struct bpf_object *obj,
2646                                  const struct bpf_object_open_opts *opts)
2647 {
2648         const char *pin_root_path;
2649         bool strict;
2650         int err = 0;
2651
2652         strict = !OPTS_GET(opts, relaxed_maps, false);
2653         pin_root_path = OPTS_GET(opts, pin_root_path, NULL);
2654
2655         err = err ?: bpf_object__init_user_btf_maps(obj, strict, pin_root_path);
2656         err = err ?: bpf_object__init_global_data_maps(obj);
2657         err = err ?: bpf_object__init_kconfig_map(obj);
2658         err = err ?: bpf_object__init_struct_ops_maps(obj);
2659
2660         return err;
2661 }
2662
2663 static bool section_have_execinstr(struct bpf_object *obj, int idx)
2664 {
2665         Elf64_Shdr *sh;
2666
2667         sh = elf_sec_hdr(obj, elf_sec_by_idx(obj, idx));
2668         if (!sh)
2669                 return false;
2670
2671         return sh->sh_flags & SHF_EXECINSTR;
2672 }
2673
2674 static bool btf_needs_sanitization(struct bpf_object *obj)
2675 {
2676         bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);
2677         bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);
2678         bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);
2679         bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
2680         bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);
2681         bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);
2682         bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);
2683
2684         return !has_func || !has_datasec || !has_func_global || !has_float ||
2685                !has_decl_tag || !has_type_tag || !has_enum64;
2686 }
2687
2688 static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)
2689 {
2690         bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);
2691         bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);
2692         bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);
2693         bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
2694         bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);
2695         bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);
2696         bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);
2697         int enum64_placeholder_id = 0;
2698         struct btf_type *t;
2699         int i, j, vlen;
2700
2701         for (i = 1; i < btf__type_cnt(btf); i++) {
2702                 t = (struct btf_type *)btf__type_by_id(btf, i);
2703
2704                 if ((!has_datasec && btf_is_var(t)) || (!has_decl_tag && btf_is_decl_tag(t))) {
2705                         /* replace VAR/DECL_TAG with INT */
2706                         t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
2707                         /*
2708                          * using size = 1 is the safest choice, 4 will be too
2709                          * big and cause kernel BTF validation failure if
2710                          * original variable took less than 4 bytes
2711                          */
2712                         t->size = 1;
2713                         *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8);
2714                 } else if (!has_datasec && btf_is_datasec(t)) {
2715                         /* replace DATASEC with STRUCT */
2716                         const struct btf_var_secinfo *v = btf_var_secinfos(t);
2717                         struct btf_member *m = btf_members(t);
2718                         struct btf_type *vt;
2719                         char *name;
2720
2721                         name = (char *)btf__name_by_offset(btf, t->name_off);
2722                         while (*name) {
2723                                 if (*name == '.')
2724                                         *name = '_';
2725                                 name++;
2726                         }
2727
2728                         vlen = btf_vlen(t);
2729                         t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);
2730                         for (j = 0; j < vlen; j++, v++, m++) {
2731                                 /* order of field assignments is important */
2732                                 m->offset = v->offset * 8;
2733                                 m->type = v->type;
2734                                 /* preserve variable name as member name */
2735                                 vt = (void *)btf__type_by_id(btf, v->type);
2736                                 m->name_off = vt->name_off;
2737                         }
2738                 } else if (!has_func && btf_is_func_proto(t)) {
2739                         /* replace FUNC_PROTO with ENUM */
2740                         vlen = btf_vlen(t);
2741                         t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);
2742                         t->size = sizeof(__u32); /* kernel enforced */
2743                 } else if (!has_func && btf_is_func(t)) {
2744                         /* replace FUNC with TYPEDEF */
2745                         t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);
2746                 } else if (!has_func_global && btf_is_func(t)) {
2747                         /* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */
2748                         t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0);
2749                 } else if (!has_float && btf_is_float(t)) {
2750                         /* replace FLOAT with an equally-sized empty STRUCT;
2751                          * since C compilers do not accept e.g. "float" as a
2752                          * valid struct name, make it anonymous
2753                          */
2754                         t->name_off = 0;
2755                         t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0);
2756                 } else if (!has_type_tag && btf_is_type_tag(t)) {
2757                         /* replace TYPE_TAG with a CONST */
2758                         t->name_off = 0;
2759                         t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0);
2760                 } else if (!has_enum64 && btf_is_enum(t)) {
2761                         /* clear the kflag */
2762                         t->info = btf_type_info(btf_kind(t), btf_vlen(t), false);
2763                 } else if (!has_enum64 && btf_is_enum64(t)) {
2764                         /* replace ENUM64 with a union */
2765                         struct btf_member *m;
2766
2767                         if (enum64_placeholder_id == 0) {
2768                                 enum64_placeholder_id = btf__add_int(btf, "enum64_placeholder", 1, 0);
2769                                 if (enum64_placeholder_id < 0)
2770                                         return enum64_placeholder_id;
2771
2772                                 t = (struct btf_type *)btf__type_by_id(btf, i);
2773                         }
2774
2775                         m = btf_members(t);
2776                         vlen = btf_vlen(t);
2777                         t->info = BTF_INFO_ENC(BTF_KIND_UNION, 0, vlen);
2778                         for (j = 0; j < vlen; j++, m++) {
2779                                 m->type = enum64_placeholder_id;
2780                                 m->offset = 0;
2781                         }
2782                 }
2783         }
2784
2785         return 0;
2786 }
2787
2788 static bool libbpf_needs_btf(const struct bpf_object *obj)
2789 {
2790         return obj->efile.btf_maps_shndx >= 0 ||
2791                obj->efile.st_ops_shndx >= 0 ||
2792                obj->nr_extern > 0;
2793 }
2794
2795 static bool kernel_needs_btf(const struct bpf_object *obj)
2796 {
2797         return obj->efile.st_ops_shndx >= 0;
2798 }
2799
2800 static int bpf_object__init_btf(struct bpf_object *obj,
2801                                 Elf_Data *btf_data,
2802                                 Elf_Data *btf_ext_data)
2803 {
2804         int err = -ENOENT;
2805
2806         if (btf_data) {
2807                 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
2808                 err = libbpf_get_error(obj->btf);
2809                 if (err) {
2810                         obj->btf = NULL;
2811                         pr_warn("Error loading ELF section %s: %d.\n", BTF_ELF_SEC, err);
2812                         goto out;
2813                 }
2814                 /* enforce 8-byte pointers for BPF-targeted BTFs */
2815                 btf__set_pointer_size(obj->btf, 8);
2816         }
2817         if (btf_ext_data) {
2818                 struct btf_ext_info *ext_segs[3];
2819                 int seg_num, sec_num;
2820
2821                 if (!obj->btf) {
2822                         pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",
2823                                  BTF_EXT_ELF_SEC, BTF_ELF_SEC);
2824                         goto out;
2825                 }
2826                 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size);
2827                 err = libbpf_get_error(obj->btf_ext);
2828                 if (err) {
2829                         pr_warn("Error loading ELF section %s: %d. Ignored and continue.\n",
2830                                 BTF_EXT_ELF_SEC, err);
2831                         obj->btf_ext = NULL;
2832                         goto out;
2833                 }
2834
2835                 /* setup .BTF.ext to ELF section mapping */
2836                 ext_segs[0] = &obj->btf_ext->func_info;
2837                 ext_segs[1] = &obj->btf_ext->line_info;
2838                 ext_segs[2] = &obj->btf_ext->core_relo_info;
2839                 for (seg_num = 0; seg_num < ARRAY_SIZE(ext_segs); seg_num++) {
2840                         struct btf_ext_info *seg = ext_segs[seg_num];
2841                         const struct btf_ext_info_sec *sec;
2842                         const char *sec_name;
2843                         Elf_Scn *scn;
2844
2845                         if (seg->sec_cnt == 0)
2846                                 continue;
2847
2848                         seg->sec_idxs = calloc(seg->sec_cnt, sizeof(*seg->sec_idxs));
2849                         if (!seg->sec_idxs) {
2850                                 err = -ENOMEM;
2851                                 goto out;
2852                         }
2853
2854                         sec_num = 0;
2855                         for_each_btf_ext_sec(seg, sec) {
2856                                 /* preventively increment index to avoid doing
2857                                  * this before every continue below
2858                                  */
2859                                 sec_num++;
2860
2861                                 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
2862                                 if (str_is_empty(sec_name))
2863                                         continue;
2864                                 scn = elf_sec_by_name(obj, sec_name);
2865                                 if (!scn)
2866                                         continue;
2867
2868                                 seg->sec_idxs[sec_num - 1] = elf_ndxscn(scn);
2869                         }
2870                 }
2871         }
2872 out:
2873         if (err && libbpf_needs_btf(obj)) {
2874                 pr_warn("BTF is required, but is missing or corrupted.\n");
2875                 return err;
2876         }
2877         return 0;
2878 }
2879
2880 static int compare_vsi_off(const void *_a, const void *_b)
2881 {
2882         const struct btf_var_secinfo *a = _a;
2883         const struct btf_var_secinfo *b = _b;
2884
2885         return a->offset - b->offset;
2886 }
2887
2888 static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf,
2889                              struct btf_type *t)
2890 {
2891         __u32 size = 0, i, vars = btf_vlen(t);
2892         const char *sec_name = btf__name_by_offset(btf, t->name_off);
2893         struct btf_var_secinfo *vsi;
2894         bool fixup_offsets = false;
2895         int err;
2896
2897         if (!sec_name) {
2898                 pr_debug("No name found in string section for DATASEC kind.\n");
2899                 return -ENOENT;
2900         }
2901
2902         /* Extern-backing datasecs (.ksyms, .kconfig) have their size and
2903          * variable offsets set at the previous step. Further, not every
2904          * extern BTF VAR has corresponding ELF symbol preserved, so we skip
2905          * all fixups altogether for such sections and go straight to sorting
2906          * VARs within their DATASEC.
2907          */
2908         if (strcmp(sec_name, KCONFIG_SEC) == 0 || strcmp(sec_name, KSYMS_SEC) == 0)
2909                 goto sort_vars;
2910
2911         /* Clang leaves DATASEC size and VAR offsets as zeroes, so we need to
2912          * fix this up. But BPF static linker already fixes this up and fills
2913          * all the sizes and offsets during static linking. So this step has
2914          * to be optional. But the STV_HIDDEN handling is non-optional for any
2915          * non-extern DATASEC, so the variable fixup loop below handles both
2916          * functions at the same time, paying the cost of BTF VAR <-> ELF
2917          * symbol matching just once.
2918          */
2919         if (t->size == 0) {
2920                 err = find_elf_sec_sz(obj, sec_name, &size);
2921                 if (err || !size) {
2922                         pr_debug("sec '%s': failed to determine size from ELF: size %u, err %d\n",
2923                                  sec_name, size, err);
2924                         return -ENOENT;
2925                 }
2926
2927                 t->size = size;
2928                 fixup_offsets = true;
2929         }
2930
2931         for (i = 0, vsi = btf_var_secinfos(t); i < vars; i++, vsi++) {
2932                 const struct btf_type *t_var;
2933                 struct btf_var *var;
2934                 const char *var_name;
2935                 Elf64_Sym *sym;
2936
2937                 t_var = btf__type_by_id(btf, vsi->type);
2938                 if (!t_var || !btf_is_var(t_var)) {
2939                         pr_debug("sec '%s': unexpected non-VAR type found\n", sec_name);
2940                         return -EINVAL;
2941                 }
2942
2943                 var = btf_var(t_var);
2944                 if (var->linkage == BTF_VAR_STATIC || var->linkage == BTF_VAR_GLOBAL_EXTERN)
2945                         continue;
2946
2947                 var_name = btf__name_by_offset(btf, t_var->name_off);
2948                 if (!var_name) {
2949                         pr_debug("sec '%s': failed to find name of DATASEC's member #%d\n",
2950                                  sec_name, i);
2951                         return -ENOENT;
2952                 }
2953
2954                 sym = find_elf_var_sym(obj, var_name);
2955                 if (IS_ERR(sym)) {
2956                         pr_debug("sec '%s': failed to find ELF symbol for VAR '%s'\n",
2957                                  sec_name, var_name);
2958                         return -ENOENT;
2959                 }
2960
2961                 if (fixup_offsets)
2962                         vsi->offset = sym->st_value;
2963
2964                 /* if variable is a global/weak symbol, but has restricted
2965                  * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF VAR
2966                  * as static. This follows similar logic for functions (BPF
2967                  * subprogs) and influences libbpf's further decisions about
2968                  * whether to make global data BPF array maps as
2969                  * BPF_F_MMAPABLE.
2970                  */
2971                 if (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN
2972                     || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL)
2973                         var->linkage = BTF_VAR_STATIC;
2974         }
2975
2976 sort_vars:
2977         qsort(btf_var_secinfos(t), vars, sizeof(*vsi), compare_vsi_off);
2978         return 0;
2979 }
2980
2981 static int bpf_object_fixup_btf(struct bpf_object *obj)
2982 {
2983         int i, n, err = 0;
2984
2985         if (!obj->btf)
2986                 return 0;
2987
2988         n = btf__type_cnt(obj->btf);
2989         for (i = 1; i < n; i++) {
2990                 struct btf_type *t = btf_type_by_id(obj->btf, i);
2991
2992                 /* Loader needs to fix up some of the things compiler
2993                  * couldn't get its hands on while emitting BTF. This
2994                  * is section size and global variable offset. We use
2995                  * the info from the ELF itself for this purpose.
2996                  */
2997                 if (btf_is_datasec(t)) {
2998                         err = btf_fixup_datasec(obj, obj->btf, t);
2999                         if (err)
3000                                 return err;
3001                 }
3002         }
3003
3004         return 0;
3005 }
3006
3007 static bool prog_needs_vmlinux_btf(struct bpf_program *prog)
3008 {
3009         if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
3010             prog->type == BPF_PROG_TYPE_LSM)
3011                 return true;
3012
3013         /* BPF_PROG_TYPE_TRACING programs which do not attach to other programs
3014          * also need vmlinux BTF
3015          */
3016         if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
3017                 return true;
3018
3019         return false;
3020 }
3021
3022 static bool obj_needs_vmlinux_btf(const struct bpf_object *obj)
3023 {
3024         struct bpf_program *prog;
3025         int i;
3026
3027         /* CO-RE relocations need kernel BTF, only when btf_custom_path
3028          * is not specified
3029          */
3030         if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path)
3031                 return true;
3032
3033         /* Support for typed ksyms needs kernel BTF */
3034         for (i = 0; i < obj->nr_extern; i++) {
3035                 const struct extern_desc *ext;
3036
3037                 ext = &obj->externs[i];
3038                 if (ext->type == EXT_KSYM && ext->ksym.type_id)
3039                         return true;
3040         }
3041
3042         bpf_object__for_each_program(prog, obj) {
3043                 if (!prog->autoload)
3044                         continue;
3045                 if (prog_needs_vmlinux_btf(prog))
3046                         return true;
3047         }
3048
3049         return false;
3050 }
3051
3052 static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force)
3053 {
3054         int err;
3055
3056         /* btf_vmlinux could be loaded earlier */
3057         if (obj->btf_vmlinux || obj->gen_loader)
3058                 return 0;
3059
3060         if (!force && !obj_needs_vmlinux_btf(obj))
3061                 return 0;
3062
3063         obj->btf_vmlinux = btf__load_vmlinux_btf();
3064         err = libbpf_get_error(obj->btf_vmlinux);
3065         if (err) {
3066                 pr_warn("Error loading vmlinux BTF: %d\n", err);
3067                 obj->btf_vmlinux = NULL;
3068                 return err;
3069         }
3070         return 0;
3071 }
3072
3073 static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj)
3074 {
3075         struct btf *kern_btf = obj->btf;
3076         bool btf_mandatory, sanitize;
3077         int i, err = 0;
3078
3079         if (!obj->btf)
3080                 return 0;
3081
3082         if (!kernel_supports(obj, FEAT_BTF)) {
3083                 if (kernel_needs_btf(obj)) {
3084                         err = -EOPNOTSUPP;
3085                         goto report;
3086                 }
3087                 pr_debug("Kernel doesn't support BTF, skipping uploading it.\n");
3088                 return 0;
3089         }
3090
3091         /* Even though some subprogs are global/weak, user might prefer more
3092          * permissive BPF verification process that BPF verifier performs for
3093          * static functions, taking into account more context from the caller
3094          * functions. In such case, they need to mark such subprogs with
3095          * __attribute__((visibility("hidden"))) and libbpf will adjust
3096          * corresponding FUNC BTF type to be marked as static and trigger more
3097          * involved BPF verification process.
3098          */
3099         for (i = 0; i < obj->nr_programs; i++) {
3100                 struct bpf_program *prog = &obj->programs[i];
3101                 struct btf_type *t;
3102                 const char *name;
3103                 int j, n;
3104
3105                 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog))
3106                         continue;
3107
3108                 n = btf__type_cnt(obj->btf);
3109                 for (j = 1; j < n; j++) {
3110                         t = btf_type_by_id(obj->btf, j);
3111                         if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL)
3112                                 continue;
3113
3114                         name = btf__str_by_offset(obj->btf, t->name_off);
3115                         if (strcmp(name, prog->name) != 0)
3116                                 continue;
3117
3118                         t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0);
3119                         break;
3120                 }
3121         }
3122
3123         sanitize = btf_needs_sanitization(obj);
3124         if (sanitize) {
3125                 const void *raw_data;
3126                 __u32 sz;
3127
3128                 /* clone BTF to sanitize a copy and leave the original intact */
3129                 raw_data = btf__raw_data(obj->btf, &sz);
3130                 kern_btf = btf__new(raw_data, sz);
3131                 err = libbpf_get_error(kern_btf);
3132                 if (err)
3133                         return err;
3134
3135                 /* enforce 8-byte pointers for BPF-targeted BTFs */
3136                 btf__set_pointer_size(obj->btf, 8);
3137                 err = bpf_object__sanitize_btf(obj, kern_btf);
3138                 if (err)
3139                         return err;
3140         }
3141
3142         if (obj->gen_loader) {
3143                 __u32 raw_size = 0;
3144                 const void *raw_data = btf__raw_data(kern_btf, &raw_size);
3145
3146                 if (!raw_data)
3147                         return -ENOMEM;
3148                 bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size);
3149                 /* Pretend to have valid FD to pass various fd >= 0 checks.
3150                  * This fd == 0 will not be used with any syscall and will be reset to -1 eventually.
3151                  */
3152                 btf__set_fd(kern_btf, 0);
3153         } else {
3154                 /* currently BPF_BTF_LOAD only supports log_level 1 */
3155                 err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size,
3156                                            obj->log_level ? 1 : 0);
3157         }
3158         if (sanitize) {
3159                 if (!err) {
3160                         /* move fd to libbpf's BTF */
3161                         btf__set_fd(obj->btf, btf__fd(kern_btf));
3162                         btf__set_fd(kern_btf, -1);
3163                 }
3164                 btf__free(kern_btf);
3165         }
3166 report:
3167         if (err) {
3168                 btf_mandatory = kernel_needs_btf(obj);
3169                 pr_warn("Error loading .BTF into kernel: %d. %s\n", err,
3170                         btf_mandatory ? "BTF is mandatory, can't proceed."
3171                                       : "BTF is optional, ignoring.");
3172                 if (!btf_mandatory)
3173                         err = 0;
3174         }
3175         return err;
3176 }
3177
3178 static const char *elf_sym_str(const struct bpf_object *obj, size_t off)
3179 {
3180         const char *name;
3181
3182         name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off);
3183         if (!name) {
3184                 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
3185                         off, obj->path, elf_errmsg(-1));
3186                 return NULL;
3187         }
3188
3189         return name;
3190 }
3191
3192 static const char *elf_sec_str(const struct bpf_object *obj, size_t off)
3193 {
3194         const char *name;
3195
3196         name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off);
3197         if (!name) {
3198                 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
3199                         off, obj->path, elf_errmsg(-1));
3200                 return NULL;
3201         }
3202
3203         return name;
3204 }
3205
3206 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx)
3207 {
3208         Elf_Scn *scn;
3209
3210         scn = elf_getscn(obj->efile.elf, idx);
3211         if (!scn) {
3212                 pr_warn("elf: failed to get section(%zu) from %s: %s\n",
3213                         idx, obj->path, elf_errmsg(-1));
3214                 return NULL;
3215         }
3216         return scn;
3217 }
3218
3219 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name)
3220 {
3221         Elf_Scn *scn = NULL;
3222         Elf *elf = obj->efile.elf;
3223         const char *sec_name;
3224
3225         while ((scn = elf_nextscn(elf, scn)) != NULL) {
3226                 sec_name = elf_sec_name(obj, scn);
3227                 if (!sec_name)
3228                         return NULL;
3229
3230                 if (strcmp(sec_name, name) != 0)
3231                         continue;
3232
3233                 return scn;
3234         }
3235         return NULL;
3236 }
3237
3238 static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn)
3239 {
3240         Elf64_Shdr *shdr;
3241
3242         if (!scn)
3243                 return NULL;
3244
3245         shdr = elf64_getshdr(scn);
3246         if (!shdr) {
3247                 pr_warn("elf: failed to get section(%zu) header from %s: %s\n",
3248                         elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3249                 return NULL;
3250         }
3251
3252         return shdr;
3253 }
3254
3255 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn)
3256 {
3257         const char *name;
3258         Elf64_Shdr *sh;
3259
3260         if (!scn)
3261                 return NULL;
3262
3263         sh = elf_sec_hdr(obj, scn);
3264         if (!sh)
3265                 return NULL;
3266
3267         name = elf_sec_str(obj, sh->sh_name);
3268         if (!name) {
3269                 pr_warn("elf: failed to get section(%zu) name from %s: %s\n",
3270                         elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3271                 return NULL;
3272         }
3273
3274         return name;
3275 }
3276
3277 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn)
3278 {
3279         Elf_Data *data;
3280
3281         if (!scn)
3282                 return NULL;
3283
3284         data = elf_getdata(scn, 0);
3285         if (!data) {
3286                 pr_warn("elf: failed to get section(%zu) %s data from %s: %s\n",
3287                         elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "<?>",
3288                         obj->path, elf_errmsg(-1));
3289                 return NULL;
3290         }
3291
3292         return data;
3293 }
3294
3295 static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx)
3296 {
3297         if (idx >= obj->efile.symbols->d_size / sizeof(Elf64_Sym))
3298                 return NULL;
3299
3300         return (Elf64_Sym *)obj->efile.symbols->d_buf + idx;
3301 }
3302
3303 static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx)
3304 {
3305         if (idx >= data->d_size / sizeof(Elf64_Rel))
3306                 return NULL;
3307
3308         return (Elf64_Rel *)data->d_buf + idx;
3309 }
3310
3311 static bool is_sec_name_dwarf(const char *name)
3312 {
3313         /* approximation, but the actual list is too long */
3314         return str_has_pfx(name, ".debug_");
3315 }
3316
3317 static bool ignore_elf_section(Elf64_Shdr *hdr, const char *name)
3318 {
3319         /* no special handling of .strtab */
3320         if (hdr->sh_type == SHT_STRTAB)
3321                 return true;
3322
3323         /* ignore .llvm_addrsig section as well */
3324         if (hdr->sh_type == SHT_LLVM_ADDRSIG)
3325                 return true;
3326
3327         /* no subprograms will lead to an empty .text section, ignore it */
3328         if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 &&
3329             strcmp(name, ".text") == 0)
3330                 return true;
3331
3332         /* DWARF sections */
3333         if (is_sec_name_dwarf(name))
3334                 return true;
3335
3336         if (str_has_pfx(name, ".rel")) {
3337                 name += sizeof(".rel") - 1;
3338                 /* DWARF section relocations */
3339                 if (is_sec_name_dwarf(name))
3340                         return true;
3341
3342                 /* .BTF and .BTF.ext don't need relocations */
3343                 if (strcmp(name, BTF_ELF_SEC) == 0 ||
3344                     strcmp(name, BTF_EXT_ELF_SEC) == 0)
3345                         return true;
3346         }
3347
3348         return false;
3349 }
3350
3351 static int cmp_progs(const void *_a, const void *_b)
3352 {
3353         const struct bpf_program *a = _a;
3354         const struct bpf_program *b = _b;
3355
3356         if (a->sec_idx != b->sec_idx)
3357                 return a->sec_idx < b->sec_idx ? -1 : 1;
3358
3359         /* sec_insn_off can't be the same within the section */
3360         return a->sec_insn_off < b->sec_insn_off ? -1 : 1;
3361 }
3362
3363 static int bpf_object__elf_collect(struct bpf_object *obj)
3364 {
3365         struct elf_sec_desc *sec_desc;
3366         Elf *elf = obj->efile.elf;
3367         Elf_Data *btf_ext_data = NULL;
3368         Elf_Data *btf_data = NULL;
3369         int idx = 0, err = 0;
3370         const char *name;
3371         Elf_Data *data;
3372         Elf_Scn *scn;
3373         Elf64_Shdr *sh;
3374
3375         /* ELF section indices are 0-based, but sec #0 is special "invalid"
3376          * section. Since section count retrieved by elf_getshdrnum() does
3377          * include sec #0, it is already the necessary size of an array to keep
3378          * all the sections.
3379          */
3380         if (elf_getshdrnum(obj->efile.elf, &obj->efile.sec_cnt)) {
3381                 pr_warn("elf: failed to get the number of sections for %s: %s\n",
3382                         obj->path, elf_errmsg(-1));
3383                 return -LIBBPF_ERRNO__FORMAT;
3384         }
3385         obj->efile.secs = calloc(obj->efile.sec_cnt, sizeof(*obj->efile.secs));
3386         if (!obj->efile.secs)
3387                 return -ENOMEM;
3388
3389         /* a bunch of ELF parsing functionality depends on processing symbols,
3390          * so do the first pass and find the symbol table
3391          */
3392         scn = NULL;
3393         while ((scn = elf_nextscn(elf, scn)) != NULL) {
3394                 sh = elf_sec_hdr(obj, scn);
3395                 if (!sh)
3396                         return -LIBBPF_ERRNO__FORMAT;
3397
3398                 if (sh->sh_type == SHT_SYMTAB) {
3399                         if (obj->efile.symbols) {
3400                                 pr_warn("elf: multiple symbol tables in %s\n", obj->path);
3401                                 return -LIBBPF_ERRNO__FORMAT;
3402                         }
3403
3404                         data = elf_sec_data(obj, scn);
3405                         if (!data)
3406                                 return -LIBBPF_ERRNO__FORMAT;
3407
3408                         idx = elf_ndxscn(scn);
3409
3410                         obj->efile.symbols = data;
3411                         obj->efile.symbols_shndx = idx;
3412                         obj->efile.strtabidx = sh->sh_link;
3413                 }
3414         }
3415
3416         if (!obj->efile.symbols) {
3417                 pr_warn("elf: couldn't find symbol table in %s, stripped object file?\n",
3418                         obj->path);
3419                 return -ENOENT;
3420         }
3421
3422         scn = NULL;
3423         while ((scn = elf_nextscn(elf, scn)) != NULL) {
3424                 idx = elf_ndxscn(scn);
3425                 sec_desc = &obj->efile.secs[idx];
3426
3427                 sh = elf_sec_hdr(obj, scn);
3428                 if (!sh)
3429                         return -LIBBPF_ERRNO__FORMAT;
3430
3431                 name = elf_sec_str(obj, sh->sh_name);
3432                 if (!name)
3433                         return -LIBBPF_ERRNO__FORMAT;
3434
3435                 if (ignore_elf_section(sh, name))
3436                         continue;
3437
3438                 data = elf_sec_data(obj, scn);
3439                 if (!data)
3440                         return -LIBBPF_ERRNO__FORMAT;
3441
3442                 pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
3443                          idx, name, (unsigned long)data->d_size,
3444                          (int)sh->sh_link, (unsigned long)sh->sh_flags,
3445                          (int)sh->sh_type);
3446
3447                 if (strcmp(name, "license") == 0) {
3448                         err = bpf_object__init_license(obj, data->d_buf, data->d_size);
3449                         if (err)
3450                                 return err;
3451                 } else if (strcmp(name, "version") == 0) {
3452                         err = bpf_object__init_kversion(obj, data->d_buf, data->d_size);
3453                         if (err)
3454                                 return err;
3455                 } else if (strcmp(name, "maps") == 0) {
3456                         pr_warn("elf: legacy map definitions in 'maps' section are not supported by libbpf v1.0+\n");
3457                         return -ENOTSUP;
3458                 } else if (strcmp(name, MAPS_ELF_SEC) == 0) {
3459                         obj->efile.btf_maps_shndx = idx;
3460                 } else if (strcmp(name, BTF_ELF_SEC) == 0) {
3461                         if (sh->sh_type != SHT_PROGBITS)
3462                                 return -LIBBPF_ERRNO__FORMAT;
3463                         btf_data = data;
3464                 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) {
3465                         if (sh->sh_type != SHT_PROGBITS)
3466                                 return -LIBBPF_ERRNO__FORMAT;
3467                         btf_ext_data = data;
3468                 } else if (sh->sh_type == SHT_SYMTAB) {
3469                         /* already processed during the first pass above */
3470                 } else if (sh->sh_type == SHT_PROGBITS && data->d_size > 0) {
3471                         if (sh->sh_flags & SHF_EXECINSTR) {
3472                                 if (strcmp(name, ".text") == 0)
3473                                         obj->efile.text_shndx = idx;
3474                                 err = bpf_object__add_programs(obj, data, name, idx);
3475                                 if (err)
3476                                         return err;
3477                         } else if (strcmp(name, DATA_SEC) == 0 ||
3478                                    str_has_pfx(name, DATA_SEC ".")) {
3479                                 sec_desc->sec_type = SEC_DATA;
3480                                 sec_desc->shdr = sh;
3481                                 sec_desc->data = data;
3482                         } else if (strcmp(name, RODATA_SEC) == 0 ||
3483                                    str_has_pfx(name, RODATA_SEC ".")) {
3484                                 sec_desc->sec_type = SEC_RODATA;
3485                                 sec_desc->shdr = sh;
3486                                 sec_desc->data = data;
3487                         } else if (strcmp(name, STRUCT_OPS_SEC) == 0) {
3488                                 obj->efile.st_ops_data = data;
3489                                 obj->efile.st_ops_shndx = idx;
3490                         } else {
3491                                 pr_info("elf: skipping unrecognized data section(%d) %s\n",
3492                                         idx, name);
3493                         }
3494                 } else if (sh->sh_type == SHT_REL) {
3495                         int targ_sec_idx = sh->sh_info; /* points to other section */
3496
3497                         if (sh->sh_entsize != sizeof(Elf64_Rel) ||
3498                             targ_sec_idx >= obj->efile.sec_cnt)
3499                                 return -LIBBPF_ERRNO__FORMAT;
3500
3501                         /* Only do relo for section with exec instructions */
3502                         if (!section_have_execinstr(obj, targ_sec_idx) &&
3503                             strcmp(name, ".rel" STRUCT_OPS_SEC) &&
3504                             strcmp(name, ".rel" MAPS_ELF_SEC)) {
3505                                 pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n",
3506                                         idx, name, targ_sec_idx,
3507                                         elf_sec_name(obj, elf_sec_by_idx(obj, targ_sec_idx)) ?: "<?>");
3508                                 continue;
3509                         }
3510
3511                         sec_desc->sec_type = SEC_RELO;
3512                         sec_desc->shdr = sh;
3513                         sec_desc->data = data;
3514                 } else if (sh->sh_type == SHT_NOBITS && (strcmp(name, BSS_SEC) == 0 ||
3515                                                          str_has_pfx(name, BSS_SEC "."))) {
3516                         sec_desc->sec_type = SEC_BSS;
3517                         sec_desc->shdr = sh;
3518                         sec_desc->data = data;
3519                 } else {
3520                         pr_info("elf: skipping section(%d) %s (size %zu)\n", idx, name,
3521                                 (size_t)sh->sh_size);
3522                 }
3523         }
3524
3525         if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {
3526                 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path);
3527                 return -LIBBPF_ERRNO__FORMAT;
3528         }
3529
3530         /* sort BPF programs by section name and in-section instruction offset
3531          * for faster search
3532          */
3533         if (obj->nr_programs)
3534                 qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs);
3535
3536         return bpf_object__init_btf(obj, btf_data, btf_ext_data);
3537 }
3538
3539 static bool sym_is_extern(const Elf64_Sym *sym)
3540 {
3541         int bind = ELF64_ST_BIND(sym->st_info);
3542         /* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */
3543         return sym->st_shndx == SHN_UNDEF &&
3544                (bind == STB_GLOBAL || bind == STB_WEAK) &&
3545                ELF64_ST_TYPE(sym->st_info) == STT_NOTYPE;
3546 }
3547
3548 static bool sym_is_subprog(const Elf64_Sym *sym, int text_shndx)
3549 {
3550         int bind = ELF64_ST_BIND(sym->st_info);
3551         int type = ELF64_ST_TYPE(sym->st_info);
3552
3553         /* in .text section */
3554         if (sym->st_shndx != text_shndx)
3555                 return false;
3556
3557         /* local function */
3558         if (bind == STB_LOCAL && type == STT_SECTION)
3559                 return true;
3560
3561         /* global function */
3562         return bind == STB_GLOBAL && type == STT_FUNC;
3563 }
3564
3565 static int find_extern_btf_id(const struct btf *btf, const char *ext_name)
3566 {
3567         const struct btf_type *t;
3568         const char *tname;
3569         int i, n;
3570
3571         if (!btf)
3572                 return -ESRCH;
3573
3574         n = btf__type_cnt(btf);
3575         for (i = 1; i < n; i++) {
3576                 t = btf__type_by_id(btf, i);
3577
3578                 if (!btf_is_var(t) && !btf_is_func(t))
3579                         continue;
3580
3581                 tname = btf__name_by_offset(btf, t->name_off);
3582                 if (strcmp(tname, ext_name))
3583                         continue;
3584
3585                 if (btf_is_var(t) &&
3586                     btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
3587                         return -EINVAL;
3588
3589                 if (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_EXTERN)
3590                         return -EINVAL;
3591
3592                 return i;
3593         }
3594
3595         return -ENOENT;
3596 }
3597
3598 static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) {
3599         const struct btf_var_secinfo *vs;
3600         const struct btf_type *t;
3601         int i, j, n;
3602
3603         if (!btf)
3604                 return -ESRCH;
3605
3606         n = btf__type_cnt(btf);
3607         for (i = 1; i < n; i++) {
3608                 t = btf__type_by_id(btf, i);
3609
3610                 if (!btf_is_datasec(t))
3611                         continue;
3612
3613                 vs = btf_var_secinfos(t);
3614                 for (j = 0; j < btf_vlen(t); j++, vs++) {
3615                         if (vs->type == ext_btf_id)
3616                                 return i;
3617                 }
3618         }
3619
3620         return -ENOENT;
3621 }
3622
3623 static enum kcfg_type find_kcfg_type(const struct btf *btf, int id,
3624                                      bool *is_signed)
3625 {
3626         const struct btf_type *t;
3627         const char *name;
3628
3629         t = skip_mods_and_typedefs(btf, id, NULL);
3630         name = btf__name_by_offset(btf, t->name_off);
3631
3632         if (is_signed)
3633                 *is_signed = false;
3634         switch (btf_kind(t)) {
3635         case BTF_KIND_INT: {
3636                 int enc = btf_int_encoding(t);
3637
3638                 if (enc & BTF_INT_BOOL)
3639                         return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN;
3640                 if (is_signed)
3641                         *is_signed = enc & BTF_INT_SIGNED;
3642                 if (t->size == 1)
3643                         return KCFG_CHAR;
3644                 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1)))
3645                         return KCFG_UNKNOWN;
3646                 return KCFG_INT;
3647         }
3648         case BTF_KIND_ENUM:
3649                 if (t->size != 4)
3650                         return KCFG_UNKNOWN;
3651                 if (strcmp(name, "libbpf_tristate"))
3652                         return KCFG_UNKNOWN;
3653                 return KCFG_TRISTATE;
3654         case BTF_KIND_ENUM64:
3655                 if (strcmp(name, "libbpf_tristate"))
3656                         return KCFG_UNKNOWN;
3657                 return KCFG_TRISTATE;
3658         case BTF_KIND_ARRAY:
3659                 if (btf_array(t)->nelems == 0)
3660                         return KCFG_UNKNOWN;
3661                 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR)
3662                         return KCFG_UNKNOWN;
3663                 return KCFG_CHAR_ARR;
3664         default:
3665                 return KCFG_UNKNOWN;
3666         }
3667 }
3668
3669 static int cmp_externs(const void *_a, const void *_b)
3670 {
3671         const struct extern_desc *a = _a;
3672         const struct extern_desc *b = _b;
3673
3674         if (a->type != b->type)
3675                 return a->type < b->type ? -1 : 1;
3676
3677         if (a->type == EXT_KCFG) {
3678                 /* descending order by alignment requirements */
3679                 if (a->kcfg.align != b->kcfg.align)
3680                         return a->kcfg.align > b->kcfg.align ? -1 : 1;
3681                 /* ascending order by size, within same alignment class */
3682                 if (a->kcfg.sz != b->kcfg.sz)
3683                         return a->kcfg.sz < b->kcfg.sz ? -1 : 1;
3684         }
3685
3686         /* resolve ties by name */
3687         return strcmp(a->name, b->name);
3688 }
3689
3690 static int find_int_btf_id(const struct btf *btf)
3691 {
3692         const struct btf_type *t;
3693         int i, n;
3694
3695         n = btf__type_cnt(btf);
3696         for (i = 1; i < n; i++) {
3697                 t = btf__type_by_id(btf, i);
3698
3699                 if (btf_is_int(t) && btf_int_bits(t) == 32)
3700                         return i;
3701         }
3702
3703         return 0;
3704 }
3705
3706 static int add_dummy_ksym_var(struct btf *btf)
3707 {
3708         int i, int_btf_id, sec_btf_id, dummy_var_btf_id;
3709         const struct btf_var_secinfo *vs;
3710         const struct btf_type *sec;
3711
3712         if (!btf)
3713                 return 0;
3714
3715         sec_btf_id = btf__find_by_name_kind(btf, KSYMS_SEC,
3716                                             BTF_KIND_DATASEC);
3717         if (sec_btf_id < 0)
3718                 return 0;
3719
3720         sec = btf__type_by_id(btf, sec_btf_id);
3721         vs = btf_var_secinfos(sec);
3722         for (i = 0; i < btf_vlen(sec); i++, vs++) {
3723                 const struct btf_type *vt;
3724
3725                 vt = btf__type_by_id(btf, vs->type);
3726                 if (btf_is_func(vt))
3727                         break;
3728         }
3729
3730         /* No func in ksyms sec.  No need to add dummy var. */
3731         if (i == btf_vlen(sec))
3732                 return 0;
3733
3734         int_btf_id = find_int_btf_id(btf);
3735         dummy_var_btf_id = btf__add_var(btf,
3736                                         "dummy_ksym",
3737                                         BTF_VAR_GLOBAL_ALLOCATED,
3738                                         int_btf_id);
3739         if (dummy_var_btf_id < 0)
3740                 pr_warn("cannot create a dummy_ksym var\n");
3741
3742         return dummy_var_btf_id;
3743 }
3744
3745 static int bpf_object__collect_externs(struct bpf_object *obj)
3746 {
3747         struct btf_type *sec, *kcfg_sec = NULL, *ksym_sec = NULL;
3748         const struct btf_type *t;
3749         struct extern_desc *ext;
3750         int i, n, off, dummy_var_btf_id;
3751         const char *ext_name, *sec_name;
3752         Elf_Scn *scn;
3753         Elf64_Shdr *sh;
3754
3755         if (!obj->efile.symbols)
3756                 return 0;
3757
3758         scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx);
3759         sh = elf_sec_hdr(obj, scn);
3760         if (!sh || sh->sh_entsize != sizeof(Elf64_Sym))
3761                 return -LIBBPF_ERRNO__FORMAT;
3762
3763         dummy_var_btf_id = add_dummy_ksym_var(obj->btf);
3764         if (dummy_var_btf_id < 0)
3765                 return dummy_var_btf_id;
3766
3767         n = sh->sh_size / sh->sh_entsize;
3768         pr_debug("looking for externs among %d symbols...\n", n);
3769
3770         for (i = 0; i < n; i++) {
3771                 Elf64_Sym *sym = elf_sym_by_idx(obj, i);
3772
3773                 if (!sym)
3774                         return -LIBBPF_ERRNO__FORMAT;
3775                 if (!sym_is_extern(sym))
3776                         continue;
3777                 ext_name = elf_sym_str(obj, sym->st_name);
3778                 if (!ext_name || !ext_name[0])
3779                         continue;
3780
3781                 ext = obj->externs;
3782                 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext));
3783                 if (!ext)
3784                         return -ENOMEM;
3785                 obj->externs = ext;
3786                 ext = &ext[obj->nr_extern];
3787                 memset(ext, 0, sizeof(*ext));
3788                 obj->nr_extern++;
3789
3790                 ext->btf_id = find_extern_btf_id(obj->btf, ext_name);
3791                 if (ext->btf_id <= 0) {
3792                         pr_warn("failed to find BTF for extern '%s': %d\n",
3793                                 ext_name, ext->btf_id);
3794                         return ext->btf_id;
3795                 }
3796                 t = btf__type_by_id(obj->btf, ext->btf_id);
3797                 ext->name = btf__name_by_offset(obj->btf, t->name_off);
3798                 ext->sym_idx = i;
3799                 ext->is_weak = ELF64_ST_BIND(sym->st_info) == STB_WEAK;
3800
3801                 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id);
3802                 if (ext->sec_btf_id <= 0) {
3803                         pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n",
3804                                 ext_name, ext->btf_id, ext->sec_btf_id);
3805                         return ext->sec_btf_id;
3806                 }
3807                 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id);
3808                 sec_name = btf__name_by_offset(obj->btf, sec->name_off);
3809
3810                 if (strcmp(sec_name, KCONFIG_SEC) == 0) {
3811                         if (btf_is_func(t)) {
3812                                 pr_warn("extern function %s is unsupported under %s section\n",
3813                                         ext->name, KCONFIG_SEC);
3814                                 return -ENOTSUP;
3815                         }
3816                         kcfg_sec = sec;
3817                         ext->type = EXT_KCFG;
3818                         ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);
3819                         if (ext->kcfg.sz <= 0) {
3820                                 pr_warn("failed to resolve size of extern (kcfg) '%s': %d\n",
3821                                         ext_name, ext->kcfg.sz);
3822                                 return ext->kcfg.sz;
3823                         }
3824                         ext->kcfg.align = btf__align_of(obj->btf, t->type);
3825                         if (ext->kcfg.align <= 0) {
3826                                 pr_warn("failed to determine alignment of extern (kcfg) '%s': %d\n",
3827                                         ext_name, ext->kcfg.align);
3828                                 return -EINVAL;
3829                         }
3830                         ext->kcfg.type = find_kcfg_type(obj->btf, t->type,
3831                                                         &ext->kcfg.is_signed);
3832                         if (ext->kcfg.type == KCFG_UNKNOWN) {
3833                                 pr_warn("extern (kcfg) '%s': type is unsupported\n", ext_name);
3834                                 return -ENOTSUP;
3835                         }
3836                 } else if (strcmp(sec_name, KSYMS_SEC) == 0) {
3837                         ksym_sec = sec;
3838                         ext->type = EXT_KSYM;
3839                         skip_mods_and_typedefs(obj->btf, t->type,
3840                                                &ext->ksym.type_id);
3841                 } else {
3842                         pr_warn("unrecognized extern section '%s'\n", sec_name);
3843                         return -ENOTSUP;
3844                 }
3845         }
3846         pr_debug("collected %d externs total\n", obj->nr_extern);
3847
3848         if (!obj->nr_extern)
3849                 return 0;
3850
3851         /* sort externs by type, for kcfg ones also by (align, size, name) */
3852         qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs);
3853
3854         /* for .ksyms section, we need to turn all externs into allocated
3855          * variables in BTF to pass kernel verification; we do this by
3856          * pretending that each extern is a 8-byte variable
3857          */
3858         if (ksym_sec) {
3859                 /* find existing 4-byte integer type in BTF to use for fake
3860                  * extern variables in DATASEC
3861                  */
3862                 int int_btf_id = find_int_btf_id(obj->btf);
3863                 /* For extern function, a dummy_var added earlier
3864                  * will be used to replace the vs->type and
3865                  * its name string will be used to refill
3866                  * the missing param's name.
3867                  */
3868                 const struct btf_type *dummy_var;
3869
3870                 dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id);
3871                 for (i = 0; i < obj->nr_extern; i++) {
3872                         ext = &obj->externs[i];
3873                         if (ext->type != EXT_KSYM)
3874                                 continue;
3875                         pr_debug("extern (ksym) #%d: symbol %d, name %s\n",
3876                                  i, ext->sym_idx, ext->name);
3877                 }
3878
3879                 sec = ksym_sec;
3880                 n = btf_vlen(sec);
3881                 for (i = 0, off = 0; i < n; i++, off += sizeof(int)) {
3882                         struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
3883                         struct btf_type *vt;
3884
3885                         vt = (void *)btf__type_by_id(obj->btf, vs->type);
3886                         ext_name = btf__name_by_offset(obj->btf, vt->name_off);
3887                         ext = find_extern_by_name(obj, ext_name);
3888                         if (!ext) {
3889                                 pr_warn("failed to find extern definition for BTF %s '%s'\n",
3890                                         btf_kind_str(vt), ext_name);
3891                                 return -ESRCH;
3892                         }
3893                         if (btf_is_func(vt)) {
3894                                 const struct btf_type *func_proto;
3895                                 struct btf_param *param;
3896                                 int j;
3897
3898                                 func_proto = btf__type_by_id(obj->btf,
3899                                                              vt->type);
3900                                 param = btf_params(func_proto);
3901                                 /* Reuse the dummy_var string if the
3902                                  * func proto does not have param name.
3903                                  */
3904                                 for (j = 0; j < btf_vlen(func_proto); j++)
3905                                         if (param[j].type && !param[j].name_off)
3906                                                 param[j].name_off =
3907                                                         dummy_var->name_off;
3908                                 vs->type = dummy_var_btf_id;
3909                                 vt->info &= ~0xffff;
3910                                 vt->info |= BTF_FUNC_GLOBAL;
3911                         } else {
3912                                 btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
3913                                 vt->type = int_btf_id;
3914                         }
3915                         vs->offset = off;
3916                         vs->size = sizeof(int);
3917                 }
3918                 sec->size = off;
3919         }
3920
3921         if (kcfg_sec) {
3922                 sec = kcfg_sec;
3923                 /* for kcfg externs calculate their offsets within a .kconfig map */
3924                 off = 0;
3925                 for (i = 0; i < obj->nr_extern; i++) {
3926                         ext = &obj->externs[i];
3927                         if (ext->type != EXT_KCFG)
3928                                 continue;
3929
3930                         ext->kcfg.data_off = roundup(off, ext->kcfg.align);
3931                         off = ext->kcfg.data_off + ext->kcfg.sz;
3932                         pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n",
3933                                  i, ext->sym_idx, ext->kcfg.data_off, ext->name);
3934                 }
3935                 sec->size = off;
3936                 n = btf_vlen(sec);
3937                 for (i = 0; i < n; i++) {
3938                         struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
3939
3940                         t = btf__type_by_id(obj->btf, vs->type);
3941                         ext_name = btf__name_by_offset(obj->btf, t->name_off);
3942                         ext = find_extern_by_name(obj, ext_name);
3943                         if (!ext) {
3944                                 pr_warn("failed to find extern definition for BTF var '%s'\n",
3945                                         ext_name);
3946                                 return -ESRCH;
3947                         }
3948                         btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
3949                         vs->offset = ext->kcfg.data_off;
3950                 }
3951         }
3952         return 0;
3953 }
3954
3955 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog)
3956 {
3957         return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1;
3958 }
3959
3960 struct bpf_program *
3961 bpf_object__find_program_by_name(const struct bpf_object *obj,
3962                                  const char *name)
3963 {
3964         struct bpf_program *prog;
3965
3966         bpf_object__for_each_program(prog, obj) {
3967                 if (prog_is_subprog(obj, prog))
3968                         continue;
3969                 if (!strcmp(prog->name, name))
3970                         return prog;
3971         }
3972         return errno = ENOENT, NULL;
3973 }
3974
3975 static bool bpf_object__shndx_is_data(const struct bpf_object *obj,
3976                                       int shndx)
3977 {
3978         switch (obj->efile.secs[shndx].sec_type) {
3979         case SEC_BSS:
3980         case SEC_DATA:
3981         case SEC_RODATA:
3982                 return true;
3983         default:
3984                 return false;
3985         }
3986 }
3987
3988 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj,
3989                                       int shndx)
3990 {
3991         return shndx == obj->efile.btf_maps_shndx;
3992 }
3993
3994 static enum libbpf_map_type
3995 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx)
3996 {
3997         if (shndx == obj->efile.symbols_shndx)
3998                 return LIBBPF_MAP_KCONFIG;
3999
4000         switch (obj->efile.secs[shndx].sec_type) {
4001         case SEC_BSS:
4002                 return LIBBPF_MAP_BSS;
4003         case SEC_DATA:
4004                 return LIBBPF_MAP_DATA;
4005         case SEC_RODATA:
4006                 return LIBBPF_MAP_RODATA;
4007         default:
4008                 return LIBBPF_MAP_UNSPEC;
4009         }
4010 }
4011
4012 static int bpf_program__record_reloc(struct bpf_program *prog,
4013                                      struct reloc_desc *reloc_desc,
4014                                      __u32 insn_idx, const char *sym_name,
4015                                      const Elf64_Sym *sym, const Elf64_Rel *rel)
4016 {
4017         struct bpf_insn *insn = &prog->insns[insn_idx];
4018         size_t map_idx, nr_maps = prog->obj->nr_maps;
4019         struct bpf_object *obj = prog->obj;
4020         __u32 shdr_idx = sym->st_shndx;
4021         enum libbpf_map_type type;
4022         const char *sym_sec_name;
4023         struct bpf_map *map;
4024
4025         if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) {
4026                 pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n",
4027                         prog->name, sym_name, insn_idx, insn->code);
4028                 return -LIBBPF_ERRNO__RELOC;
4029         }
4030
4031         if (sym_is_extern(sym)) {
4032                 int sym_idx = ELF64_R_SYM(rel->r_info);
4033                 int i, n = obj->nr_extern;
4034                 struct extern_desc *ext;
4035
4036                 for (i = 0; i < n; i++) {
4037                         ext = &obj->externs[i];
4038                         if (ext->sym_idx == sym_idx)
4039                                 break;
4040                 }
4041                 if (i >= n) {
4042                         pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n",
4043                                 prog->name, sym_name, sym_idx);
4044                         return -LIBBPF_ERRNO__RELOC;
4045                 }
4046                 pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n",
4047                          prog->name, i, ext->name, ext->sym_idx, insn_idx);
4048                 if (insn->code == (BPF_JMP | BPF_CALL))
4049                         reloc_desc->type = RELO_EXTERN_FUNC;
4050                 else
4051                         reloc_desc->type = RELO_EXTERN_VAR;
4052                 reloc_desc->insn_idx = insn_idx;
4053                 reloc_desc->sym_off = i; /* sym_off stores extern index */
4054                 return 0;
4055         }
4056
4057         /* sub-program call relocation */
4058         if (is_call_insn(insn)) {
4059                 if (insn->src_reg != BPF_PSEUDO_CALL) {
4060                         pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
4061                         return -LIBBPF_ERRNO__RELOC;
4062                 }
4063                 /* text_shndx can be 0, if no default "main" program exists */
4064                 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {
4065                         sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
4066                         pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n",
4067                                 prog->name, sym_name, sym_sec_name);
4068                         return -LIBBPF_ERRNO__RELOC;
4069                 }
4070                 if (sym->st_value % BPF_INSN_SZ) {
4071                         pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n",
4072                                 prog->name, sym_name, (size_t)sym->st_value);
4073                         return -LIBBPF_ERRNO__RELOC;
4074                 }
4075                 reloc_desc->type = RELO_CALL;
4076                 reloc_desc->insn_idx = insn_idx;
4077                 reloc_desc->sym_off = sym->st_value;
4078                 return 0;
4079         }
4080
4081         if (!shdr_idx || shdr_idx >= SHN_LORESERVE) {
4082                 pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n",
4083                         prog->name, sym_name, shdr_idx);
4084                 return -LIBBPF_ERRNO__RELOC;
4085         }
4086
4087         /* loading subprog addresses */
4088         if (sym_is_subprog(sym, obj->efile.text_shndx)) {
4089                 /* global_func: sym->st_value = offset in the section, insn->imm = 0.
4090                  * local_func: sym->st_value = 0, insn->imm = offset in the section.
4091                  */
4092                 if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) {
4093                         pr_warn("prog '%s': bad subprog addr relo against '%s' at offset %zu+%d\n",
4094                                 prog->name, sym_name, (size_t)sym->st_value, insn->imm);
4095                         return -LIBBPF_ERRNO__RELOC;
4096                 }
4097
4098                 reloc_desc->type = RELO_SUBPROG_ADDR;
4099                 reloc_desc->insn_idx = insn_idx;
4100                 reloc_desc->sym_off = sym->st_value;
4101                 return 0;
4102         }
4103
4104         type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);
4105         sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
4106
4107         /* generic map reference relocation */
4108         if (type == LIBBPF_MAP_UNSPEC) {
4109                 if (!bpf_object__shndx_is_maps(obj, shdr_idx)) {
4110                         pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n",
4111                                 prog->name, sym_name, sym_sec_name);
4112                         return -LIBBPF_ERRNO__RELOC;
4113                 }
4114                 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
4115                         map = &obj->maps[map_idx];
4116                         if (map->libbpf_type != type ||
4117                             map->sec_idx != sym->st_shndx ||
4118                             map->sec_offset != sym->st_value)
4119                                 continue;
4120                         pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n",
4121                                  prog->name, map_idx, map->name, map->sec_idx,
4122                                  map->sec_offset, insn_idx);
4123                         break;
4124                 }
4125                 if (map_idx >= nr_maps) {
4126                         pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n",
4127                                 prog->name, sym_sec_name, (size_t)sym->st_value);
4128                         return -LIBBPF_ERRNO__RELOC;
4129                 }
4130                 reloc_desc->type = RELO_LD64;
4131                 reloc_desc->insn_idx = insn_idx;
4132                 reloc_desc->map_idx = map_idx;
4133                 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */
4134                 return 0;
4135         }
4136
4137         /* global data map relocation */
4138         if (!bpf_object__shndx_is_data(obj, shdr_idx)) {
4139                 pr_warn("prog '%s': bad data relo against section '%s'\n",
4140                         prog->name, sym_sec_name);
4141                 return -LIBBPF_ERRNO__RELOC;
4142         }
4143         for (map_idx = 0; map_idx < nr_maps; map_idx++) {
4144                 map = &obj->maps[map_idx];
4145                 if (map->libbpf_type != type || map->sec_idx != sym->st_shndx)
4146                         continue;
4147                 pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n",
4148                          prog->name, map_idx, map->name, map->sec_idx,
4149                          map->sec_offset, insn_idx);
4150                 break;
4151         }
4152         if (map_idx >= nr_maps) {
4153                 pr_warn("prog '%s': data relo failed to find map for section '%s'\n",
4154                         prog->name, sym_sec_name);
4155                 return -LIBBPF_ERRNO__RELOC;
4156         }
4157
4158         reloc_desc->type = RELO_DATA;
4159         reloc_desc->insn_idx = insn_idx;
4160         reloc_desc->map_idx = map_idx;
4161         reloc_desc->sym_off = sym->st_value;
4162         return 0;
4163 }
4164
4165 static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx)
4166 {
4167         return insn_idx >= prog->sec_insn_off &&
4168                insn_idx < prog->sec_insn_off + prog->sec_insn_cnt;
4169 }
4170
4171 static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj,
4172                                                  size_t sec_idx, size_t insn_idx)
4173 {
4174         int l = 0, r = obj->nr_programs - 1, m;
4175         struct bpf_program *prog;
4176
4177         if (!obj->nr_programs)
4178                 return NULL;
4179
4180         while (l < r) {
4181                 m = l + (r - l + 1) / 2;
4182                 prog = &obj->programs[m];
4183
4184                 if (prog->sec_idx < sec_idx ||
4185                     (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx))
4186                         l = m;
4187                 else
4188                         r = m - 1;
4189         }
4190         /* matching program could be at index l, but it still might be the
4191          * wrong one, so we need to double check conditions for the last time
4192          */
4193         prog = &obj->programs[l];
4194         if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx))
4195                 return prog;
4196         return NULL;
4197 }
4198
4199 static int
4200 bpf_object__collect_prog_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Data *data)
4201 {
4202         const char *relo_sec_name, *sec_name;
4203         size_t sec_idx = shdr->sh_info, sym_idx;
4204         struct bpf_program *prog;
4205         struct reloc_desc *relos;
4206         int err, i, nrels;
4207         const char *sym_name;
4208         __u32 insn_idx;
4209         Elf_Scn *scn;
4210         Elf_Data *scn_data;
4211         Elf64_Sym *sym;
4212         Elf64_Rel *rel;
4213
4214         if (sec_idx >= obj->efile.sec_cnt)
4215                 return -EINVAL;
4216
4217         scn = elf_sec_by_idx(obj, sec_idx);
4218         scn_data = elf_sec_data(obj, scn);
4219
4220         relo_sec_name = elf_sec_str(obj, shdr->sh_name);
4221         sec_name = elf_sec_name(obj, scn);
4222         if (!relo_sec_name || !sec_name)
4223                 return -EINVAL;
4224
4225         pr_debug("sec '%s': collecting relocation for section(%zu) '%s'\n",
4226                  relo_sec_name, sec_idx, sec_name);
4227         nrels = shdr->sh_size / shdr->sh_entsize;
4228
4229         for (i = 0; i < nrels; i++) {
4230                 rel = elf_rel_by_idx(data, i);
4231                 if (!rel) {
4232                         pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i);
4233                         return -LIBBPF_ERRNO__FORMAT;
4234                 }
4235
4236                 sym_idx = ELF64_R_SYM(rel->r_info);
4237                 sym = elf_sym_by_idx(obj, sym_idx);
4238                 if (!sym) {
4239                         pr_warn("sec '%s': symbol #%zu not found for relo #%d\n",
4240                                 relo_sec_name, sym_idx, i);
4241                         return -LIBBPF_ERRNO__FORMAT;
4242                 }
4243
4244                 if (sym->st_shndx >= obj->efile.sec_cnt) {
4245                         pr_warn("sec '%s': corrupted symbol #%zu pointing to invalid section #%zu for relo #%d\n",
4246                                 relo_sec_name, sym_idx, (size_t)sym->st_shndx, i);
4247                         return -LIBBPF_ERRNO__FORMAT;
4248                 }
4249
4250                 if (rel->r_offset % BPF_INSN_SZ || rel->r_offset >= scn_data->d_size) {
4251                         pr_warn("sec '%s': invalid offset 0x%zx for relo #%d\n",
4252                                 relo_sec_name, (size_t)rel->r_offset, i);
4253                         return -LIBBPF_ERRNO__FORMAT;
4254                 }
4255
4256                 insn_idx = rel->r_offset / BPF_INSN_SZ;
4257                 /* relocations against static functions are recorded as
4258                  * relocations against the section that contains a function;
4259                  * in such case, symbol will be STT_SECTION and sym.st_name
4260                  * will point to empty string (0), so fetch section name
4261                  * instead
4262                  */
4263                 if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_name == 0)
4264                         sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx));
4265                 else
4266                         sym_name = elf_sym_str(obj, sym->st_name);
4267                 sym_name = sym_name ?: "<?";
4268
4269                 pr_debug("sec '%s': relo #%d: insn #%u against '%s'\n",
4270                          relo_sec_name, i, insn_idx, sym_name);
4271
4272                 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
4273                 if (!prog) {
4274                         pr_debug("sec '%s': relo #%d: couldn't find program in section '%s' for insn #%u, probably overridden weak function, skipping...\n",
4275                                 relo_sec_name, i, sec_name, insn_idx);
4276                         continue;
4277                 }
4278
4279                 relos = libbpf_reallocarray(prog->reloc_desc,
4280                                             prog->nr_reloc + 1, sizeof(*relos));
4281                 if (!relos)
4282                         return -ENOMEM;
4283                 prog->reloc_desc = relos;
4284
4285                 /* adjust insn_idx to local BPF program frame of reference */
4286                 insn_idx -= prog->sec_insn_off;
4287                 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc],
4288                                                 insn_idx, sym_name, sym, rel);
4289                 if (err)
4290                         return err;
4291
4292                 prog->nr_reloc++;
4293         }
4294         return 0;
4295 }
4296
4297 static int map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map)
4298 {
4299         int id;
4300
4301         if (!obj->btf)
4302                 return -ENOENT;
4303
4304         /* if it's BTF-defined map, we don't need to search for type IDs.
4305          * For struct_ops map, it does not need btf_key_type_id and
4306          * btf_value_type_id.
4307          */
4308         if (map->sec_idx == obj->efile.btf_maps_shndx || bpf_map__is_struct_ops(map))
4309                 return 0;
4310
4311         /*
4312          * LLVM annotates global data differently in BTF, that is,
4313          * only as '.data', '.bss' or '.rodata'.
4314          */
4315         if (!bpf_map__is_internal(map))
4316                 return -ENOENT;
4317
4318         id = btf__find_by_name(obj->btf, map->real_name);
4319         if (id < 0)
4320                 return id;
4321
4322         map->btf_key_type_id = 0;
4323         map->btf_value_type_id = id;
4324         return 0;
4325 }
4326
4327 static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info)
4328 {
4329         char file[PATH_MAX], buff[4096];
4330         FILE *fp;
4331         __u32 val;
4332         int err;
4333
4334         snprintf(file, sizeof(file), "/proc/%d/fdinfo/%d", getpid(), fd);
4335         memset(info, 0, sizeof(*info));
4336
4337         fp = fopen(file, "r");
4338         if (!fp) {
4339                 err = -errno;
4340                 pr_warn("failed to open %s: %d. No procfs support?\n", file,
4341                         err);
4342                 return err;
4343         }
4344
4345         while (fgets(buff, sizeof(buff), fp)) {
4346                 if (sscanf(buff, "map_type:\t%u", &val) == 1)
4347                         info->type = val;
4348                 else if (sscanf(buff, "key_size:\t%u", &val) == 1)
4349                         info->key_size = val;
4350                 else if (sscanf(buff, "value_size:\t%u", &val) == 1)
4351                         info->value_size = val;
4352                 else if (sscanf(buff, "max_entries:\t%u", &val) == 1)
4353                         info->max_entries = val;
4354                 else if (sscanf(buff, "map_flags:\t%i", &val) == 1)
4355                         info->map_flags = val;
4356         }
4357
4358         fclose(fp);
4359
4360         return 0;
4361 }
4362
4363 bool bpf_map__autocreate(const struct bpf_map *map)
4364 {
4365         return map->autocreate;
4366 }
4367
4368 int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate)
4369 {
4370         if (map->obj->loaded)
4371                 return libbpf_err(-EBUSY);
4372
4373         map->autocreate = autocreate;
4374         return 0;
4375 }
4376
4377 int bpf_map__reuse_fd(struct bpf_map *map, int fd)
4378 {
4379         struct bpf_map_info info;
4380         __u32 len = sizeof(info), name_len;
4381         int new_fd, err;
4382         char *new_name;
4383
4384         memset(&info, 0, len);
4385         err = bpf_obj_get_info_by_fd(fd, &info, &len);
4386         if (err && errno == EINVAL)
4387                 err = bpf_get_map_info_from_fdinfo(fd, &info);
4388         if (err)
4389                 return libbpf_err(err);
4390
4391         name_len = strlen(info.name);
4392         if (name_len == BPF_OBJ_NAME_LEN - 1 && strncmp(map->name, info.name, name_len) == 0)
4393                 new_name = strdup(map->name);
4394         else
4395                 new_name = strdup(info.name);
4396
4397         if (!new_name)
4398                 return libbpf_err(-errno);
4399
4400         new_fd = open("/", O_RDONLY | O_CLOEXEC);
4401         if (new_fd < 0) {
4402                 err = -errno;
4403                 goto err_free_new_name;
4404         }
4405
4406         new_fd = dup3(fd, new_fd, O_CLOEXEC);
4407         if (new_fd < 0) {
4408                 err = -errno;
4409                 goto err_close_new_fd;
4410         }
4411
4412         err = zclose(map->fd);
4413         if (err) {
4414                 err = -errno;
4415                 goto err_close_new_fd;
4416         }
4417         free(map->name);
4418
4419         map->fd = new_fd;
4420         map->name = new_name;
4421         map->def.type = info.type;
4422         map->def.key_size = info.key_size;
4423         map->def.value_size = info.value_size;
4424         map->def.max_entries = info.max_entries;
4425         map->def.map_flags = info.map_flags;
4426         map->btf_key_type_id = info.btf_key_type_id;
4427         map->btf_value_type_id = info.btf_value_type_id;
4428         map->reused = true;
4429         map->map_extra = info.map_extra;
4430
4431         return 0;
4432
4433 err_close_new_fd:
4434         close(new_fd);
4435 err_free_new_name:
4436         free(new_name);
4437         return libbpf_err(err);
4438 }
4439
4440 __u32 bpf_map__max_entries(const struct bpf_map *map)
4441 {
4442         return map->def.max_entries;
4443 }
4444
4445 struct bpf_map *bpf_map__inner_map(struct bpf_map *map)
4446 {
4447         if (!bpf_map_type__is_map_in_map(map->def.type))
4448                 return errno = EINVAL, NULL;
4449
4450         return map->inner_map;
4451 }
4452
4453 int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries)
4454 {
4455         if (map->obj->loaded)
4456                 return libbpf_err(-EBUSY);
4457
4458         map->def.max_entries = max_entries;
4459
4460         /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
4461         if (map_is_ringbuf(map))
4462                 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
4463
4464         return 0;
4465 }
4466
4467 static int
4468 bpf_object__probe_loading(struct bpf_object *obj)
4469 {
4470         char *cp, errmsg[STRERR_BUFSIZE];
4471         struct bpf_insn insns[] = {
4472                 BPF_MOV64_IMM(BPF_REG_0, 0),
4473                 BPF_EXIT_INSN(),
4474         };
4475         int ret, insn_cnt = ARRAY_SIZE(insns);
4476
4477         if (obj->gen_loader)
4478                 return 0;
4479
4480         ret = bump_rlimit_memlock();
4481         if (ret)
4482                 pr_warn("Failed to bump RLIMIT_MEMLOCK (err = %d), you might need to do it explicitly!\n", ret);
4483
4484         /* make sure basic loading works */
4485         ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL);
4486         if (ret < 0)
4487                 ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL);
4488         if (ret < 0) {
4489                 ret = errno;
4490                 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
4491                 pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF "
4492                         "program. Make sure your kernel supports BPF "
4493                         "(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is "
4494                         "set to big enough value.\n", __func__, cp, ret);
4495                 return -ret;
4496         }
4497         close(ret);
4498
4499         return 0;
4500 }
4501
4502 static int probe_fd(int fd)
4503 {
4504         if (fd >= 0)
4505                 close(fd);
4506         return fd >= 0;
4507 }
4508
4509 static int probe_kern_prog_name(void)
4510 {
4511         const size_t attr_sz = offsetofend(union bpf_attr, prog_name);
4512         struct bpf_insn insns[] = {
4513                 BPF_MOV64_IMM(BPF_REG_0, 0),
4514                 BPF_EXIT_INSN(),
4515         };
4516         union bpf_attr attr;
4517         int ret;
4518
4519         memset(&attr, 0, attr_sz);
4520         attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
4521         attr.license = ptr_to_u64("GPL");
4522         attr.insns = ptr_to_u64(insns);
4523         attr.insn_cnt = (__u32)ARRAY_SIZE(insns);
4524         libbpf_strlcpy(attr.prog_name, "libbpf_nametest", sizeof(attr.prog_name));
4525
4526         /* make sure loading with name works */
4527         ret = sys_bpf_prog_load(&attr, attr_sz, PROG_LOAD_ATTEMPTS);
4528         return probe_fd(ret);
4529 }
4530
4531 static int probe_kern_global_data(void)
4532 {
4533         char *cp, errmsg[STRERR_BUFSIZE];
4534         struct bpf_insn insns[] = {
4535                 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16),
4536                 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42),
4537                 BPF_MOV64_IMM(BPF_REG_0, 0),
4538                 BPF_EXIT_INSN(),
4539         };
4540         int ret, map, insn_cnt = ARRAY_SIZE(insns);
4541
4542         map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_global", sizeof(int), 32, 1, NULL);
4543         if (map < 0) {
4544                 ret = -errno;
4545                 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
4546                 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
4547                         __func__, cp, -ret);
4548                 return ret;
4549         }
4550
4551         insns[0].imm = map;
4552
4553         ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL);
4554         close(map);
4555         return probe_fd(ret);
4556 }
4557
4558 static int probe_kern_btf(void)
4559 {
4560         static const char strs[] = "\0int";
4561         __u32 types[] = {
4562                 /* int */
4563                 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
4564         };
4565
4566         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4567                                              strs, sizeof(strs)));
4568 }
4569
4570 static int probe_kern_btf_func(void)
4571 {
4572         static const char strs[] = "\0int\0x\0a";
4573         /* void x(int a) {} */
4574         __u32 types[] = {
4575                 /* int */
4576                 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
4577                 /* FUNC_PROTO */                                /* [2] */
4578                 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
4579                 BTF_PARAM_ENC(7, 1),
4580                 /* FUNC x */                                    /* [3] */
4581                 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2),
4582         };
4583
4584         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4585                                              strs, sizeof(strs)));
4586 }
4587
4588 static int probe_kern_btf_func_global(void)
4589 {
4590         static const char strs[] = "\0int\0x\0a";
4591         /* static void x(int a) {} */
4592         __u32 types[] = {
4593                 /* int */
4594                 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
4595                 /* FUNC_PROTO */                                /* [2] */
4596                 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
4597                 BTF_PARAM_ENC(7, 1),
4598                 /* FUNC x BTF_FUNC_GLOBAL */                    /* [3] */
4599                 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2),
4600         };
4601
4602         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4603                                              strs, sizeof(strs)));
4604 }
4605
4606 static int probe_kern_btf_datasec(void)
4607 {
4608         static const char strs[] = "\0x\0.data";
4609         /* static int a; */
4610         __u32 types[] = {
4611                 /* int */
4612                 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
4613                 /* VAR x */                                     /* [2] */
4614                 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1),
4615                 BTF_VAR_STATIC,
4616                 /* DATASEC val */                               /* [3] */
4617                 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4),
4618                 BTF_VAR_SECINFO_ENC(2, 0, 4),
4619         };
4620
4621         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4622                                              strs, sizeof(strs)));
4623 }
4624
4625 static int probe_kern_btf_float(void)
4626 {
4627         static const char strs[] = "\0float";
4628         __u32 types[] = {
4629                 /* float */
4630                 BTF_TYPE_FLOAT_ENC(1, 4),
4631         };
4632
4633         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4634                                              strs, sizeof(strs)));
4635 }
4636
4637 static int probe_kern_btf_decl_tag(void)
4638 {
4639         static const char strs[] = "\0tag";
4640         __u32 types[] = {
4641                 /* int */
4642                 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
4643                 /* VAR x */                                     /* [2] */
4644                 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1),
4645                 BTF_VAR_STATIC,
4646                 /* attr */
4647                 BTF_TYPE_DECL_TAG_ENC(1, 2, -1),
4648         };
4649
4650         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4651                                              strs, sizeof(strs)));
4652 }
4653
4654 static int probe_kern_btf_type_tag(void)
4655 {
4656         static const char strs[] = "\0tag";
4657         __u32 types[] = {
4658                 /* int */
4659                 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),          /* [1] */
4660                 /* attr */
4661                 BTF_TYPE_TYPE_TAG_ENC(1, 1),                            /* [2] */
4662                 /* ptr */
4663                 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), 2),   /* [3] */
4664         };
4665
4666         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4667                                              strs, sizeof(strs)));
4668 }
4669
4670 static int probe_kern_array_mmap(void)
4671 {
4672         LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_MMAPABLE);
4673         int fd;
4674
4675         fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_mmap", sizeof(int), sizeof(int), 1, &opts);
4676         return probe_fd(fd);
4677 }
4678
4679 static int probe_kern_exp_attach_type(void)
4680 {
4681         LIBBPF_OPTS(bpf_prog_load_opts, opts, .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE);
4682         struct bpf_insn insns[] = {
4683                 BPF_MOV64_IMM(BPF_REG_0, 0),
4684                 BPF_EXIT_INSN(),
4685         };
4686         int fd, insn_cnt = ARRAY_SIZE(insns);
4687
4688         /* use any valid combination of program type and (optional)
4689          * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS)
4690          * to see if kernel supports expected_attach_type field for
4691          * BPF_PROG_LOAD command
4692          */
4693         fd = bpf_prog_load(BPF_PROG_TYPE_CGROUP_SOCK, NULL, "GPL", insns, insn_cnt, &opts);
4694         return probe_fd(fd);
4695 }
4696
4697 static int probe_kern_probe_read_kernel(void)
4698 {
4699         struct bpf_insn insns[] = {
4700                 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),   /* r1 = r10 (fp) */
4701                 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8),  /* r1 += -8 */
4702                 BPF_MOV64_IMM(BPF_REG_2, 8),            /* r2 = 8 */
4703                 BPF_MOV64_IMM(BPF_REG_3, 0),            /* r3 = 0 */
4704                 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_probe_read_kernel),
4705                 BPF_EXIT_INSN(),
4706         };
4707         int fd, insn_cnt = ARRAY_SIZE(insns);
4708
4709         fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL);
4710         return probe_fd(fd);
4711 }
4712
4713 static int probe_prog_bind_map(void)
4714 {
4715         char *cp, errmsg[STRERR_BUFSIZE];
4716         struct bpf_insn insns[] = {
4717                 BPF_MOV64_IMM(BPF_REG_0, 0),
4718                 BPF_EXIT_INSN(),
4719         };
4720         int ret, map, prog, insn_cnt = ARRAY_SIZE(insns);
4721
4722         map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_bind", sizeof(int), 32, 1, NULL);
4723         if (map < 0) {
4724                 ret = -errno;
4725                 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
4726                 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
4727                         __func__, cp, -ret);
4728                 return ret;
4729         }
4730
4731         prog = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL);
4732         if (prog < 0) {
4733                 close(map);
4734                 return 0;
4735         }
4736
4737         ret = bpf_prog_bind_map(prog, map, NULL);
4738
4739         close(map);
4740         close(prog);
4741
4742         return ret >= 0;
4743 }
4744
4745 static int probe_module_btf(void)
4746 {
4747         static const char strs[] = "\0int";
4748         __u32 types[] = {
4749                 /* int */
4750                 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
4751         };
4752         struct bpf_btf_info info;
4753         __u32 len = sizeof(info);
4754         char name[16];
4755         int fd, err;
4756
4757         fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs));
4758         if (fd < 0)
4759                 return 0; /* BTF not supported at all */
4760
4761         memset(&info, 0, sizeof(info));
4762         info.name = ptr_to_u64(name);
4763         info.name_len = sizeof(name);
4764
4765         /* check that BPF_OBJ_GET_INFO_BY_FD supports specifying name pointer;
4766          * kernel's module BTF support coincides with support for
4767          * name/name_len fields in struct bpf_btf_info.
4768          */
4769         err = bpf_obj_get_info_by_fd(fd, &info, &len);
4770         close(fd);
4771         return !err;
4772 }
4773
4774 static int probe_perf_link(void)
4775 {
4776         struct bpf_insn insns[] = {
4777                 BPF_MOV64_IMM(BPF_REG_0, 0),
4778                 BPF_EXIT_INSN(),
4779         };
4780         int prog_fd, link_fd, err;
4781
4782         prog_fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL",
4783                                 insns, ARRAY_SIZE(insns), NULL);
4784         if (prog_fd < 0)
4785                 return -errno;
4786
4787         /* use invalid perf_event FD to get EBADF, if link is supported;
4788          * otherwise EINVAL should be returned
4789          */
4790         link_fd = bpf_link_create(prog_fd, -1, BPF_PERF_EVENT, NULL);
4791         err = -errno; /* close() can clobber errno */
4792
4793         if (link_fd >= 0)
4794                 close(link_fd);
4795         close(prog_fd);
4796
4797         return link_fd < 0 && err == -EBADF;
4798 }
4799
4800 static int probe_kern_bpf_cookie(void)
4801 {
4802         struct bpf_insn insns[] = {
4803                 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_attach_cookie),
4804                 BPF_EXIT_INSN(),
4805         };
4806         int ret, insn_cnt = ARRAY_SIZE(insns);
4807
4808         ret = bpf_prog_load(BPF_PROG_TYPE_KPROBE, NULL, "GPL", insns, insn_cnt, NULL);
4809         return probe_fd(ret);
4810 }
4811
4812 static int probe_kern_btf_enum64(void)
4813 {
4814         static const char strs[] = "\0enum64";
4815         __u32 types[] = {
4816                 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 0), 8),
4817         };
4818
4819         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4820                                              strs, sizeof(strs)));
4821 }
4822
4823 static int probe_kern_syscall_wrapper(void);
4824
4825 enum kern_feature_result {
4826         FEAT_UNKNOWN = 0,
4827         FEAT_SUPPORTED = 1,
4828         FEAT_MISSING = 2,
4829 };
4830
4831 typedef int (*feature_probe_fn)(void);
4832
4833 static struct kern_feature_desc {
4834         const char *desc;
4835         feature_probe_fn probe;
4836         enum kern_feature_result res;
4837 } feature_probes[__FEAT_CNT] = {
4838         [FEAT_PROG_NAME] = {
4839                 "BPF program name", probe_kern_prog_name,
4840         },
4841         [FEAT_GLOBAL_DATA] = {
4842                 "global variables", probe_kern_global_data,
4843         },
4844         [FEAT_BTF] = {
4845                 "minimal BTF", probe_kern_btf,
4846         },
4847         [FEAT_BTF_FUNC] = {
4848                 "BTF functions", probe_kern_btf_func,
4849         },
4850         [FEAT_BTF_GLOBAL_FUNC] = {
4851                 "BTF global function", probe_kern_btf_func_global,
4852         },
4853         [FEAT_BTF_DATASEC] = {
4854                 "BTF data section and variable", probe_kern_btf_datasec,
4855         },
4856         [FEAT_ARRAY_MMAP] = {
4857                 "ARRAY map mmap()", probe_kern_array_mmap,
4858         },
4859         [FEAT_EXP_ATTACH_TYPE] = {
4860                 "BPF_PROG_LOAD expected_attach_type attribute",
4861                 probe_kern_exp_attach_type,
4862         },
4863         [FEAT_PROBE_READ_KERN] = {
4864                 "bpf_probe_read_kernel() helper", probe_kern_probe_read_kernel,
4865         },
4866         [FEAT_PROG_BIND_MAP] = {
4867                 "BPF_PROG_BIND_MAP support", probe_prog_bind_map,
4868         },
4869         [FEAT_MODULE_BTF] = {
4870                 "module BTF support", probe_module_btf,
4871         },
4872         [FEAT_BTF_FLOAT] = {
4873                 "BTF_KIND_FLOAT support", probe_kern_btf_float,
4874         },
4875         [FEAT_PERF_LINK] = {
4876                 "BPF perf link support", probe_perf_link,
4877         },
4878         [FEAT_BTF_DECL_TAG] = {
4879                 "BTF_KIND_DECL_TAG support", probe_kern_btf_decl_tag,
4880         },
4881         [FEAT_BTF_TYPE_TAG] = {
4882                 "BTF_KIND_TYPE_TAG support", probe_kern_btf_type_tag,
4883         },
4884         [FEAT_MEMCG_ACCOUNT] = {
4885                 "memcg-based memory accounting", probe_memcg_account,
4886         },
4887         [FEAT_BPF_COOKIE] = {
4888                 "BPF cookie support", probe_kern_bpf_cookie,
4889         },
4890         [FEAT_BTF_ENUM64] = {
4891                 "BTF_KIND_ENUM64 support", probe_kern_btf_enum64,
4892         },
4893         [FEAT_SYSCALL_WRAPPER] = {
4894                 "Kernel using syscall wrapper", probe_kern_syscall_wrapper,
4895         },
4896 };
4897
4898 bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id)
4899 {
4900         struct kern_feature_desc *feat = &feature_probes[feat_id];
4901         int ret;
4902
4903         if (obj && obj->gen_loader)
4904                 /* To generate loader program assume the latest kernel
4905                  * to avoid doing extra prog_load, map_create syscalls.
4906                  */
4907                 return true;
4908
4909         if (READ_ONCE(feat->res) == FEAT_UNKNOWN) {
4910                 ret = feat->probe();
4911                 if (ret > 0) {
4912                         WRITE_ONCE(feat->res, FEAT_SUPPORTED);
4913                 } else if (ret == 0) {
4914                         WRITE_ONCE(feat->res, FEAT_MISSING);
4915                 } else {
4916                         pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret);
4917                         WRITE_ONCE(feat->res, FEAT_MISSING);
4918                 }
4919         }
4920
4921         return READ_ONCE(feat->res) == FEAT_SUPPORTED;
4922 }
4923
4924 static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd)
4925 {
4926         struct bpf_map_info map_info;
4927         char msg[STRERR_BUFSIZE];
4928         __u32 map_info_len = sizeof(map_info);
4929         int err;
4930
4931         memset(&map_info, 0, map_info_len);
4932         err = bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len);
4933         if (err && errno == EINVAL)
4934                 err = bpf_get_map_info_from_fdinfo(map_fd, &map_info);
4935         if (err) {
4936                 pr_warn("failed to get map info for map FD %d: %s\n", map_fd,
4937                         libbpf_strerror_r(errno, msg, sizeof(msg)));
4938                 return false;
4939         }
4940
4941         return (map_info.type == map->def.type &&
4942                 map_info.key_size == map->def.key_size &&
4943                 map_info.value_size == map->def.value_size &&
4944                 map_info.max_entries == map->def.max_entries &&
4945                 map_info.map_flags == map->def.map_flags &&
4946                 map_info.map_extra == map->map_extra);
4947 }
4948
4949 static int
4950 bpf_object__reuse_map(struct bpf_map *map)
4951 {
4952         char *cp, errmsg[STRERR_BUFSIZE];
4953         int err, pin_fd;
4954
4955         pin_fd = bpf_obj_get(map->pin_path);
4956         if (pin_fd < 0) {
4957                 err = -errno;
4958                 if (err == -ENOENT) {
4959                         pr_debug("found no pinned map to reuse at '%s'\n",
4960                                  map->pin_path);
4961                         return 0;
4962                 }
4963
4964                 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
4965                 pr_warn("couldn't retrieve pinned map '%s': %s\n",
4966                         map->pin_path, cp);
4967                 return err;
4968         }
4969
4970         if (!map_is_reuse_compat(map, pin_fd)) {
4971                 pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n",
4972                         map->pin_path);
4973                 close(pin_fd);
4974                 return -EINVAL;
4975         }
4976
4977         err = bpf_map__reuse_fd(map, pin_fd);
4978         close(pin_fd);
4979         if (err)
4980                 return err;
4981
4982         map->pinned = true;
4983         pr_debug("reused pinned map at '%s'\n", map->pin_path);
4984
4985         return 0;
4986 }
4987
4988 static int
4989 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map)
4990 {
4991         enum libbpf_map_type map_type = map->libbpf_type;
4992         char *cp, errmsg[STRERR_BUFSIZE];
4993         int err, zero = 0;
4994
4995         if (obj->gen_loader) {
4996                 bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps,
4997                                          map->mmaped, map->def.value_size);
4998                 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG)
4999                         bpf_gen__map_freeze(obj->gen_loader, map - obj->maps);
5000                 return 0;
5001         }
5002         err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0);
5003         if (err) {
5004                 err = -errno;
5005                 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
5006                 pr_warn("Error setting initial map(%s) contents: %s\n",
5007                         map->name, cp);
5008                 return err;
5009         }
5010
5011         /* Freeze .rodata and .kconfig map as read-only from syscall side. */
5012         if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) {
5013                 err = bpf_map_freeze(map->fd);
5014                 if (err) {
5015                         err = -errno;
5016                         cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
5017                         pr_warn("Error freezing map(%s) as read-only: %s\n",
5018                                 map->name, cp);
5019                         return err;
5020                 }
5021         }
5022         return 0;
5023 }
5024
5025 static void bpf_map__destroy(struct bpf_map *map);
5026
5027 static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, bool is_inner)
5028 {
5029         LIBBPF_OPTS(bpf_map_create_opts, create_attr);
5030         struct bpf_map_def *def = &map->def;
5031         const char *map_name = NULL;
5032         int err = 0;
5033
5034         if (kernel_supports(obj, FEAT_PROG_NAME))
5035                 map_name = map->name;
5036         create_attr.map_ifindex = map->map_ifindex;
5037         create_attr.map_flags = def->map_flags;
5038         create_attr.numa_node = map->numa_node;
5039         create_attr.map_extra = map->map_extra;
5040
5041         if (bpf_map__is_struct_ops(map))
5042                 create_attr.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;
5043
5044         if (obj->btf && btf__fd(obj->btf) >= 0) {
5045                 create_attr.btf_fd = btf__fd(obj->btf);
5046                 create_attr.btf_key_type_id = map->btf_key_type_id;
5047                 create_attr.btf_value_type_id = map->btf_value_type_id;
5048         }
5049
5050         if (bpf_map_type__is_map_in_map(def->type)) {
5051                 if (map->inner_map) {
5052                         err = bpf_object__create_map(obj, map->inner_map, true);
5053                         if (err) {
5054                                 pr_warn("map '%s': failed to create inner map: %d\n",
5055                                         map->name, err);
5056                                 return err;
5057                         }
5058                         map->inner_map_fd = bpf_map__fd(map->inner_map);
5059                 }
5060                 if (map->inner_map_fd >= 0)
5061                         create_attr.inner_map_fd = map->inner_map_fd;
5062         }
5063
5064         switch (def->type) {
5065         case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
5066         case BPF_MAP_TYPE_CGROUP_ARRAY:
5067         case BPF_MAP_TYPE_STACK_TRACE:
5068         case BPF_MAP_TYPE_ARRAY_OF_MAPS:
5069         case BPF_MAP_TYPE_HASH_OF_MAPS:
5070         case BPF_MAP_TYPE_DEVMAP:
5071         case BPF_MAP_TYPE_DEVMAP_HASH:
5072         case BPF_MAP_TYPE_CPUMAP:
5073         case BPF_MAP_TYPE_XSKMAP:
5074         case BPF_MAP_TYPE_SOCKMAP:
5075         case BPF_MAP_TYPE_SOCKHASH:
5076         case BPF_MAP_TYPE_QUEUE:
5077         case BPF_MAP_TYPE_STACK:
5078                 create_attr.btf_fd = 0;
5079                 create_attr.btf_key_type_id = 0;
5080                 create_attr.btf_value_type_id = 0;
5081                 map->btf_key_type_id = 0;
5082                 map->btf_value_type_id = 0;
5083         default:
5084                 break;
5085         }
5086
5087         if (obj->gen_loader) {
5088                 bpf_gen__map_create(obj->gen_loader, def->type, map_name,
5089                                     def->key_size, def->value_size, def->max_entries,
5090                                     &create_attr, is_inner ? -1 : map - obj->maps);
5091                 /* Pretend to have valid FD to pass various fd >= 0 checks.
5092                  * This fd == 0 will not be used with any syscall and will be reset to -1 eventually.
5093                  */
5094                 map->fd = 0;
5095         } else {
5096                 map->fd = bpf_map_create(def->type, map_name,
5097                                          def->key_size, def->value_size,
5098                                          def->max_entries, &create_attr);
5099         }
5100         if (map->fd < 0 && (create_attr.btf_key_type_id ||
5101                             create_attr.btf_value_type_id)) {
5102                 char *cp, errmsg[STRERR_BUFSIZE];
5103
5104                 err = -errno;
5105                 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
5106                 pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
5107                         map->name, cp, err);
5108                 create_attr.btf_fd = 0;
5109                 create_attr.btf_key_type_id = 0;
5110                 create_attr.btf_value_type_id = 0;
5111                 map->btf_key_type_id = 0;
5112                 map->btf_value_type_id = 0;
5113                 map->fd = bpf_map_create(def->type, map_name,
5114                                          def->key_size, def->value_size,
5115                                          def->max_entries, &create_attr);
5116         }
5117
5118         err = map->fd < 0 ? -errno : 0;
5119
5120         if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) {
5121                 if (obj->gen_loader)
5122                         map->inner_map->fd = -1;
5123                 bpf_map__destroy(map->inner_map);
5124                 zfree(&map->inner_map);
5125         }
5126
5127         return err;
5128 }
5129
5130 static int init_map_in_map_slots(struct bpf_object *obj, struct bpf_map *map)
5131 {
5132         const struct bpf_map *targ_map;
5133         unsigned int i;
5134         int fd, err = 0;
5135
5136         for (i = 0; i < map->init_slots_sz; i++) {
5137                 if (!map->init_slots[i])
5138                         continue;
5139
5140                 targ_map = map->init_slots[i];
5141                 fd = bpf_map__fd(targ_map);
5142
5143                 if (obj->gen_loader) {
5144                         bpf_gen__populate_outer_map(obj->gen_loader,
5145                                                     map - obj->maps, i,
5146                                                     targ_map - obj->maps);
5147                 } else {
5148                         err = bpf_map_update_elem(map->fd, &i, &fd, 0);
5149                 }
5150                 if (err) {
5151                         err = -errno;
5152                         pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n",
5153                                 map->name, i, targ_map->name, fd, err);
5154                         return err;
5155                 }
5156                 pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n",
5157                          map->name, i, targ_map->name, fd);
5158         }
5159
5160         zfree(&map->init_slots);
5161         map->init_slots_sz = 0;
5162
5163         return 0;
5164 }
5165
5166 static int init_prog_array_slots(struct bpf_object *obj, struct bpf_map *map)
5167 {
5168         const struct bpf_program *targ_prog;
5169         unsigned int i;
5170         int fd, err;
5171
5172         if (obj->gen_loader)
5173                 return -ENOTSUP;
5174
5175         for (i = 0; i < map->init_slots_sz; i++) {
5176                 if (!map->init_slots[i])
5177                         continue;
5178
5179                 targ_prog = map->init_slots[i];
5180                 fd = bpf_program__fd(targ_prog);
5181
5182                 err = bpf_map_update_elem(map->fd, &i, &fd, 0);
5183                 if (err) {
5184                         err = -errno;
5185                         pr_warn("map '%s': failed to initialize slot [%d] to prog '%s' fd=%d: %d\n",
5186                                 map->name, i, targ_prog->name, fd, err);
5187                         return err;
5188                 }
5189                 pr_debug("map '%s': slot [%d] set to prog '%s' fd=%d\n",
5190                          map->name, i, targ_prog->name, fd);
5191         }
5192
5193         zfree(&map->init_slots);
5194         map->init_slots_sz = 0;
5195
5196         return 0;
5197 }
5198
5199 static int bpf_object_init_prog_arrays(struct bpf_object *obj)
5200 {
5201         struct bpf_map *map;
5202         int i, err;
5203
5204         for (i = 0; i < obj->nr_maps; i++) {
5205                 map = &obj->maps[i];
5206
5207                 if (!map->init_slots_sz || map->def.type != BPF_MAP_TYPE_PROG_ARRAY)
5208                         continue;
5209
5210                 err = init_prog_array_slots(obj, map);
5211                 if (err < 0) {
5212                         zclose(map->fd);
5213                         return err;
5214                 }
5215         }
5216         return 0;
5217 }
5218
5219 static int map_set_def_max_entries(struct bpf_map *map)
5220 {
5221         if (map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !map->def.max_entries) {
5222                 int nr_cpus;
5223
5224                 nr_cpus = libbpf_num_possible_cpus();
5225                 if (nr_cpus < 0) {
5226                         pr_warn("map '%s': failed to determine number of system CPUs: %d\n",
5227                                 map->name, nr_cpus);
5228                         return nr_cpus;
5229                 }
5230                 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus);
5231                 map->def.max_entries = nr_cpus;
5232         }
5233
5234         return 0;
5235 }
5236
5237 static int
5238 bpf_object__create_maps(struct bpf_object *obj)
5239 {
5240         struct bpf_map *map;
5241         char *cp, errmsg[STRERR_BUFSIZE];
5242         unsigned int i, j;
5243         int err;
5244         bool retried;
5245
5246         for (i = 0; i < obj->nr_maps; i++) {
5247                 map = &obj->maps[i];
5248
5249                 /* To support old kernels, we skip creating global data maps
5250                  * (.rodata, .data, .kconfig, etc); later on, during program
5251                  * loading, if we detect that at least one of the to-be-loaded
5252                  * programs is referencing any global data map, we'll error
5253                  * out with program name and relocation index logged.
5254                  * This approach allows to accommodate Clang emitting
5255                  * unnecessary .rodata.str1.1 sections for string literals,
5256                  * but also it allows to have CO-RE applications that use
5257                  * global variables in some of BPF programs, but not others.
5258                  * If those global variable-using programs are not loaded at
5259                  * runtime due to bpf_program__set_autoload(prog, false),
5260                  * bpf_object loading will succeed just fine even on old
5261                  * kernels.
5262                  */
5263                 if (bpf_map__is_internal(map) && !kernel_supports(obj, FEAT_GLOBAL_DATA))
5264                         map->autocreate = false;
5265
5266                 if (!map->autocreate) {
5267                         pr_debug("map '%s': skipped auto-creating...\n", map->name);
5268                         continue;
5269                 }
5270
5271                 err = map_set_def_max_entries(map);
5272                 if (err)
5273                         goto err_out;
5274
5275                 retried = false;
5276 retry:
5277                 if (map->pin_path) {
5278                         err = bpf_object__reuse_map(map);
5279                         if (err) {
5280                                 pr_warn("map '%s': error reusing pinned map\n",
5281                                         map->name);
5282                                 goto err_out;
5283                         }
5284                         if (retried && map->fd < 0) {
5285                                 pr_warn("map '%s': cannot find pinned map\n",
5286                                         map->name);
5287                                 err = -ENOENT;
5288                                 goto err_out;
5289                         }
5290                 }
5291
5292                 if (map->fd >= 0) {
5293                         pr_debug("map '%s': skipping creation (preset fd=%d)\n",
5294                                  map->name, map->fd);
5295                 } else {
5296                         err = bpf_object__create_map(obj, map, false);
5297                         if (err)
5298                                 goto err_out;
5299
5300                         pr_debug("map '%s': created successfully, fd=%d\n",
5301                                  map->name, map->fd);
5302
5303                         if (bpf_map__is_internal(map)) {
5304                                 err = bpf_object__populate_internal_map(obj, map);
5305                                 if (err < 0) {
5306                                         zclose(map->fd);
5307                                         goto err_out;
5308                                 }
5309                         }
5310
5311                         if (map->init_slots_sz && map->def.type != BPF_MAP_TYPE_PROG_ARRAY) {
5312                                 err = init_map_in_map_slots(obj, map);
5313                                 if (err < 0) {
5314                                         zclose(map->fd);
5315                                         goto err_out;
5316                                 }
5317                         }
5318                 }
5319
5320                 if (map->pin_path && !map->pinned) {
5321                         err = bpf_map__pin(map, NULL);
5322                         if (err) {
5323                                 zclose(map->fd);
5324                                 if (!retried && err == -EEXIST) {
5325                                         retried = true;
5326                                         goto retry;
5327                                 }
5328                                 pr_warn("map '%s': failed to auto-pin at '%s': %d\n",
5329                                         map->name, map->pin_path, err);
5330                                 goto err_out;
5331                         }
5332                 }
5333         }
5334
5335         return 0;
5336
5337 err_out:
5338         cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
5339         pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err);
5340         pr_perm_msg(err);
5341         for (j = 0; j < i; j++)
5342                 zclose(obj->maps[j].fd);
5343         return err;
5344 }
5345
5346 static bool bpf_core_is_flavor_sep(const char *s)
5347 {
5348         /* check X___Y name pattern, where X and Y are not underscores */
5349         return s[0] != '_' &&                                 /* X */
5350                s[1] == '_' && s[2] == '_' && s[3] == '_' &&   /* ___ */
5351                s[4] != '_';                                   /* Y */
5352 }
5353
5354 /* Given 'some_struct_name___with_flavor' return the length of a name prefix
5355  * before last triple underscore. Struct name part after last triple
5356  * underscore is ignored by BPF CO-RE relocation during relocation matching.
5357  */
5358 size_t bpf_core_essential_name_len(const char *name)
5359 {
5360         size_t n = strlen(name);
5361         int i;
5362
5363         for (i = n - 5; i >= 0; i--) {
5364                 if (bpf_core_is_flavor_sep(name + i))
5365                         return i + 1;
5366         }
5367         return n;
5368 }
5369
5370 void bpf_core_free_cands(struct bpf_core_cand_list *cands)
5371 {
5372         if (!cands)
5373                 return;
5374
5375         free(cands->cands);
5376         free(cands);
5377 }
5378
5379 int bpf_core_add_cands(struct bpf_core_cand *local_cand,
5380                        size_t local_essent_len,
5381                        const struct btf *targ_btf,
5382                        const char *targ_btf_name,
5383                        int targ_start_id,
5384                        struct bpf_core_cand_list *cands)
5385 {
5386         struct bpf_core_cand *new_cands, *cand;
5387         const struct btf_type *t, *local_t;
5388         const char *targ_name, *local_name;
5389         size_t targ_essent_len;
5390         int n, i;
5391
5392         local_t = btf__type_by_id(local_cand->btf, local_cand->id);
5393         local_name = btf__str_by_offset(local_cand->btf, local_t->name_off);
5394
5395         n = btf__type_cnt(targ_btf);
5396         for (i = targ_start_id; i < n; i++) {
5397                 t = btf__type_by_id(targ_btf, i);
5398                 if (!btf_kind_core_compat(t, local_t))
5399                         continue;
5400
5401                 targ_name = btf__name_by_offset(targ_btf, t->name_off);
5402                 if (str_is_empty(targ_name))
5403                         continue;
5404
5405                 targ_essent_len = bpf_core_essential_name_len(targ_name);
5406                 if (targ_essent_len != local_essent_len)
5407                         continue;
5408
5409                 if (strncmp(local_name, targ_name, local_essent_len) != 0)
5410                         continue;
5411
5412                 pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n",
5413                          local_cand->id, btf_kind_str(local_t),
5414                          local_name, i, btf_kind_str(t), targ_name,
5415                          targ_btf_name);
5416                 new_cands = libbpf_reallocarray(cands->cands, cands->len + 1,
5417                                               sizeof(*cands->cands));
5418                 if (!new_cands)
5419                         return -ENOMEM;
5420
5421                 cand = &new_cands[cands->len];
5422                 cand->btf = targ_btf;
5423                 cand->id = i;
5424
5425                 cands->cands = new_cands;
5426                 cands->len++;
5427         }
5428         return 0;
5429 }
5430
5431 static int load_module_btfs(struct bpf_object *obj)
5432 {
5433         struct bpf_btf_info info;
5434         struct module_btf *mod_btf;
5435         struct btf *btf;
5436         char name[64];
5437         __u32 id = 0, len;
5438         int err, fd;
5439
5440         if (obj->btf_modules_loaded)
5441                 return 0;
5442
5443         if (obj->gen_loader)
5444                 return 0;
5445
5446         /* don't do this again, even if we find no module BTFs */
5447         obj->btf_modules_loaded = true;
5448
5449         /* kernel too old to support module BTFs */
5450         if (!kernel_supports(obj, FEAT_MODULE_BTF))
5451                 return 0;
5452
5453         while (true) {
5454                 err = bpf_btf_get_next_id(id, &id);
5455                 if (err && errno == ENOENT)
5456                         return 0;
5457                 if (err) {
5458                         err = -errno;
5459                         pr_warn("failed to iterate BTF objects: %d\n", err);
5460                         return err;
5461                 }
5462
5463                 fd = bpf_btf_get_fd_by_id(id);
5464                 if (fd < 0) {
5465                         if (errno == ENOENT)
5466                                 continue; /* expected race: BTF was unloaded */
5467                         err = -errno;
5468                         pr_warn("failed to get BTF object #%d FD: %d\n", id, err);
5469                         return err;
5470                 }
5471
5472                 len = sizeof(info);
5473                 memset(&info, 0, sizeof(info));
5474                 info.name = ptr_to_u64(name);
5475                 info.name_len = sizeof(name);
5476
5477                 err = bpf_obj_get_info_by_fd(fd, &info, &len);
5478                 if (err) {
5479                         err = -errno;
5480                         pr_warn("failed to get BTF object #%d info: %d\n", id, err);
5481                         goto err_out;
5482                 }
5483
5484                 /* ignore non-module BTFs */
5485                 if (!info.kernel_btf || strcmp(name, "vmlinux") == 0) {
5486                         close(fd);
5487                         continue;
5488                 }
5489
5490                 btf = btf_get_from_fd(fd, obj->btf_vmlinux);
5491                 err = libbpf_get_error(btf);
5492                 if (err) {
5493                         pr_warn("failed to load module [%s]'s BTF object #%d: %d\n",
5494                                 name, id, err);
5495                         goto err_out;
5496                 }
5497
5498                 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap,
5499                                         sizeof(*obj->btf_modules), obj->btf_module_cnt + 1);
5500                 if (err)
5501                         goto err_out;
5502
5503                 mod_btf = &obj->btf_modules[obj->btf_module_cnt++];
5504
5505                 mod_btf->btf = btf;
5506                 mod_btf->id = id;
5507                 mod_btf->fd = fd;
5508                 mod_btf->name = strdup(name);
5509                 if (!mod_btf->name) {
5510                         err = -ENOMEM;
5511                         goto err_out;
5512                 }
5513                 continue;
5514
5515 err_out:
5516                 close(fd);
5517                 return err;
5518         }
5519
5520         return 0;
5521 }
5522
5523 static struct bpf_core_cand_list *
5524 bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 local_type_id)
5525 {
5526         struct bpf_core_cand local_cand = {};
5527         struct bpf_core_cand_list *cands;
5528         const struct btf *main_btf;
5529         const struct btf_type *local_t;
5530         const char *local_name;
5531         size_t local_essent_len;
5532         int err, i;
5533
5534         local_cand.btf = local_btf;
5535         local_cand.id = local_type_id;
5536         local_t = btf__type_by_id(local_btf, local_type_id);
5537         if (!local_t)
5538                 return ERR_PTR(-EINVAL);
5539
5540         local_name = btf__name_by_offset(local_btf, local_t->name_off);
5541         if (str_is_empty(local_name))
5542                 return ERR_PTR(-EINVAL);
5543         local_essent_len = bpf_core_essential_name_len(local_name);
5544
5545         cands = calloc(1, sizeof(*cands));
5546         if (!cands)
5547                 return ERR_PTR(-ENOMEM);
5548
5549         /* Attempt to find target candidates in vmlinux BTF first */
5550         main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux;
5551         err = bpf_core_add_cands(&local_cand, local_essent_len, main_btf, "vmlinux", 1, cands);
5552         if (err)
5553                 goto err_out;
5554
5555         /* if vmlinux BTF has any candidate, don't got for module BTFs */
5556         if (cands->len)
5557                 return cands;
5558
5559         /* if vmlinux BTF was overridden, don't attempt to load module BTFs */
5560         if (obj->btf_vmlinux_override)
5561                 return cands;
5562
5563         /* now look through module BTFs, trying to still find candidates */
5564         err = load_module_btfs(obj);
5565         if (err)
5566                 goto err_out;
5567
5568         for (i = 0; i < obj->btf_module_cnt; i++) {
5569                 err = bpf_core_add_cands(&local_cand, local_essent_len,
5570                                          obj->btf_modules[i].btf,
5571                                          obj->btf_modules[i].name,
5572                                          btf__type_cnt(obj->btf_vmlinux),
5573                                          cands);
5574                 if (err)
5575                         goto err_out;
5576         }
5577
5578         return cands;
5579 err_out:
5580         bpf_core_free_cands(cands);
5581         return ERR_PTR(err);
5582 }
5583
5584 /* Check local and target types for compatibility. This check is used for
5585  * type-based CO-RE relocations and follow slightly different rules than
5586  * field-based relocations. This function assumes that root types were already
5587  * checked for name match. Beyond that initial root-level name check, names
5588  * are completely ignored. Compatibility rules are as follows:
5589  *   - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but
5590  *     kind should match for local and target types (i.e., STRUCT is not
5591  *     compatible with UNION);
5592  *   - for ENUMs, the size is ignored;
5593  *   - for INT, size and signedness are ignored;
5594  *   - for ARRAY, dimensionality is ignored, element types are checked for
5595  *     compatibility recursively;
5596  *   - CONST/VOLATILE/RESTRICT modifiers are ignored;
5597  *   - TYPEDEFs/PTRs are compatible if types they pointing to are compatible;
5598  *   - FUNC_PROTOs are compatible if they have compatible signature: same
5599  *     number of input args and compatible return and argument types.
5600  * These rules are not set in stone and probably will be adjusted as we get
5601  * more experience with using BPF CO-RE relocations.
5602  */
5603 int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id,
5604                               const struct btf *targ_btf, __u32 targ_id)
5605 {
5606         return __bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id, 32);
5607 }
5608
5609 int bpf_core_types_match(const struct btf *local_btf, __u32 local_id,
5610                          const struct btf *targ_btf, __u32 targ_id)
5611 {
5612         return __bpf_core_types_match(local_btf, local_id, targ_btf, targ_id, false, 32);
5613 }
5614
5615 static size_t bpf_core_hash_fn(const long key, void *ctx)
5616 {
5617         return key;
5618 }
5619
5620 static bool bpf_core_equal_fn(const long k1, const long k2, void *ctx)
5621 {
5622         return k1 == k2;
5623 }
5624
5625 static int record_relo_core(struct bpf_program *prog,
5626                             const struct bpf_core_relo *core_relo, int insn_idx)
5627 {
5628         struct reloc_desc *relos, *relo;
5629
5630         relos = libbpf_reallocarray(prog->reloc_desc,
5631                                     prog->nr_reloc + 1, sizeof(*relos));
5632         if (!relos)
5633                 return -ENOMEM;
5634         relo = &relos[prog->nr_reloc];
5635         relo->type = RELO_CORE;
5636         relo->insn_idx = insn_idx;
5637         relo->core_relo = core_relo;
5638         prog->reloc_desc = relos;
5639         prog->nr_reloc++;
5640         return 0;
5641 }
5642
5643 static const struct bpf_core_relo *find_relo_core(struct bpf_program *prog, int insn_idx)
5644 {
5645         struct reloc_desc *relo;
5646         int i;
5647
5648         for (i = 0; i < prog->nr_reloc; i++) {
5649                 relo = &prog->reloc_desc[i];
5650                 if (relo->type != RELO_CORE || relo->insn_idx != insn_idx)
5651                         continue;
5652
5653                 return relo->core_relo;
5654         }
5655
5656         return NULL;
5657 }
5658
5659 static int bpf_core_resolve_relo(struct bpf_program *prog,
5660                                  const struct bpf_core_relo *relo,
5661                                  int relo_idx,
5662                                  const struct btf *local_btf,
5663                                  struct hashmap *cand_cache,
5664                                  struct bpf_core_relo_res *targ_res)
5665 {
5666         struct bpf_core_spec specs_scratch[3] = {};
5667         struct bpf_core_cand_list *cands = NULL;
5668         const char *prog_name = prog->name;
5669         const struct btf_type *local_type;
5670         const char *local_name;
5671         __u32 local_id = relo->type_id;
5672         int err;
5673
5674         local_type = btf__type_by_id(local_btf, local_id);
5675         if (!local_type)
5676                 return -EINVAL;
5677
5678         local_name = btf__name_by_offset(local_btf, local_type->name_off);
5679         if (!local_name)
5680                 return -EINVAL;
5681
5682         if (relo->kind != BPF_CORE_TYPE_ID_LOCAL &&
5683             !hashmap__find(cand_cache, local_id, &cands)) {
5684                 cands = bpf_core_find_cands(prog->obj, local_btf, local_id);
5685                 if (IS_ERR(cands)) {
5686                         pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n",
5687                                 prog_name, relo_idx, local_id, btf_kind_str(local_type),
5688                                 local_name, PTR_ERR(cands));
5689                         return PTR_ERR(cands);
5690                 }
5691                 err = hashmap__set(cand_cache, local_id, cands, NULL, NULL);
5692                 if (err) {
5693                         bpf_core_free_cands(cands);
5694                         return err;
5695                 }
5696         }
5697
5698         return bpf_core_calc_relo_insn(prog_name, relo, relo_idx, local_btf, cands, specs_scratch,
5699                                        targ_res);
5700 }
5701
5702 static int
5703 bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path)
5704 {
5705         const struct btf_ext_info_sec *sec;
5706         struct bpf_core_relo_res targ_res;
5707         const struct bpf_core_relo *rec;
5708         const struct btf_ext_info *seg;
5709         struct hashmap_entry *entry;
5710         struct hashmap *cand_cache = NULL;
5711         struct bpf_program *prog;
5712         struct bpf_insn *insn;
5713         const char *sec_name;
5714         int i, err = 0, insn_idx, sec_idx, sec_num;
5715
5716         if (obj->btf_ext->core_relo_info.len == 0)
5717                 return 0;
5718
5719         if (targ_btf_path) {
5720                 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL);
5721                 err = libbpf_get_error(obj->btf_vmlinux_override);
5722                 if (err) {
5723                         pr_warn("failed to parse target BTF: %d\n", err);
5724                         return err;
5725                 }
5726         }
5727
5728         cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL);
5729         if (IS_ERR(cand_cache)) {
5730                 err = PTR_ERR(cand_cache);
5731                 goto out;
5732         }
5733
5734         seg = &obj->btf_ext->core_relo_info;
5735         sec_num = 0;
5736         for_each_btf_ext_sec(seg, sec) {
5737                 sec_idx = seg->sec_idxs[sec_num];
5738                 sec_num++;
5739
5740                 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
5741                 if (str_is_empty(sec_name)) {
5742                         err = -EINVAL;
5743                         goto out;
5744                 }
5745
5746                 pr_debug("sec '%s': found %d CO-RE relocations\n", sec_name, sec->num_info);
5747
5748                 for_each_btf_ext_rec(seg, sec, i, rec) {
5749                         if (rec->insn_off % BPF_INSN_SZ)
5750                                 return -EINVAL;
5751                         insn_idx = rec->insn_off / BPF_INSN_SZ;
5752                         prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
5753                         if (!prog) {
5754                                 /* When __weak subprog is "overridden" by another instance
5755                                  * of the subprog from a different object file, linker still
5756                                  * appends all the .BTF.ext info that used to belong to that
5757                                  * eliminated subprogram.
5758                                  * This is similar to what x86-64 linker does for relocations.
5759                                  * So just ignore such relocations just like we ignore
5760                                  * subprog instructions when discovering subprograms.
5761                                  */
5762                                 pr_debug("sec '%s': skipping CO-RE relocation #%d for insn #%d belonging to eliminated weak subprogram\n",
5763                                          sec_name, i, insn_idx);
5764                                 continue;
5765                         }
5766                         /* no need to apply CO-RE relocation if the program is
5767                          * not going to be loaded
5768                          */
5769                         if (!prog->autoload)
5770                                 continue;
5771
5772                         /* adjust insn_idx from section frame of reference to the local
5773                          * program's frame of reference; (sub-)program code is not yet
5774                          * relocated, so it's enough to just subtract in-section offset
5775                          */
5776                         insn_idx = insn_idx - prog->sec_insn_off;
5777                         if (insn_idx >= prog->insns_cnt)
5778                                 return -EINVAL;
5779                         insn = &prog->insns[insn_idx];
5780
5781                         err = record_relo_core(prog, rec, insn_idx);
5782                         if (err) {
5783                                 pr_warn("prog '%s': relo #%d: failed to record relocation: %d\n",
5784                                         prog->name, i, err);
5785                                 goto out;
5786                         }
5787
5788                         if (prog->obj->gen_loader)
5789                                 continue;
5790
5791                         err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res);
5792                         if (err) {
5793                                 pr_warn("prog '%s': relo #%d: failed to relocate: %d\n",
5794                                         prog->name, i, err);
5795                                 goto out;
5796                         }
5797
5798                         err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res);
5799                         if (err) {
5800                                 pr_warn("prog '%s': relo #%d: failed to patch insn #%u: %d\n",
5801                                         prog->name, i, insn_idx, err);
5802                                 goto out;
5803                         }
5804                 }
5805         }
5806
5807 out:
5808         /* obj->btf_vmlinux and module BTFs are freed after object load */
5809         btf__free(obj->btf_vmlinux_override);
5810         obj->btf_vmlinux_override = NULL;
5811
5812         if (!IS_ERR_OR_NULL(cand_cache)) {
5813                 hashmap__for_each_entry(cand_cache, entry, i) {
5814                         bpf_core_free_cands(entry->pvalue);
5815                 }
5816                 hashmap__free(cand_cache);
5817         }
5818         return err;
5819 }
5820
5821 /* base map load ldimm64 special constant, used also for log fixup logic */
5822 #define MAP_LDIMM64_POISON_BASE 2001000000
5823 #define MAP_LDIMM64_POISON_PFX "200100"
5824
5825 static void poison_map_ldimm64(struct bpf_program *prog, int relo_idx,
5826                                int insn_idx, struct bpf_insn *insn,
5827                                int map_idx, const struct bpf_map *map)
5828 {
5829         int i;
5830
5831         pr_debug("prog '%s': relo #%d: poisoning insn #%d that loads map #%d '%s'\n",
5832                  prog->name, relo_idx, insn_idx, map_idx, map->name);
5833
5834         /* we turn single ldimm64 into two identical invalid calls */
5835         for (i = 0; i < 2; i++) {
5836                 insn->code = BPF_JMP | BPF_CALL;
5837                 insn->dst_reg = 0;
5838                 insn->src_reg = 0;
5839                 insn->off = 0;
5840                 /* if this instruction is reachable (not a dead code),
5841                  * verifier will complain with something like:
5842                  * invalid func unknown#2001000123
5843                  * where lower 123 is map index into obj->maps[] array
5844                  */
5845                 insn->imm = MAP_LDIMM64_POISON_BASE + map_idx;
5846
5847                 insn++;
5848         }
5849 }
5850
5851 /* Relocate data references within program code:
5852  *  - map references;
5853  *  - global variable references;
5854  *  - extern references.
5855  */
5856 static int
5857 bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog)
5858 {
5859         int i;
5860
5861         for (i = 0; i < prog->nr_reloc; i++) {
5862                 struct reloc_desc *relo = &prog->reloc_desc[i];
5863                 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
5864                 const struct bpf_map *map;
5865                 struct extern_desc *ext;
5866
5867                 switch (relo->type) {
5868                 case RELO_LD64:
5869                         map = &obj->maps[relo->map_idx];
5870                         if (obj->gen_loader) {
5871                                 insn[0].src_reg = BPF_PSEUDO_MAP_IDX;
5872                                 insn[0].imm = relo->map_idx;
5873                         } else if (map->autocreate) {
5874                                 insn[0].src_reg = BPF_PSEUDO_MAP_FD;
5875                                 insn[0].imm = map->fd;
5876                         } else {
5877                                 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
5878                                                    relo->map_idx, map);
5879                         }
5880                         break;
5881                 case RELO_DATA:
5882                         map = &obj->maps[relo->map_idx];
5883                         insn[1].imm = insn[0].imm + relo->sym_off;
5884                         if (obj->gen_loader) {
5885                                 insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE;
5886                                 insn[0].imm = relo->map_idx;
5887                         } else if (map->autocreate) {
5888                                 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
5889                                 insn[0].imm = map->fd;
5890                         } else {
5891                                 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
5892                                                    relo->map_idx, map);
5893                         }
5894                         break;
5895                 case RELO_EXTERN_VAR:
5896                         ext = &obj->externs[relo->sym_off];
5897                         if (ext->type == EXT_KCFG) {
5898                                 if (obj->gen_loader) {
5899                                         insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE;
5900                                         insn[0].imm = obj->kconfig_map_idx;
5901                                 } else {
5902                                         insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
5903                                         insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;
5904                                 }
5905                                 insn[1].imm = ext->kcfg.data_off;
5906                         } else /* EXT_KSYM */ {
5907                                 if (ext->ksym.type_id && ext->is_set) { /* typed ksyms */
5908                                         insn[0].src_reg = BPF_PSEUDO_BTF_ID;
5909                                         insn[0].imm = ext->ksym.kernel_btf_id;
5910                                         insn[1].imm = ext->ksym.kernel_btf_obj_fd;
5911                                 } else { /* typeless ksyms or unresolved typed ksyms */
5912                                         insn[0].imm = (__u32)ext->ksym.addr;
5913                                         insn[1].imm = ext->ksym.addr >> 32;
5914                                 }
5915                         }
5916                         break;
5917                 case RELO_EXTERN_FUNC:
5918                         ext = &obj->externs[relo->sym_off];
5919                         insn[0].src_reg = BPF_PSEUDO_KFUNC_CALL;
5920                         if (ext->is_set) {
5921                                 insn[0].imm = ext->ksym.kernel_btf_id;
5922                                 insn[0].off = ext->ksym.btf_fd_idx;
5923                         } else { /* unresolved weak kfunc */
5924                                 insn[0].imm = 0;
5925                                 insn[0].off = 0;
5926                         }
5927                         break;
5928                 case RELO_SUBPROG_ADDR:
5929                         if (insn[0].src_reg != BPF_PSEUDO_FUNC) {
5930                                 pr_warn("prog '%s': relo #%d: bad insn\n",
5931                                         prog->name, i);
5932                                 return -EINVAL;
5933                         }
5934                         /* handled already */
5935                         break;
5936                 case RELO_CALL:
5937                         /* handled already */
5938                         break;
5939                 case RELO_CORE:
5940                         /* will be handled by bpf_program_record_relos() */
5941                         break;
5942                 default:
5943                         pr_warn("prog '%s': relo #%d: bad relo type %d\n",
5944                                 prog->name, i, relo->type);
5945                         return -EINVAL;
5946                 }
5947         }
5948
5949         return 0;
5950 }
5951
5952 static int adjust_prog_btf_ext_info(const struct bpf_object *obj,
5953                                     const struct bpf_program *prog,
5954                                     const struct btf_ext_info *ext_info,
5955                                     void **prog_info, __u32 *prog_rec_cnt,
5956                                     __u32 *prog_rec_sz)
5957 {
5958         void *copy_start = NULL, *copy_end = NULL;
5959         void *rec, *rec_end, *new_prog_info;
5960         const struct btf_ext_info_sec *sec;
5961         size_t old_sz, new_sz;
5962         int i, sec_num, sec_idx, off_adj;
5963
5964         sec_num = 0;
5965         for_each_btf_ext_sec(ext_info, sec) {
5966                 sec_idx = ext_info->sec_idxs[sec_num];
5967                 sec_num++;
5968                 if (prog->sec_idx != sec_idx)
5969                         continue;
5970
5971                 for_each_btf_ext_rec(ext_info, sec, i, rec) {
5972                         __u32 insn_off = *(__u32 *)rec / BPF_INSN_SZ;
5973
5974                         if (insn_off < prog->sec_insn_off)
5975                                 continue;
5976                         if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt)
5977                                 break;
5978
5979                         if (!copy_start)
5980                                 copy_start = rec;
5981                         copy_end = rec + ext_info->rec_size;
5982                 }
5983
5984                 if (!copy_start)
5985                         return -ENOENT;
5986
5987                 /* append func/line info of a given (sub-)program to the main
5988                  * program func/line info
5989                  */
5990                 old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size;
5991                 new_sz = old_sz + (copy_end - copy_start);
5992                 new_prog_info = realloc(*prog_info, new_sz);
5993                 if (!new_prog_info)
5994                         return -ENOMEM;
5995                 *prog_info = new_prog_info;
5996                 *prog_rec_cnt = new_sz / ext_info->rec_size;
5997                 memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start);
5998
5999                 /* Kernel instruction offsets are in units of 8-byte
6000                  * instructions, while .BTF.ext instruction offsets generated
6001                  * by Clang are in units of bytes. So convert Clang offsets
6002                  * into kernel offsets and adjust offset according to program
6003                  * relocated position.
6004                  */
6005                 off_adj = prog->sub_insn_off - prog->sec_insn_off;
6006                 rec = new_prog_info + old_sz;
6007                 rec_end = new_prog_info + new_sz;
6008                 for (; rec < rec_end; rec += ext_info->rec_size) {
6009                         __u32 *insn_off = rec;
6010
6011                         *insn_off = *insn_off / BPF_INSN_SZ + off_adj;
6012                 }
6013                 *prog_rec_sz = ext_info->rec_size;
6014                 return 0;
6015         }
6016
6017         return -ENOENT;
6018 }
6019
6020 static int
6021 reloc_prog_func_and_line_info(const struct bpf_object *obj,
6022                               struct bpf_program *main_prog,
6023                               const struct bpf_program *prog)
6024 {
6025         int err;
6026
6027         /* no .BTF.ext relocation if .BTF.ext is missing or kernel doesn't
6028          * supprot func/line info
6029          */
6030         if (!obj->btf_ext || !kernel_supports(obj, FEAT_BTF_FUNC))
6031                 return 0;
6032
6033         /* only attempt func info relocation if main program's func_info
6034          * relocation was successful
6035          */
6036         if (main_prog != prog && !main_prog->func_info)
6037                 goto line_info;
6038
6039         err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info,
6040                                        &main_prog->func_info,
6041                                        &main_prog->func_info_cnt,
6042                                        &main_prog->func_info_rec_size);
6043         if (err) {
6044                 if (err != -ENOENT) {
6045                         pr_warn("prog '%s': error relocating .BTF.ext function info: %d\n",
6046                                 prog->name, err);
6047                         return err;
6048                 }
6049                 if (main_prog->func_info) {
6050                         /*
6051                          * Some info has already been found but has problem
6052                          * in the last btf_ext reloc. Must have to error out.
6053                          */
6054                         pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name);
6055                         return err;
6056                 }
6057                 /* Have problem loading the very first info. Ignore the rest. */
6058                 pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext func info.\n",
6059                         prog->name);
6060         }
6061
6062 line_info:
6063         /* don't relocate line info if main program's relocation failed */
6064         if (main_prog != prog && !main_prog->line_info)
6065                 return 0;
6066
6067         err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info,
6068                                        &main_prog->line_info,
6069                                        &main_prog->line_info_cnt,
6070                                        &main_prog->line_info_rec_size);
6071         if (err) {
6072                 if (err != -ENOENT) {
6073                         pr_warn("prog '%s': error relocating .BTF.ext line info: %d\n",
6074                                 prog->name, err);
6075                         return err;
6076                 }
6077                 if (main_prog->line_info) {
6078                         /*
6079                          * Some info has already been found but has problem
6080                          * in the last btf_ext reloc. Must have to error out.
6081                          */
6082                         pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name);
6083                         return err;
6084                 }
6085                 /* Have problem loading the very first info. Ignore the rest. */
6086                 pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line info.\n",
6087                         prog->name);
6088         }
6089         return 0;
6090 }
6091
6092 static int cmp_relo_by_insn_idx(const void *key, const void *elem)
6093 {
6094         size_t insn_idx = *(const size_t *)key;
6095         const struct reloc_desc *relo = elem;
6096
6097         if (insn_idx == relo->insn_idx)
6098                 return 0;
6099         return insn_idx < relo->insn_idx ? -1 : 1;
6100 }
6101
6102 static struct reloc_desc *find_prog_insn_relo(const struct bpf_program *prog, size_t insn_idx)
6103 {
6104         if (!prog->nr_reloc)
6105                 return NULL;
6106         return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc,
6107                        sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx);
6108 }
6109
6110 static int append_subprog_relos(struct bpf_program *main_prog, struct bpf_program *subprog)
6111 {
6112         int new_cnt = main_prog->nr_reloc + subprog->nr_reloc;
6113         struct reloc_desc *relos;
6114         int i;
6115
6116         if (main_prog == subprog)
6117                 return 0;
6118         relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos));
6119         if (!relos)
6120                 return -ENOMEM;
6121         if (subprog->nr_reloc)
6122                 memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc,
6123                        sizeof(*relos) * subprog->nr_reloc);
6124
6125         for (i = main_prog->nr_reloc; i < new_cnt; i++)
6126                 relos[i].insn_idx += subprog->sub_insn_off;
6127         /* After insn_idx adjustment the 'relos' array is still sorted
6128          * by insn_idx and doesn't break bsearch.
6129          */
6130         main_prog->reloc_desc = relos;
6131         main_prog->nr_reloc = new_cnt;
6132         return 0;
6133 }
6134
6135 static int
6136 bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog,
6137                        struct bpf_program *prog)
6138 {
6139         size_t sub_insn_idx, insn_idx, new_cnt;
6140         struct bpf_program *subprog;
6141         struct bpf_insn *insns, *insn;
6142         struct reloc_desc *relo;
6143         int err;
6144
6145         err = reloc_prog_func_and_line_info(obj, main_prog, prog);
6146         if (err)
6147                 return err;
6148
6149         for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) {
6150                 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6151                 if (!insn_is_subprog_call(insn) && !insn_is_pseudo_func(insn))
6152                         continue;
6153
6154                 relo = find_prog_insn_relo(prog, insn_idx);
6155                 if (relo && relo->type == RELO_EXTERN_FUNC)
6156                         /* kfunc relocations will be handled later
6157                          * in bpf_object__relocate_data()
6158                          */
6159                         continue;
6160                 if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) {
6161                         pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n",
6162                                 prog->name, insn_idx, relo->type);
6163                         return -LIBBPF_ERRNO__RELOC;
6164                 }
6165                 if (relo) {
6166                         /* sub-program instruction index is a combination of
6167                          * an offset of a symbol pointed to by relocation and
6168                          * call instruction's imm field; for global functions,
6169                          * call always has imm = -1, but for static functions
6170                          * relocation is against STT_SECTION and insn->imm
6171                          * points to a start of a static function
6172                          *
6173                          * for subprog addr relocation, the relo->sym_off + insn->imm is
6174                          * the byte offset in the corresponding section.
6175                          */
6176                         if (relo->type == RELO_CALL)
6177                                 sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1;
6178                         else
6179                                 sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ;
6180                 } else if (insn_is_pseudo_func(insn)) {
6181                         /*
6182                          * RELO_SUBPROG_ADDR relo is always emitted even if both
6183                          * functions are in the same section, so it shouldn't reach here.
6184                          */
6185                         pr_warn("prog '%s': missing subprog addr relo for insn #%zu\n",
6186                                 prog->name, insn_idx);
6187                         return -LIBBPF_ERRNO__RELOC;
6188                 } else {
6189                         /* if subprogram call is to a static function within
6190                          * the same ELF section, there won't be any relocation
6191                          * emitted, but it also means there is no additional
6192                          * offset necessary, insns->imm is relative to
6193                          * instruction's original position within the section
6194                          */
6195                         sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1;
6196                 }
6197
6198                 /* we enforce that sub-programs should be in .text section */
6199                 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx);
6200                 if (!subprog) {
6201                         pr_warn("prog '%s': no .text section found yet sub-program call exists\n",
6202                                 prog->name);
6203                         return -LIBBPF_ERRNO__RELOC;
6204                 }
6205
6206                 /* if it's the first call instruction calling into this
6207                  * subprogram (meaning this subprog hasn't been processed
6208                  * yet) within the context of current main program:
6209                  *   - append it at the end of main program's instructions blog;
6210                  *   - process is recursively, while current program is put on hold;
6211                  *   - if that subprogram calls some other not yet processes
6212                  *   subprogram, same thing will happen recursively until
6213                  *   there are no more unprocesses subprograms left to append
6214                  *   and relocate.
6215                  */
6216                 if (subprog->sub_insn_off == 0) {
6217                         subprog->sub_insn_off = main_prog->insns_cnt;
6218
6219                         new_cnt = main_prog->insns_cnt + subprog->insns_cnt;
6220                         insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns));
6221                         if (!insns) {
6222                                 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name);
6223                                 return -ENOMEM;
6224                         }
6225                         main_prog->insns = insns;
6226                         main_prog->insns_cnt = new_cnt;
6227
6228                         memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns,
6229                                subprog->insns_cnt * sizeof(*insns));
6230
6231                         pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n",
6232                                  main_prog->name, subprog->insns_cnt, subprog->name);
6233
6234                         /* The subprog insns are now appended. Append its relos too. */
6235                         err = append_subprog_relos(main_prog, subprog);
6236                         if (err)
6237                                 return err;
6238                         err = bpf_object__reloc_code(obj, main_prog, subprog);
6239                         if (err)
6240                                 return err;
6241                 }
6242
6243                 /* main_prog->insns memory could have been re-allocated, so
6244                  * calculate pointer again
6245                  */
6246                 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6247                 /* calculate correct instruction position within current main
6248                  * prog; each main prog can have a different set of
6249                  * subprograms appended (potentially in different order as
6250                  * well), so position of any subprog can be different for
6251                  * different main programs
6252                  */
6253                 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1;
6254
6255                 pr_debug("prog '%s': insn #%zu relocated, imm %d points to subprog '%s' (now at %zu offset)\n",
6256                          prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off);
6257         }
6258
6259         return 0;
6260 }
6261
6262 /*
6263  * Relocate sub-program calls.
6264  *
6265  * Algorithm operates as follows. Each entry-point BPF program (referred to as
6266  * main prog) is processed separately. For each subprog (non-entry functions,
6267  * that can be called from either entry progs or other subprogs) gets their
6268  * sub_insn_off reset to zero. This serves as indicator that this subprogram
6269  * hasn't been yet appended and relocated within current main prog. Once its
6270  * relocated, sub_insn_off will point at the position within current main prog
6271  * where given subprog was appended. This will further be used to relocate all
6272  * the call instructions jumping into this subprog.
6273  *
6274  * We start with main program and process all call instructions. If the call
6275  * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off
6276  * is zero), subprog instructions are appended at the end of main program's
6277  * instruction array. Then main program is "put on hold" while we recursively
6278  * process newly appended subprogram. If that subprogram calls into another
6279  * subprogram that hasn't been appended, new subprogram is appended again to
6280  * the *main* prog's instructions (subprog's instructions are always left
6281  * untouched, as they need to be in unmodified state for subsequent main progs
6282  * and subprog instructions are always sent only as part of a main prog) and
6283  * the process continues recursively. Once all the subprogs called from a main
6284  * prog or any of its subprogs are appended (and relocated), all their
6285  * positions within finalized instructions array are known, so it's easy to
6286  * rewrite call instructions with correct relative offsets, corresponding to
6287  * desired target subprog.
6288  *
6289  * Its important to realize that some subprogs might not be called from some
6290  * main prog and any of its called/used subprogs. Those will keep their
6291  * subprog->sub_insn_off as zero at all times and won't be appended to current
6292  * main prog and won't be relocated within the context of current main prog.
6293  * They might still be used from other main progs later.
6294  *
6295  * Visually this process can be shown as below. Suppose we have two main
6296  * programs mainA and mainB and BPF object contains three subprogs: subA,
6297  * subB, and subC. mainA calls only subA, mainB calls only subC, but subA and
6298  * subC both call subB:
6299  *
6300  *        +--------+ +-------+
6301  *        |        v v       |
6302  *     +--+---+ +--+-+-+ +---+--+
6303  *     | subA | | subB | | subC |
6304  *     +--+---+ +------+ +---+--+
6305  *        ^                  ^
6306  *        |                  |
6307  *    +---+-------+   +------+----+
6308  *    |   mainA   |   |   mainB   |
6309  *    +-----------+   +-----------+
6310  *
6311  * We'll start relocating mainA, will find subA, append it and start
6312  * processing sub A recursively:
6313  *
6314  *    +-----------+------+
6315  *    |   mainA   | subA |
6316  *    +-----------+------+
6317  *
6318  * At this point we notice that subB is used from subA, so we append it and
6319  * relocate (there are no further subcalls from subB):
6320  *
6321  *    +-----------+------+------+
6322  *    |   mainA   | subA | subB |
6323  *    +-----------+------+------+
6324  *
6325  * At this point, we relocate subA calls, then go one level up and finish with
6326  * relocatin mainA calls. mainA is done.
6327  *
6328  * For mainB process is similar but results in different order. We start with
6329  * mainB and skip subA and subB, as mainB never calls them (at least
6330  * directly), but we see subC is needed, so we append and start processing it:
6331  *
6332  *    +-----------+------+
6333  *    |   mainB   | subC |
6334  *    +-----------+------+
6335  * Now we see subC needs subB, so we go back to it, append and relocate it:
6336  *
6337  *    +-----------+------+------+
6338  *    |   mainB   | subC | subB |
6339  *    +-----------+------+------+
6340  *
6341  * At this point we unwind recursion, relocate calls in subC, then in mainB.
6342  */
6343 static int
6344 bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog)
6345 {
6346         struct bpf_program *subprog;
6347         int i, err;
6348
6349         /* mark all subprogs as not relocated (yet) within the context of
6350          * current main program
6351          */
6352         for (i = 0; i < obj->nr_programs; i++) {
6353                 subprog = &obj->programs[i];
6354                 if (!prog_is_subprog(obj, subprog))
6355                         continue;
6356
6357                 subprog->sub_insn_off = 0;
6358         }
6359
6360         err = bpf_object__reloc_code(obj, prog, prog);
6361         if (err)
6362                 return err;
6363
6364         return 0;
6365 }
6366
6367 static void
6368 bpf_object__free_relocs(struct bpf_object *obj)
6369 {
6370         struct bpf_program *prog;
6371         int i;
6372
6373         /* free up relocation descriptors */
6374         for (i = 0; i < obj->nr_programs; i++) {
6375                 prog = &obj->programs[i];
6376                 zfree(&prog->reloc_desc);
6377                 prog->nr_reloc = 0;
6378         }
6379 }
6380
6381 static int cmp_relocs(const void *_a, const void *_b)
6382 {
6383         const struct reloc_desc *a = _a;
6384         const struct reloc_desc *b = _b;
6385
6386         if (a->insn_idx != b->insn_idx)
6387                 return a->insn_idx < b->insn_idx ? -1 : 1;
6388
6389         /* no two relocations should have the same insn_idx, but ... */
6390         if (a->type != b->type)
6391                 return a->type < b->type ? -1 : 1;
6392
6393         return 0;
6394 }
6395
6396 static void bpf_object__sort_relos(struct bpf_object *obj)
6397 {
6398         int i;
6399
6400         for (i = 0; i < obj->nr_programs; i++) {
6401                 struct bpf_program *p = &obj->programs[i];
6402
6403                 if (!p->nr_reloc)
6404                         continue;
6405
6406                 qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs);
6407         }
6408 }
6409
6410 static int
6411 bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path)
6412 {
6413         struct bpf_program *prog;
6414         size_t i, j;
6415         int err;
6416
6417         if (obj->btf_ext) {
6418                 err = bpf_object__relocate_core(obj, targ_btf_path);
6419                 if (err) {
6420                         pr_warn("failed to perform CO-RE relocations: %d\n",
6421                                 err);
6422                         return err;
6423                 }
6424                 bpf_object__sort_relos(obj);
6425         }
6426
6427         /* Before relocating calls pre-process relocations and mark
6428          * few ld_imm64 instructions that points to subprogs.
6429          * Otherwise bpf_object__reloc_code() later would have to consider
6430          * all ld_imm64 insns as relocation candidates. That would
6431          * reduce relocation speed, since amount of find_prog_insn_relo()
6432          * would increase and most of them will fail to find a relo.
6433          */
6434         for (i = 0; i < obj->nr_programs; i++) {
6435                 prog = &obj->programs[i];
6436                 for (j = 0; j < prog->nr_reloc; j++) {
6437                         struct reloc_desc *relo = &prog->reloc_desc[j];
6438                         struct bpf_insn *insn = &prog->insns[relo->insn_idx];
6439
6440                         /* mark the insn, so it's recognized by insn_is_pseudo_func() */
6441                         if (relo->type == RELO_SUBPROG_ADDR)
6442                                 insn[0].src_reg = BPF_PSEUDO_FUNC;
6443                 }
6444         }
6445
6446         /* relocate subprogram calls and append used subprograms to main
6447          * programs; each copy of subprogram code needs to be relocated
6448          * differently for each main program, because its code location might
6449          * have changed.
6450          * Append subprog relos to main programs to allow data relos to be
6451          * processed after text is completely relocated.
6452          */
6453         for (i = 0; i < obj->nr_programs; i++) {
6454                 prog = &obj->programs[i];
6455                 /* sub-program's sub-calls are relocated within the context of
6456                  * its main program only
6457                  */
6458                 if (prog_is_subprog(obj, prog))
6459                         continue;
6460                 if (!prog->autoload)
6461                         continue;
6462
6463                 err = bpf_object__relocate_calls(obj, prog);
6464                 if (err) {
6465                         pr_warn("prog '%s': failed to relocate calls: %d\n",
6466                                 prog->name, err);
6467                         return err;
6468                 }
6469         }
6470         /* Process data relos for main programs */
6471         for (i = 0; i < obj->nr_programs; i++) {
6472                 prog = &obj->programs[i];
6473                 if (prog_is_subprog(obj, prog))
6474                         continue;
6475                 if (!prog->autoload)
6476                         continue;
6477                 err = bpf_object__relocate_data(obj, prog);
6478                 if (err) {
6479                         pr_warn("prog '%s': failed to relocate data references: %d\n",
6480                                 prog->name, err);
6481                         return err;
6482                 }
6483         }
6484
6485         return 0;
6486 }
6487
6488 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
6489                                             Elf64_Shdr *shdr, Elf_Data *data);
6490
6491 static int bpf_object__collect_map_relos(struct bpf_object *obj,
6492                                          Elf64_Shdr *shdr, Elf_Data *data)
6493 {
6494         const int bpf_ptr_sz = 8, host_ptr_sz = sizeof(void *);
6495         int i, j, nrels, new_sz;
6496         const struct btf_var_secinfo *vi = NULL;
6497         const struct btf_type *sec, *var, *def;
6498         struct bpf_map *map = NULL, *targ_map = NULL;
6499         struct bpf_program *targ_prog = NULL;
6500         bool is_prog_array, is_map_in_map;
6501         const struct btf_member *member;
6502         const char *name, *mname, *type;
6503         unsigned int moff;
6504         Elf64_Sym *sym;
6505         Elf64_Rel *rel;
6506         void *tmp;
6507
6508         if (!obj->efile.btf_maps_sec_btf_id || !obj->btf)
6509                 return -EINVAL;
6510         sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id);
6511         if (!sec)
6512                 return -EINVAL;
6513
6514         nrels = shdr->sh_size / shdr->sh_entsize;
6515         for (i = 0; i < nrels; i++) {
6516                 rel = elf_rel_by_idx(data, i);
6517                 if (!rel) {
6518                         pr_warn(".maps relo #%d: failed to get ELF relo\n", i);
6519                         return -LIBBPF_ERRNO__FORMAT;
6520                 }
6521
6522                 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
6523                 if (!sym) {
6524                         pr_warn(".maps relo #%d: symbol %zx not found\n",
6525                                 i, (size_t)ELF64_R_SYM(rel->r_info));
6526                         return -LIBBPF_ERRNO__FORMAT;
6527                 }
6528                 name = elf_sym_str(obj, sym->st_name) ?: "<?>";
6529
6530                 pr_debug(".maps relo #%d: for %zd value %zd rel->r_offset %zu name %d ('%s')\n",
6531                          i, (ssize_t)(rel->r_info >> 32), (size_t)sym->st_value,
6532                          (size_t)rel->r_offset, sym->st_name, name);
6533
6534                 for (j = 0; j < obj->nr_maps; j++) {
6535                         map = &obj->maps[j];
6536                         if (map->sec_idx != obj->efile.btf_maps_shndx)
6537                                 continue;
6538
6539                         vi = btf_var_secinfos(sec) + map->btf_var_idx;
6540                         if (vi->offset <= rel->r_offset &&
6541                             rel->r_offset + bpf_ptr_sz <= vi->offset + vi->size)
6542                                 break;
6543                 }
6544                 if (j == obj->nr_maps) {
6545                         pr_warn(".maps relo #%d: cannot find map '%s' at rel->r_offset %zu\n",
6546                                 i, name, (size_t)rel->r_offset);
6547                         return -EINVAL;
6548                 }
6549
6550                 is_map_in_map = bpf_map_type__is_map_in_map(map->def.type);
6551                 is_prog_array = map->def.type == BPF_MAP_TYPE_PROG_ARRAY;
6552                 type = is_map_in_map ? "map" : "prog";
6553                 if (is_map_in_map) {
6554                         if (sym->st_shndx != obj->efile.btf_maps_shndx) {
6555                                 pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n",
6556                                         i, name);
6557                                 return -LIBBPF_ERRNO__RELOC;
6558                         }
6559                         if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS &&
6560                             map->def.key_size != sizeof(int)) {
6561                                 pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n",
6562                                         i, map->name, sizeof(int));
6563                                 return -EINVAL;
6564                         }
6565                         targ_map = bpf_object__find_map_by_name(obj, name);
6566                         if (!targ_map) {
6567                                 pr_warn(".maps relo #%d: '%s' isn't a valid map reference\n",
6568                                         i, name);
6569                                 return -ESRCH;
6570                         }
6571                 } else if (is_prog_array) {
6572                         targ_prog = bpf_object__find_program_by_name(obj, name);
6573                         if (!targ_prog) {
6574                                 pr_warn(".maps relo #%d: '%s' isn't a valid program reference\n",
6575                                         i, name);
6576                                 return -ESRCH;
6577                         }
6578                         if (targ_prog->sec_idx != sym->st_shndx ||
6579                             targ_prog->sec_insn_off * 8 != sym->st_value ||
6580                             prog_is_subprog(obj, targ_prog)) {
6581                                 pr_warn(".maps relo #%d: '%s' isn't an entry-point program\n",
6582                                         i, name);
6583                                 return -LIBBPF_ERRNO__RELOC;
6584                         }
6585                 } else {
6586                         return -EINVAL;
6587                 }
6588
6589                 var = btf__type_by_id(obj->btf, vi->type);
6590                 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
6591                 if (btf_vlen(def) == 0)
6592                         return -EINVAL;
6593                 member = btf_members(def) + btf_vlen(def) - 1;
6594                 mname = btf__name_by_offset(obj->btf, member->name_off);
6595                 if (strcmp(mname, "values"))
6596                         return -EINVAL;
6597
6598                 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8;
6599                 if (rel->r_offset - vi->offset < moff)
6600                         return -EINVAL;
6601
6602                 moff = rel->r_offset - vi->offset - moff;
6603                 /* here we use BPF pointer size, which is always 64 bit, as we
6604                  * are parsing ELF that was built for BPF target
6605                  */
6606                 if (moff % bpf_ptr_sz)
6607                         return -EINVAL;
6608                 moff /= bpf_ptr_sz;
6609                 if (moff >= map->init_slots_sz) {
6610                         new_sz = moff + 1;
6611                         tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz);
6612                         if (!tmp)
6613                                 return -ENOMEM;
6614                         map->init_slots = tmp;
6615                         memset(map->init_slots + map->init_slots_sz, 0,
6616                                (new_sz - map->init_slots_sz) * host_ptr_sz);
6617                         map->init_slots_sz = new_sz;
6618                 }
6619                 map->init_slots[moff] = is_map_in_map ? (void *)targ_map : (void *)targ_prog;
6620
6621                 pr_debug(".maps relo #%d: map '%s' slot [%d] points to %s '%s'\n",
6622                          i, map->name, moff, type, name);
6623         }
6624
6625         return 0;
6626 }
6627
6628 static int bpf_object__collect_relos(struct bpf_object *obj)
6629 {
6630         int i, err;
6631
6632         for (i = 0; i < obj->efile.sec_cnt; i++) {
6633                 struct elf_sec_desc *sec_desc = &obj->efile.secs[i];
6634                 Elf64_Shdr *shdr;
6635                 Elf_Data *data;
6636                 int idx;
6637
6638                 if (sec_desc->sec_type != SEC_RELO)
6639                         continue;
6640
6641                 shdr = sec_desc->shdr;
6642                 data = sec_desc->data;
6643                 idx = shdr->sh_info;
6644
6645                 if (shdr->sh_type != SHT_REL) {
6646                         pr_warn("internal error at %d\n", __LINE__);
6647                         return -LIBBPF_ERRNO__INTERNAL;
6648                 }
6649
6650                 if (idx == obj->efile.st_ops_shndx)
6651                         err = bpf_object__collect_st_ops_relos(obj, shdr, data);
6652                 else if (idx == obj->efile.btf_maps_shndx)
6653                         err = bpf_object__collect_map_relos(obj, shdr, data);
6654                 else
6655                         err = bpf_object__collect_prog_relos(obj, shdr, data);
6656                 if (err)
6657                         return err;
6658         }
6659
6660         bpf_object__sort_relos(obj);
6661         return 0;
6662 }
6663
6664 static bool insn_is_helper_call(struct bpf_insn *insn, enum bpf_func_id *func_id)
6665 {
6666         if (BPF_CLASS(insn->code) == BPF_JMP &&
6667             BPF_OP(insn->code) == BPF_CALL &&
6668             BPF_SRC(insn->code) == BPF_K &&
6669             insn->src_reg == 0 &&
6670             insn->dst_reg == 0) {
6671                     *func_id = insn->imm;
6672                     return true;
6673         }
6674         return false;
6675 }
6676
6677 static int bpf_object__sanitize_prog(struct bpf_object *obj, struct bpf_program *prog)
6678 {
6679         struct bpf_insn *insn = prog->insns;
6680         enum bpf_func_id func_id;
6681         int i;
6682
6683         if (obj->gen_loader)
6684                 return 0;
6685
6686         for (i = 0; i < prog->insns_cnt; i++, insn++) {
6687                 if (!insn_is_helper_call(insn, &func_id))
6688                         continue;
6689
6690                 /* on kernels that don't yet support
6691                  * bpf_probe_read_{kernel,user}[_str] helpers, fall back
6692                  * to bpf_probe_read() which works well for old kernels
6693                  */
6694                 switch (func_id) {
6695                 case BPF_FUNC_probe_read_kernel:
6696                 case BPF_FUNC_probe_read_user:
6697                         if (!kernel_supports(obj, FEAT_PROBE_READ_KERN))
6698                                 insn->imm = BPF_FUNC_probe_read;
6699                         break;
6700                 case BPF_FUNC_probe_read_kernel_str:
6701                 case BPF_FUNC_probe_read_user_str:
6702                         if (!kernel_supports(obj, FEAT_PROBE_READ_KERN))
6703                                 insn->imm = BPF_FUNC_probe_read_str;
6704                         break;
6705                 default:
6706                         break;
6707                 }
6708         }
6709         return 0;
6710 }
6711
6712 static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name,
6713                                      int *btf_obj_fd, int *btf_type_id);
6714
6715 /* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */
6716 static int libbpf_prepare_prog_load(struct bpf_program *prog,
6717                                     struct bpf_prog_load_opts *opts, long cookie)
6718 {
6719         enum sec_def_flags def = cookie;
6720
6721         /* old kernels might not support specifying expected_attach_type */
6722         if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE))
6723                 opts->expected_attach_type = 0;
6724
6725         if (def & SEC_SLEEPABLE)
6726                 opts->prog_flags |= BPF_F_SLEEPABLE;
6727
6728         if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS))
6729                 opts->prog_flags |= BPF_F_XDP_HAS_FRAGS;
6730
6731         if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {
6732                 int btf_obj_fd = 0, btf_type_id = 0, err;
6733                 const char *attach_name;
6734
6735                 attach_name = strchr(prog->sec_name, '/');
6736                 if (!attach_name) {
6737                         /* if BPF program is annotated with just SEC("fentry")
6738                          * (or similar) without declaratively specifying
6739                          * target, then it is expected that target will be
6740                          * specified with bpf_program__set_attach_target() at
6741                          * runtime before BPF object load step. If not, then
6742                          * there is nothing to load into the kernel as BPF
6743                          * verifier won't be able to validate BPF program
6744                          * correctness anyways.
6745                          */
6746                         pr_warn("prog '%s': no BTF-based attach target is specified, use bpf_program__set_attach_target()\n",
6747                                 prog->name);
6748                         return -EINVAL;
6749                 }
6750                 attach_name++; /* skip over / */
6751
6752                 err = libbpf_find_attach_btf_id(prog, attach_name, &btf_obj_fd, &btf_type_id);
6753                 if (err)
6754                         return err;
6755
6756                 /* cache resolved BTF FD and BTF type ID in the prog */
6757                 prog->attach_btf_obj_fd = btf_obj_fd;
6758                 prog->attach_btf_id = btf_type_id;
6759
6760                 /* but by now libbpf common logic is not utilizing
6761                  * prog->atach_btf_obj_fd/prog->attach_btf_id anymore because
6762                  * this callback is called after opts were populated by
6763                  * libbpf, so this callback has to update opts explicitly here
6764                  */
6765                 opts->attach_btf_obj_fd = btf_obj_fd;
6766                 opts->attach_btf_id = btf_type_id;
6767         }
6768         return 0;
6769 }
6770
6771 static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz);
6772
6773 static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog,
6774                                 struct bpf_insn *insns, int insns_cnt,
6775                                 const char *license, __u32 kern_version, int *prog_fd)
6776 {
6777         LIBBPF_OPTS(bpf_prog_load_opts, load_attr);
6778         const char *prog_name = NULL;
6779         char *cp, errmsg[STRERR_BUFSIZE];
6780         size_t log_buf_size = 0;
6781         char *log_buf = NULL, *tmp;
6782         int btf_fd, ret, err;
6783         bool own_log_buf = true;
6784         __u32 log_level = prog->log_level;
6785
6786         if (prog->type == BPF_PROG_TYPE_UNSPEC) {
6787                 /*
6788                  * The program type must be set.  Most likely we couldn't find a proper
6789                  * section definition at load time, and thus we didn't infer the type.
6790                  */
6791                 pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n",
6792                         prog->name, prog->sec_name);
6793                 return -EINVAL;
6794         }
6795
6796         if (!insns || !insns_cnt)
6797                 return -EINVAL;
6798
6799         load_attr.expected_attach_type = prog->expected_attach_type;
6800         if (kernel_supports(obj, FEAT_PROG_NAME))
6801                 prog_name = prog->name;
6802         load_attr.attach_prog_fd = prog->attach_prog_fd;
6803         load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
6804         load_attr.attach_btf_id = prog->attach_btf_id;
6805         load_attr.kern_version = kern_version;
6806         load_attr.prog_ifindex = prog->prog_ifindex;
6807
6808         /* specify func_info/line_info only if kernel supports them */
6809         btf_fd = bpf_object__btf_fd(obj);
6810         if (btf_fd >= 0 && kernel_supports(obj, FEAT_BTF_FUNC)) {
6811                 load_attr.prog_btf_fd = btf_fd;
6812                 load_attr.func_info = prog->func_info;
6813                 load_attr.func_info_rec_size = prog->func_info_rec_size;
6814                 load_attr.func_info_cnt = prog->func_info_cnt;
6815                 load_attr.line_info = prog->line_info;
6816                 load_attr.line_info_rec_size = prog->line_info_rec_size;
6817                 load_attr.line_info_cnt = prog->line_info_cnt;
6818         }
6819         load_attr.log_level = log_level;
6820         load_attr.prog_flags = prog->prog_flags;
6821         load_attr.fd_array = obj->fd_array;
6822
6823         /* adjust load_attr if sec_def provides custom preload callback */
6824         if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) {
6825                 err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie);
6826                 if (err < 0) {
6827                         pr_warn("prog '%s': failed to prepare load attributes: %d\n",
6828                                 prog->name, err);
6829                         return err;
6830                 }
6831                 insns = prog->insns;
6832                 insns_cnt = prog->insns_cnt;
6833         }
6834
6835         if (obj->gen_loader) {
6836                 bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,
6837                                    license, insns, insns_cnt, &load_attr,
6838                                    prog - obj->programs);
6839                 *prog_fd = -1;
6840                 return 0;
6841         }
6842
6843 retry_load:
6844         /* if log_level is zero, we don't request logs initially even if
6845          * custom log_buf is specified; if the program load fails, then we'll
6846          * bump log_level to 1 and use either custom log_buf or we'll allocate
6847          * our own and retry the load to get details on what failed
6848          */
6849         if (log_level) {
6850                 if (prog->log_buf) {
6851                         log_buf = prog->log_buf;
6852                         log_buf_size = prog->log_size;
6853                         own_log_buf = false;
6854                 } else if (obj->log_buf) {
6855                         log_buf = obj->log_buf;
6856                         log_buf_size = obj->log_size;
6857                         own_log_buf = false;
6858                 } else {
6859                         log_buf_size = max((size_t)BPF_LOG_BUF_SIZE, log_buf_size * 2);
6860                         tmp = realloc(log_buf, log_buf_size);
6861                         if (!tmp) {
6862                                 ret = -ENOMEM;
6863                                 goto out;
6864                         }
6865                         log_buf = tmp;
6866                         log_buf[0] = '\0';
6867                         own_log_buf = true;
6868                 }
6869         }
6870
6871         load_attr.log_buf = log_buf;
6872         load_attr.log_size = log_buf_size;
6873         load_attr.log_level = log_level;
6874
6875         ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr);
6876         if (ret >= 0) {
6877                 if (log_level && own_log_buf) {
6878                         pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
6879                                  prog->name, log_buf);
6880                 }
6881
6882                 if (obj->has_rodata && kernel_supports(obj, FEAT_PROG_BIND_MAP)) {
6883                         struct bpf_map *map;
6884                         int i;
6885
6886                         for (i = 0; i < obj->nr_maps; i++) {
6887                                 map = &prog->obj->maps[i];
6888                                 if (map->libbpf_type != LIBBPF_MAP_RODATA)
6889                                         continue;
6890
6891                                 if (bpf_prog_bind_map(ret, bpf_map__fd(map), NULL)) {
6892                                         cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
6893                                         pr_warn("prog '%s': failed to bind map '%s': %s\n",
6894                                                 prog->name, map->real_name, cp);
6895                                         /* Don't fail hard if can't bind rodata. */
6896                                 }
6897                         }
6898                 }
6899
6900                 *prog_fd = ret;
6901                 ret = 0;
6902                 goto out;
6903         }
6904
6905         if (log_level == 0) {
6906                 log_level = 1;
6907                 goto retry_load;
6908         }
6909         /* On ENOSPC, increase log buffer size and retry, unless custom
6910          * log_buf is specified.
6911          * Be careful to not overflow u32, though. Kernel's log buf size limit
6912          * isn't part of UAPI so it can always be bumped to full 4GB. So don't
6913          * multiply by 2 unless we are sure we'll fit within 32 bits.
6914          * Currently, we'll get -EINVAL when we reach (UINT_MAX >> 2).
6915          */
6916         if (own_log_buf && errno == ENOSPC && log_buf_size <= UINT_MAX / 2)
6917                 goto retry_load;
6918
6919         ret = -errno;
6920
6921         /* post-process verifier log to improve error descriptions */
6922         fixup_verifier_log(prog, log_buf, log_buf_size);
6923
6924         cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
6925         pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, cp);
6926         pr_perm_msg(ret);
6927
6928         if (own_log_buf && log_buf && log_buf[0] != '\0') {
6929                 pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
6930                         prog->name, log_buf);
6931         }
6932
6933 out:
6934         if (own_log_buf)
6935                 free(log_buf);
6936         return ret;
6937 }
6938
6939 static char *find_prev_line(char *buf, char *cur)
6940 {
6941         char *p;
6942
6943         if (cur == buf) /* end of a log buf */
6944                 return NULL;
6945
6946         p = cur - 1;
6947         while (p - 1 >= buf && *(p - 1) != '\n')
6948                 p--;
6949
6950         return p;
6951 }
6952
6953 static void patch_log(char *buf, size_t buf_sz, size_t log_sz,
6954                       char *orig, size_t orig_sz, const char *patch)
6955 {
6956         /* size of the remaining log content to the right from the to-be-replaced part */
6957         size_t rem_sz = (buf + log_sz) - (orig + orig_sz);
6958         size_t patch_sz = strlen(patch);
6959
6960         if (patch_sz != orig_sz) {
6961                 /* If patch line(s) are longer than original piece of verifier log,
6962                  * shift log contents by (patch_sz - orig_sz) bytes to the right
6963                  * starting from after to-be-replaced part of the log.
6964                  *
6965                  * If patch line(s) are shorter than original piece of verifier log,
6966                  * shift log contents by (orig_sz - patch_sz) bytes to the left
6967                  * starting from after to-be-replaced part of the log
6968                  *
6969                  * We need to be careful about not overflowing available
6970                  * buf_sz capacity. If that's the case, we'll truncate the end
6971                  * of the original log, as necessary.
6972                  */
6973                 if (patch_sz > orig_sz) {
6974                         if (orig + patch_sz >= buf + buf_sz) {
6975                                 /* patch is big enough to cover remaining space completely */
6976                                 patch_sz -= (orig + patch_sz) - (buf + buf_sz) + 1;
6977                                 rem_sz = 0;
6978                         } else if (patch_sz - orig_sz > buf_sz - log_sz) {
6979                                 /* patch causes part of remaining log to be truncated */
6980                                 rem_sz -= (patch_sz - orig_sz) - (buf_sz - log_sz);
6981                         }
6982                 }
6983                 /* shift remaining log to the right by calculated amount */
6984                 memmove(orig + patch_sz, orig + orig_sz, rem_sz);
6985         }
6986
6987         memcpy(orig, patch, patch_sz);
6988 }
6989
6990 static void fixup_log_failed_core_relo(struct bpf_program *prog,
6991                                        char *buf, size_t buf_sz, size_t log_sz,
6992                                        char *line1, char *line2, char *line3)
6993 {
6994         /* Expected log for failed and not properly guarded CO-RE relocation:
6995          * line1 -> 123: (85) call unknown#195896080
6996          * line2 -> invalid func unknown#195896080
6997          * line3 -> <anything else or end of buffer>
6998          *
6999          * "123" is the index of the instruction that was poisoned. We extract
7000          * instruction index to find corresponding CO-RE relocation and
7001          * replace this part of the log with more relevant information about
7002          * failed CO-RE relocation.
7003          */
7004         const struct bpf_core_relo *relo;
7005         struct bpf_core_spec spec;
7006         char patch[512], spec_buf[256];
7007         int insn_idx, err, spec_len;
7008
7009         if (sscanf(line1, "%d: (%*d) call unknown#195896080\n", &insn_idx) != 1)
7010                 return;
7011
7012         relo = find_relo_core(prog, insn_idx);
7013         if (!relo)
7014                 return;
7015
7016         err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec);
7017         if (err)
7018                 return;
7019
7020         spec_len = bpf_core_format_spec(spec_buf, sizeof(spec_buf), &spec);
7021         snprintf(patch, sizeof(patch),
7022                  "%d: <invalid CO-RE relocation>\n"
7023                  "failed to resolve CO-RE relocation %s%s\n",
7024                  insn_idx, spec_buf, spec_len >= sizeof(spec_buf) ? "..." : "");
7025
7026         patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
7027 }
7028
7029 static void fixup_log_missing_map_load(struct bpf_program *prog,
7030                                        char *buf, size_t buf_sz, size_t log_sz,
7031                                        char *line1, char *line2, char *line3)
7032 {
7033         /* Expected log for failed and not properly guarded CO-RE relocation:
7034          * line1 -> 123: (85) call unknown#2001000345
7035          * line2 -> invalid func unknown#2001000345
7036          * line3 -> <anything else or end of buffer>
7037          *
7038          * "123" is the index of the instruction that was poisoned.
7039          * "345" in "2001000345" are map index in obj->maps to fetch map name.
7040          */
7041         struct bpf_object *obj = prog->obj;
7042         const struct bpf_map *map;
7043         int insn_idx, map_idx;
7044         char patch[128];
7045
7046         if (sscanf(line1, "%d: (%*d) call unknown#%d\n", &insn_idx, &map_idx) != 2)
7047                 return;
7048
7049         map_idx -= MAP_LDIMM64_POISON_BASE;
7050         if (map_idx < 0 || map_idx >= obj->nr_maps)
7051                 return;
7052         map = &obj->maps[map_idx];
7053
7054         snprintf(patch, sizeof(patch),
7055                  "%d: <invalid BPF map reference>\n"
7056                  "BPF map '%s' is referenced but wasn't created\n",
7057                  insn_idx, map->name);
7058
7059         patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
7060 }
7061
7062 static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz)
7063 {
7064         /* look for familiar error patterns in last N lines of the log */
7065         const size_t max_last_line_cnt = 10;
7066         char *prev_line, *cur_line, *next_line;
7067         size_t log_sz;
7068         int i;
7069
7070         if (!buf)
7071                 return;
7072
7073         log_sz = strlen(buf) + 1;
7074         next_line = buf + log_sz - 1;
7075
7076         for (i = 0; i < max_last_line_cnt; i++, next_line = cur_line) {
7077                 cur_line = find_prev_line(buf, next_line);
7078                 if (!cur_line)
7079                         return;
7080
7081                 /* failed CO-RE relocation case */
7082                 if (str_has_pfx(cur_line, "invalid func unknown#195896080\n")) {
7083                         prev_line = find_prev_line(buf, cur_line);
7084                         if (!prev_line)
7085                                 continue;
7086
7087                         fixup_log_failed_core_relo(prog, buf, buf_sz, log_sz,
7088                                                    prev_line, cur_line, next_line);
7089                         return;
7090                 } else if (str_has_pfx(cur_line, "invalid func unknown#"MAP_LDIMM64_POISON_PFX)) {
7091                         prev_line = find_prev_line(buf, cur_line);
7092                         if (!prev_line)
7093                                 continue;
7094
7095                         fixup_log_missing_map_load(prog, buf, buf_sz, log_sz,
7096                                                    prev_line, cur_line, next_line);
7097                         return;
7098                 }
7099         }
7100 }
7101
7102 static int bpf_program_record_relos(struct bpf_program *prog)
7103 {
7104         struct bpf_object *obj = prog->obj;
7105         int i;
7106
7107         for (i = 0; i < prog->nr_reloc; i++) {
7108                 struct reloc_desc *relo = &prog->reloc_desc[i];
7109                 struct extern_desc *ext = &obj->externs[relo->sym_off];
7110
7111                 switch (relo->type) {
7112                 case RELO_EXTERN_VAR:
7113                         if (ext->type != EXT_KSYM)
7114                                 continue;
7115                         bpf_gen__record_extern(obj->gen_loader, ext->name,
7116                                                ext->is_weak, !ext->ksym.type_id,
7117                                                BTF_KIND_VAR, relo->insn_idx);
7118                         break;
7119                 case RELO_EXTERN_FUNC:
7120                         bpf_gen__record_extern(obj->gen_loader, ext->name,
7121                                                ext->is_weak, false, BTF_KIND_FUNC,
7122                                                relo->insn_idx);
7123                         break;
7124                 case RELO_CORE: {
7125                         struct bpf_core_relo cr = {
7126                                 .insn_off = relo->insn_idx * 8,
7127                                 .type_id = relo->core_relo->type_id,
7128                                 .access_str_off = relo->core_relo->access_str_off,
7129                                 .kind = relo->core_relo->kind,
7130                         };
7131
7132                         bpf_gen__record_relo_core(obj->gen_loader, &cr);
7133                         break;
7134                 }
7135                 default:
7136                         continue;
7137                 }
7138         }
7139         return 0;
7140 }
7141
7142 static int
7143 bpf_object__load_progs(struct bpf_object *obj, int log_level)
7144 {
7145         struct bpf_program *prog;
7146         size_t i;
7147         int err;
7148
7149         for (i = 0; i < obj->nr_programs; i++) {
7150                 prog = &obj->programs[i];
7151                 err = bpf_object__sanitize_prog(obj, prog);
7152                 if (err)
7153                         return err;
7154         }
7155
7156         for (i = 0; i < obj->nr_programs; i++) {
7157                 prog = &obj->programs[i];
7158                 if (prog_is_subprog(obj, prog))
7159                         continue;
7160                 if (!prog->autoload) {
7161                         pr_debug("prog '%s': skipped loading\n", prog->name);
7162                         continue;
7163                 }
7164                 prog->log_level |= log_level;
7165
7166                 if (obj->gen_loader)
7167                         bpf_program_record_relos(prog);
7168
7169                 err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt,
7170                                            obj->license, obj->kern_version, &prog->fd);
7171                 if (err) {
7172                         pr_warn("prog '%s': failed to load: %d\n", prog->name, err);
7173                         return err;
7174                 }
7175         }
7176
7177         bpf_object__free_relocs(obj);
7178         return 0;
7179 }
7180
7181 static const struct bpf_sec_def *find_sec_def(const char *sec_name);
7182
7183 static int bpf_object_init_progs(struct bpf_object *obj, const struct bpf_object_open_opts *opts)
7184 {
7185         struct bpf_program *prog;
7186         int err;
7187
7188         bpf_object__for_each_program(prog, obj) {
7189                 prog->sec_def = find_sec_def(prog->sec_name);
7190                 if (!prog->sec_def) {
7191                         /* couldn't guess, but user might manually specify */
7192                         pr_debug("prog '%s': unrecognized ELF section name '%s'\n",
7193                                 prog->name, prog->sec_name);
7194                         continue;
7195                 }
7196
7197                 prog->type = prog->sec_def->prog_type;
7198                 prog->expected_attach_type = prog->sec_def->expected_attach_type;
7199
7200                 /* sec_def can have custom callback which should be called
7201                  * after bpf_program is initialized to adjust its properties
7202                  */
7203                 if (prog->sec_def->prog_setup_fn) {
7204                         err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie);
7205                         if (err < 0) {
7206                                 pr_warn("prog '%s': failed to initialize: %d\n",
7207                                         prog->name, err);
7208                                 return err;
7209                         }
7210                 }
7211         }
7212
7213         return 0;
7214 }
7215
7216 static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, size_t obj_buf_sz,
7217                                           const struct bpf_object_open_opts *opts)
7218 {
7219         const char *obj_name, *kconfig, *btf_tmp_path;
7220         struct bpf_object *obj;
7221         char tmp_name[64];
7222         int err;
7223         char *log_buf;
7224         size_t log_size;
7225         __u32 log_level;
7226
7227         if (elf_version(EV_CURRENT) == EV_NONE) {
7228                 pr_warn("failed to init libelf for %s\n",
7229                         path ? : "(mem buf)");
7230                 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
7231         }
7232
7233         if (!OPTS_VALID(opts, bpf_object_open_opts))
7234                 return ERR_PTR(-EINVAL);
7235
7236         obj_name = OPTS_GET(opts, object_name, NULL);
7237         if (obj_buf) {
7238                 if (!obj_name) {
7239                         snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
7240                                  (unsigned long)obj_buf,
7241                                  (unsigned long)obj_buf_sz);
7242                         obj_name = tmp_name;
7243                 }
7244                 path = obj_name;
7245                 pr_debug("loading object '%s' from buffer\n", obj_name);
7246         }
7247
7248         log_buf = OPTS_GET(opts, kernel_log_buf, NULL);
7249         log_size = OPTS_GET(opts, kernel_log_size, 0);
7250         log_level = OPTS_GET(opts, kernel_log_level, 0);
7251         if (log_size > UINT_MAX)
7252                 return ERR_PTR(-EINVAL);
7253         if (log_size && !log_buf)
7254                 return ERR_PTR(-EINVAL);
7255
7256         obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name);
7257         if (IS_ERR(obj))
7258                 return obj;
7259
7260         obj->log_buf = log_buf;
7261         obj->log_size = log_size;
7262         obj->log_level = log_level;
7263
7264         btf_tmp_path = OPTS_GET(opts, btf_custom_path, NULL);
7265         if (btf_tmp_path) {
7266                 if (strlen(btf_tmp_path) >= PATH_MAX) {
7267                         err = -ENAMETOOLONG;
7268                         goto out;
7269                 }
7270                 obj->btf_custom_path = strdup(btf_tmp_path);
7271                 if (!obj->btf_custom_path) {
7272                         err = -ENOMEM;
7273                         goto out;
7274                 }
7275         }
7276
7277         kconfig = OPTS_GET(opts, kconfig, NULL);
7278         if (kconfig) {
7279                 obj->kconfig = strdup(kconfig);
7280                 if (!obj->kconfig) {
7281                         err = -ENOMEM;
7282                         goto out;
7283                 }
7284         }
7285
7286         err = bpf_object__elf_init(obj);
7287         err = err ? : bpf_object__check_endianness(obj);
7288         err = err ? : bpf_object__elf_collect(obj);
7289         err = err ? : bpf_object__collect_externs(obj);
7290         err = err ? : bpf_object_fixup_btf(obj);
7291         err = err ? : bpf_object__init_maps(obj, opts);
7292         err = err ? : bpf_object_init_progs(obj, opts);
7293         err = err ? : bpf_object__collect_relos(obj);
7294         if (err)
7295                 goto out;
7296
7297         bpf_object__elf_finish(obj);
7298
7299         return obj;
7300 out:
7301         bpf_object__close(obj);
7302         return ERR_PTR(err);
7303 }
7304
7305 struct bpf_object *
7306 bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts)
7307 {
7308         if (!path)
7309                 return libbpf_err_ptr(-EINVAL);
7310
7311         pr_debug("loading %s\n", path);
7312
7313         return libbpf_ptr(bpf_object_open(path, NULL, 0, opts));
7314 }
7315
7316 struct bpf_object *bpf_object__open(const char *path)
7317 {
7318         return bpf_object__open_file(path, NULL);
7319 }
7320
7321 struct bpf_object *
7322 bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,
7323                      const struct bpf_object_open_opts *opts)
7324 {
7325         if (!obj_buf || obj_buf_sz == 0)
7326                 return libbpf_err_ptr(-EINVAL);
7327
7328         return libbpf_ptr(bpf_object_open(NULL, obj_buf, obj_buf_sz, opts));
7329 }
7330
7331 static int bpf_object_unload(struct bpf_object *obj)
7332 {
7333         size_t i;
7334
7335         if (!obj)
7336                 return libbpf_err(-EINVAL);
7337
7338         for (i = 0; i < obj->nr_maps; i++) {
7339                 zclose(obj->maps[i].fd);
7340                 if (obj->maps[i].st_ops)
7341                         zfree(&obj->maps[i].st_ops->kern_vdata);
7342         }
7343
7344         for (i = 0; i < obj->nr_programs; i++)
7345                 bpf_program__unload(&obj->programs[i]);
7346
7347         return 0;
7348 }
7349
7350 static int bpf_object__sanitize_maps(struct bpf_object *obj)
7351 {
7352         struct bpf_map *m;
7353
7354         bpf_object__for_each_map(m, obj) {
7355                 if (!bpf_map__is_internal(m))
7356                         continue;
7357                 if (!kernel_supports(obj, FEAT_ARRAY_MMAP))
7358                         m->def.map_flags ^= BPF_F_MMAPABLE;
7359         }
7360
7361         return 0;
7362 }
7363
7364 int libbpf_kallsyms_parse(kallsyms_cb_t cb, void *ctx)
7365 {
7366         char sym_type, sym_name[500];
7367         unsigned long long sym_addr;
7368         int ret, err = 0;
7369         FILE *f;
7370
7371         f = fopen("/proc/kallsyms", "r");
7372         if (!f) {
7373                 err = -errno;
7374                 pr_warn("failed to open /proc/kallsyms: %d\n", err);
7375                 return err;
7376         }
7377
7378         while (true) {
7379                 ret = fscanf(f, "%llx %c %499s%*[^\n]\n",
7380                              &sym_addr, &sym_type, sym_name);
7381                 if (ret == EOF && feof(f))
7382                         break;
7383                 if (ret != 3) {
7384                         pr_warn("failed to read kallsyms entry: %d\n", ret);
7385                         err = -EINVAL;
7386                         break;
7387                 }
7388
7389                 err = cb(sym_addr, sym_type, sym_name, ctx);
7390                 if (err)
7391                         break;
7392         }
7393
7394         fclose(f);
7395         return err;
7396 }
7397
7398 static int kallsyms_cb(unsigned long long sym_addr, char sym_type,
7399                        const char *sym_name, void *ctx)
7400 {
7401         struct bpf_object *obj = ctx;
7402         const struct btf_type *t;
7403         struct extern_desc *ext;
7404
7405         ext = find_extern_by_name(obj, sym_name);
7406         if (!ext || ext->type != EXT_KSYM)
7407                 return 0;
7408
7409         t = btf__type_by_id(obj->btf, ext->btf_id);
7410         if (!btf_is_var(t))
7411                 return 0;
7412
7413         if (ext->is_set && ext->ksym.addr != sym_addr) {
7414                 pr_warn("extern (ksym) '%s': resolution is ambiguous: 0x%llx or 0x%llx\n",
7415                         sym_name, ext->ksym.addr, sym_addr);
7416                 return -EINVAL;
7417         }
7418         if (!ext->is_set) {
7419                 ext->is_set = true;
7420                 ext->ksym.addr = sym_addr;
7421                 pr_debug("extern (ksym) '%s': set to 0x%llx\n", sym_name, sym_addr);
7422         }
7423         return 0;
7424 }
7425
7426 static int bpf_object__read_kallsyms_file(struct bpf_object *obj)
7427 {
7428         return libbpf_kallsyms_parse(kallsyms_cb, obj);
7429 }
7430
7431 static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name,
7432                             __u16 kind, struct btf **res_btf,
7433                             struct module_btf **res_mod_btf)
7434 {
7435         struct module_btf *mod_btf;
7436         struct btf *btf;
7437         int i, id, err;
7438
7439         btf = obj->btf_vmlinux;
7440         mod_btf = NULL;
7441         id = btf__find_by_name_kind(btf, ksym_name, kind);
7442
7443         if (id == -ENOENT) {
7444                 err = load_module_btfs(obj);
7445                 if (err)
7446                         return err;
7447
7448                 for (i = 0; i < obj->btf_module_cnt; i++) {
7449                         /* we assume module_btf's BTF FD is always >0 */
7450                         mod_btf = &obj->btf_modules[i];
7451                         btf = mod_btf->btf;
7452                         id = btf__find_by_name_kind_own(btf, ksym_name, kind);
7453                         if (id != -ENOENT)
7454                                 break;
7455                 }
7456         }
7457         if (id <= 0)
7458                 return -ESRCH;
7459
7460         *res_btf = btf;
7461         *res_mod_btf = mod_btf;
7462         return id;
7463 }
7464
7465 static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj,
7466                                                struct extern_desc *ext)
7467 {
7468         const struct btf_type *targ_var, *targ_type;
7469         __u32 targ_type_id, local_type_id;
7470         struct module_btf *mod_btf = NULL;
7471         const char *targ_var_name;
7472         struct btf *btf = NULL;
7473         int id, err;
7474
7475         id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &mod_btf);
7476         if (id < 0) {
7477                 if (id == -ESRCH && ext->is_weak)
7478                         return 0;
7479                 pr_warn("extern (var ksym) '%s': not found in kernel BTF\n",
7480                         ext->name);
7481                 return id;
7482         }
7483
7484         /* find local type_id */
7485         local_type_id = ext->ksym.type_id;
7486
7487         /* find target type_id */
7488         targ_var = btf__type_by_id(btf, id);
7489         targ_var_name = btf__name_by_offset(btf, targ_var->name_off);
7490         targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id);
7491
7492         err = bpf_core_types_are_compat(obj->btf, local_type_id,
7493                                         btf, targ_type_id);
7494         if (err <= 0) {
7495                 const struct btf_type *local_type;
7496                 const char *targ_name, *local_name;
7497
7498                 local_type = btf__type_by_id(obj->btf, local_type_id);
7499                 local_name = btf__name_by_offset(obj->btf, local_type->name_off);
7500                 targ_name = btf__name_by_offset(btf, targ_type->name_off);
7501
7502                 pr_warn("extern (var ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n",
7503                         ext->name, local_type_id,
7504                         btf_kind_str(local_type), local_name, targ_type_id,
7505                         btf_kind_str(targ_type), targ_name);
7506                 return -EINVAL;
7507         }
7508
7509         ext->is_set = true;
7510         ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0;
7511         ext->ksym.kernel_btf_id = id;
7512         pr_debug("extern (var ksym) '%s': resolved to [%d] %s %s\n",
7513                  ext->name, id, btf_kind_str(targ_var), targ_var_name);
7514
7515         return 0;
7516 }
7517
7518 static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj,
7519                                                 struct extern_desc *ext)
7520 {
7521         int local_func_proto_id, kfunc_proto_id, kfunc_id;
7522         struct module_btf *mod_btf = NULL;
7523         const struct btf_type *kern_func;
7524         struct btf *kern_btf = NULL;
7525         int ret;
7526
7527         local_func_proto_id = ext->ksym.type_id;
7528
7529         kfunc_id = find_ksym_btf_id(obj, ext->name, BTF_KIND_FUNC, &kern_btf, &mod_btf);
7530         if (kfunc_id < 0) {
7531                 if (kfunc_id == -ESRCH && ext->is_weak)
7532                         return 0;
7533                 pr_warn("extern (func ksym) '%s': not found in kernel or module BTFs\n",
7534                         ext->name);
7535                 return kfunc_id;
7536         }
7537
7538         kern_func = btf__type_by_id(kern_btf, kfunc_id);
7539         kfunc_proto_id = kern_func->type;
7540
7541         ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id,
7542                                         kern_btf, kfunc_proto_id);
7543         if (ret <= 0) {
7544                 pr_warn("extern (func ksym) '%s': func_proto [%d] incompatible with kernel [%d]\n",
7545                         ext->name, local_func_proto_id, kfunc_proto_id);
7546                 return -EINVAL;
7547         }
7548
7549         /* set index for module BTF fd in fd_array, if unset */
7550         if (mod_btf && !mod_btf->fd_array_idx) {
7551                 /* insn->off is s16 */
7552                 if (obj->fd_array_cnt == INT16_MAX) {
7553                         pr_warn("extern (func ksym) '%s': module BTF fd index %d too big to fit in bpf_insn offset\n",
7554                                 ext->name, mod_btf->fd_array_idx);
7555                         return -E2BIG;
7556                 }
7557                 /* Cannot use index 0 for module BTF fd */
7558                 if (!obj->fd_array_cnt)
7559                         obj->fd_array_cnt = 1;
7560
7561                 ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int),
7562                                         obj->fd_array_cnt + 1);
7563                 if (ret)
7564                         return ret;
7565                 mod_btf->fd_array_idx = obj->fd_array_cnt;
7566                 /* we assume module BTF FD is always >0 */
7567                 obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd;
7568         }
7569
7570         ext->is_set = true;
7571         ext->ksym.kernel_btf_id = kfunc_id;
7572         ext->ksym.btf_fd_idx = mod_btf ? mod_btf->fd_array_idx : 0;
7573         pr_debug("extern (func ksym) '%s': resolved to kernel [%d]\n",
7574                  ext->name, kfunc_id);
7575
7576         return 0;
7577 }
7578
7579 static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj)
7580 {
7581         const struct btf_type *t;
7582         struct extern_desc *ext;
7583         int i, err;
7584
7585         for (i = 0; i < obj->nr_extern; i++) {
7586                 ext = &obj->externs[i];
7587                 if (ext->type != EXT_KSYM || !ext->ksym.type_id)
7588                         continue;
7589
7590                 if (obj->gen_loader) {
7591                         ext->is_set = true;
7592                         ext->ksym.kernel_btf_obj_fd = 0;
7593                         ext->ksym.kernel_btf_id = 0;
7594                         continue;
7595                 }
7596                 t = btf__type_by_id(obj->btf, ext->btf_id);
7597                 if (btf_is_var(t))
7598                         err = bpf_object__resolve_ksym_var_btf_id(obj, ext);
7599                 else
7600                         err = bpf_object__resolve_ksym_func_btf_id(obj, ext);
7601                 if (err)
7602                         return err;
7603         }
7604         return 0;
7605 }
7606
7607 static int bpf_object__resolve_externs(struct bpf_object *obj,
7608                                        const char *extra_kconfig)
7609 {
7610         bool need_config = false, need_kallsyms = false;
7611         bool need_vmlinux_btf = false;
7612         struct extern_desc *ext;
7613         void *kcfg_data = NULL;
7614         int err, i;
7615
7616         if (obj->nr_extern == 0)
7617                 return 0;
7618
7619         if (obj->kconfig_map_idx >= 0)
7620                 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped;
7621
7622         for (i = 0; i < obj->nr_extern; i++) {
7623                 ext = &obj->externs[i];
7624
7625                 if (ext->type == EXT_KSYM) {
7626                         if (ext->ksym.type_id)
7627                                 need_vmlinux_btf = true;
7628                         else
7629                                 need_kallsyms = true;
7630                         continue;
7631                 } else if (ext->type == EXT_KCFG) {
7632                         void *ext_ptr = kcfg_data + ext->kcfg.data_off;
7633                         __u64 value = 0;
7634
7635                         /* Kconfig externs need actual /proc/config.gz */
7636                         if (str_has_pfx(ext->name, "CONFIG_")) {
7637                                 need_config = true;
7638                                 continue;
7639                         }
7640
7641                         /* Virtual kcfg externs are customly handled by libbpf */
7642                         if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) {
7643                                 value = get_kernel_version();
7644                                 if (!value) {
7645                                         pr_warn("extern (kcfg) '%s': failed to get kernel version\n", ext->name);
7646                                         return -EINVAL;
7647                                 }
7648                         } else if (strcmp(ext->name, "LINUX_HAS_BPF_COOKIE") == 0) {
7649                                 value = kernel_supports(obj, FEAT_BPF_COOKIE);
7650                         } else if (strcmp(ext->name, "LINUX_HAS_SYSCALL_WRAPPER") == 0) {
7651                                 value = kernel_supports(obj, FEAT_SYSCALL_WRAPPER);
7652                         } else if (!str_has_pfx(ext->name, "LINUX_") || !ext->is_weak) {
7653                                 /* Currently libbpf supports only CONFIG_ and LINUX_ prefixed
7654                                  * __kconfig externs, where LINUX_ ones are virtual and filled out
7655                                  * customly by libbpf (their values don't come from Kconfig).
7656                                  * If LINUX_xxx variable is not recognized by libbpf, but is marked
7657                                  * __weak, it defaults to zero value, just like for CONFIG_xxx
7658                                  * externs.
7659                                  */
7660                                 pr_warn("extern (kcfg) '%s': unrecognized virtual extern\n", ext->name);
7661                                 return -EINVAL;
7662                         }
7663
7664                         err = set_kcfg_value_num(ext, ext_ptr, value);
7665                         if (err)
7666                                 return err;
7667                         pr_debug("extern (kcfg) '%s': set to 0x%llx\n",
7668                                  ext->name, (long long)value);
7669                 } else {
7670                         pr_warn("extern '%s': unrecognized extern kind\n", ext->name);
7671                         return -EINVAL;
7672                 }
7673         }
7674         if (need_config && extra_kconfig) {
7675                 err = bpf_object__read_kconfig_mem(obj, extra_kconfig, kcfg_data);
7676                 if (err)
7677                         return -EINVAL;
7678                 need_config = false;
7679                 for (i = 0; i < obj->nr_extern; i++) {
7680                         ext = &obj->externs[i];
7681                         if (ext->type == EXT_KCFG && !ext->is_set) {
7682                                 need_config = true;
7683                                 break;
7684                         }
7685                 }
7686         }
7687         if (need_config) {
7688                 err = bpf_object__read_kconfig_file(obj, kcfg_data);
7689                 if (err)
7690                         return -EINVAL;
7691         }
7692         if (need_kallsyms) {
7693                 err = bpf_object__read_kallsyms_file(obj);
7694                 if (err)
7695                         return -EINVAL;
7696         }
7697         if (need_vmlinux_btf) {
7698                 err = bpf_object__resolve_ksyms_btf_id(obj);
7699                 if (err)
7700                         return -EINVAL;
7701         }
7702         for (i = 0; i < obj->nr_extern; i++) {
7703                 ext = &obj->externs[i];
7704
7705                 if (!ext->is_set && !ext->is_weak) {
7706                         pr_warn("extern '%s' (strong): not resolved\n", ext->name);
7707                         return -ESRCH;
7708                 } else if (!ext->is_set) {
7709                         pr_debug("extern '%s' (weak): not resolved, defaulting to zero\n",
7710                                  ext->name);
7711                 }
7712         }
7713
7714         return 0;
7715 }
7716
7717 static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const char *target_btf_path)
7718 {
7719         int err, i;
7720
7721         if (!obj)
7722                 return libbpf_err(-EINVAL);
7723
7724         if (obj->loaded) {
7725                 pr_warn("object '%s': load can't be attempted twice\n", obj->name);
7726                 return libbpf_err(-EINVAL);
7727         }
7728
7729         if (obj->gen_loader)
7730                 bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps);
7731
7732         err = bpf_object__probe_loading(obj);
7733         err = err ? : bpf_object__load_vmlinux_btf(obj, false);
7734         err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
7735         err = err ? : bpf_object__sanitize_and_load_btf(obj);
7736         err = err ? : bpf_object__sanitize_maps(obj);
7737         err = err ? : bpf_object__init_kern_struct_ops_maps(obj);
7738         err = err ? : bpf_object__create_maps(obj);
7739         err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path);
7740         err = err ? : bpf_object__load_progs(obj, extra_log_level);
7741         err = err ? : bpf_object_init_prog_arrays(obj);
7742
7743         if (obj->gen_loader) {
7744                 /* reset FDs */
7745                 if (obj->btf)
7746                         btf__set_fd(obj->btf, -1);
7747                 for (i = 0; i < obj->nr_maps; i++)
7748                         obj->maps[i].fd = -1;
7749                 if (!err)
7750                         err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps);
7751         }
7752
7753         /* clean up fd_array */
7754         zfree(&obj->fd_array);
7755
7756         /* clean up module BTFs */
7757         for (i = 0; i < obj->btf_module_cnt; i++) {
7758                 close(obj->btf_modules[i].fd);
7759                 btf__free(obj->btf_modules[i].btf);
7760                 free(obj->btf_modules[i].name);
7761         }
7762         free(obj->btf_modules);
7763
7764         /* clean up vmlinux BTF */
7765         btf__free(obj->btf_vmlinux);
7766         obj->btf_vmlinux = NULL;
7767
7768         obj->loaded = true; /* doesn't matter if successfully or not */
7769
7770         if (err)
7771                 goto out;
7772
7773         return 0;
7774 out:
7775         /* unpin any maps that were auto-pinned during load */
7776         for (i = 0; i < obj->nr_maps; i++)
7777                 if (obj->maps[i].pinned && !obj->maps[i].reused)
7778                         bpf_map__unpin(&obj->maps[i], NULL);
7779
7780         bpf_object_unload(obj);
7781         pr_warn("failed to load object '%s'\n", obj->path);
7782         return libbpf_err(err);
7783 }
7784
7785 int bpf_object__load(struct bpf_object *obj)
7786 {
7787         return bpf_object_load(obj, 0, NULL);
7788 }
7789
7790 static int make_parent_dir(const char *path)
7791 {
7792         char *cp, errmsg[STRERR_BUFSIZE];
7793         char *dname, *dir;
7794         int err = 0;
7795
7796         dname = strdup(path);
7797         if (dname == NULL)
7798                 return -ENOMEM;
7799
7800         dir = dirname(dname);
7801         if (mkdir(dir, 0700) && errno != EEXIST)
7802                 err = -errno;
7803
7804         free(dname);
7805         if (err) {
7806                 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
7807                 pr_warn("failed to mkdir %s: %s\n", path, cp);
7808         }
7809         return err;
7810 }
7811
7812 static int check_path(const char *path)
7813 {
7814         char *cp, errmsg[STRERR_BUFSIZE];
7815         struct statfs st_fs;
7816         char *dname, *dir;
7817         int err = 0;
7818
7819         if (path == NULL)
7820                 return -EINVAL;
7821
7822         dname = strdup(path);
7823         if (dname == NULL)
7824                 return -ENOMEM;
7825
7826         dir = dirname(dname);
7827         if (statfs(dir, &st_fs)) {
7828                 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
7829                 pr_warn("failed to statfs %s: %s\n", dir, cp);
7830                 err = -errno;
7831         }
7832         free(dname);
7833
7834         if (!err && st_fs.f_type != BPF_FS_MAGIC) {
7835                 pr_warn("specified path %s is not on BPF FS\n", path);
7836                 err = -EINVAL;
7837         }
7838
7839         return err;
7840 }
7841
7842 int bpf_program__pin(struct bpf_program *prog, const char *path)
7843 {
7844         char *cp, errmsg[STRERR_BUFSIZE];
7845         int err;
7846
7847         if (prog->fd < 0) {
7848                 pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name);
7849                 return libbpf_err(-EINVAL);
7850         }
7851
7852         err = make_parent_dir(path);
7853         if (err)
7854                 return libbpf_err(err);
7855
7856         err = check_path(path);
7857         if (err)
7858                 return libbpf_err(err);
7859
7860         if (bpf_obj_pin(prog->fd, path)) {
7861                 err = -errno;
7862                 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
7863                 pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, cp);
7864                 return libbpf_err(err);
7865         }
7866
7867         pr_debug("prog '%s': pinned at '%s'\n", prog->name, path);
7868         return 0;
7869 }
7870
7871 int bpf_program__unpin(struct bpf_program *prog, const char *path)
7872 {
7873         int err;
7874
7875         if (prog->fd < 0) {
7876                 pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name);
7877                 return libbpf_err(-EINVAL);
7878         }
7879
7880         err = check_path(path);
7881         if (err)
7882                 return libbpf_err(err);
7883
7884         err = unlink(path);
7885         if (err)
7886                 return libbpf_err(-errno);
7887
7888         pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path);
7889         return 0;
7890 }
7891
7892 int bpf_map__pin(struct bpf_map *map, const char *path)
7893 {
7894         char *cp, errmsg[STRERR_BUFSIZE];
7895         int err;
7896
7897         if (map == NULL) {
7898                 pr_warn("invalid map pointer\n");
7899                 return libbpf_err(-EINVAL);
7900         }
7901
7902         if (map->pin_path) {
7903                 if (path && strcmp(path, map->pin_path)) {
7904                         pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
7905                                 bpf_map__name(map), map->pin_path, path);
7906                         return libbpf_err(-EINVAL);
7907                 } else if (map->pinned) {
7908                         pr_debug("map '%s' already pinned at '%s'; not re-pinning\n",
7909                                  bpf_map__name(map), map->pin_path);
7910                         return 0;
7911                 }
7912         } else {
7913                 if (!path) {
7914                         pr_warn("missing a path to pin map '%s' at\n",
7915                                 bpf_map__name(map));
7916                         return libbpf_err(-EINVAL);
7917                 } else if (map->pinned) {
7918                         pr_warn("map '%s' already pinned\n", bpf_map__name(map));
7919                         return libbpf_err(-EEXIST);
7920                 }
7921
7922                 map->pin_path = strdup(path);
7923                 if (!map->pin_path) {
7924                         err = -errno;
7925                         goto out_err;
7926                 }
7927         }
7928
7929         err = make_parent_dir(map->pin_path);
7930         if (err)
7931                 return libbpf_err(err);
7932
7933         err = check_path(map->pin_path);
7934         if (err)
7935                 return libbpf_err(err);
7936
7937         if (bpf_obj_pin(map->fd, map->pin_path)) {
7938                 err = -errno;
7939                 goto out_err;
7940         }
7941
7942         map->pinned = true;
7943         pr_debug("pinned map '%s'\n", map->pin_path);
7944
7945         return 0;
7946
7947 out_err:
7948         cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
7949         pr_warn("failed to pin map: %s\n", cp);
7950         return libbpf_err(err);
7951 }
7952
7953 int bpf_map__unpin(struct bpf_map *map, const char *path)
7954 {
7955         int err;
7956
7957         if (map == NULL) {
7958                 pr_warn("invalid map pointer\n");
7959                 return libbpf_err(-EINVAL);
7960         }
7961
7962         if (map->pin_path) {
7963                 if (path && strcmp(path, map->pin_path)) {
7964                         pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
7965                                 bpf_map__name(map), map->pin_path, path);
7966                         return libbpf_err(-EINVAL);
7967                 }
7968                 path = map->pin_path;
7969         } else if (!path) {
7970                 pr_warn("no path to unpin map '%s' from\n",
7971                         bpf_map__name(map));
7972                 return libbpf_err(-EINVAL);
7973         }
7974
7975         err = check_path(path);
7976         if (err)
7977                 return libbpf_err(err);
7978
7979         err = unlink(path);
7980         if (err != 0)
7981                 return libbpf_err(-errno);
7982
7983         map->pinned = false;
7984         pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path);
7985
7986         return 0;
7987 }
7988
7989 int bpf_map__set_pin_path(struct bpf_map *map, const char *path)
7990 {
7991         char *new = NULL;
7992
7993         if (path) {
7994                 new = strdup(path);
7995                 if (!new)
7996                         return libbpf_err(-errno);
7997         }
7998
7999         free(map->pin_path);
8000         map->pin_path = new;
8001         return 0;
8002 }
8003
8004 __alias(bpf_map__pin_path)
8005 const char *bpf_map__get_pin_path(const struct bpf_map *map);
8006
8007 const char *bpf_map__pin_path(const struct bpf_map *map)
8008 {
8009         return map->pin_path;
8010 }
8011
8012 bool bpf_map__is_pinned(const struct bpf_map *map)
8013 {
8014         return map->pinned;
8015 }
8016
8017 static void sanitize_pin_path(char *s)
8018 {
8019         /* bpffs disallows periods in path names */
8020         while (*s) {
8021                 if (*s == '.')
8022                         *s = '_';
8023                 s++;
8024         }
8025 }
8026
8027 int bpf_object__pin_maps(struct bpf_object *obj, const char *path)
8028 {
8029         struct bpf_map *map;
8030         int err;
8031
8032         if (!obj)
8033                 return libbpf_err(-ENOENT);
8034
8035         if (!obj->loaded) {
8036                 pr_warn("object not yet loaded; load it first\n");
8037                 return libbpf_err(-ENOENT);
8038         }
8039
8040         bpf_object__for_each_map(map, obj) {
8041                 char *pin_path = NULL;
8042                 char buf[PATH_MAX];
8043
8044                 if (!map->autocreate)
8045                         continue;
8046
8047                 if (path) {
8048                         err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map));
8049                         if (err)
8050                                 goto err_unpin_maps;
8051                         sanitize_pin_path(buf);
8052                         pin_path = buf;
8053                 } else if (!map->pin_path) {
8054                         continue;
8055                 }
8056
8057                 err = bpf_map__pin(map, pin_path);
8058                 if (err)
8059                         goto err_unpin_maps;
8060         }
8061
8062         return 0;
8063
8064 err_unpin_maps:
8065         while ((map = bpf_object__prev_map(obj, map))) {
8066                 if (!map->pin_path)
8067                         continue;
8068
8069                 bpf_map__unpin(map, NULL);
8070         }
8071
8072         return libbpf_err(err);
8073 }
8074
8075 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path)
8076 {
8077         struct bpf_map *map;
8078         int err;
8079
8080         if (!obj)
8081                 return libbpf_err(-ENOENT);
8082
8083         bpf_object__for_each_map(map, obj) {
8084                 char *pin_path = NULL;
8085                 char buf[PATH_MAX];
8086
8087                 if (path) {
8088                         err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map));
8089                         if (err)
8090                                 return libbpf_err(err);
8091                         sanitize_pin_path(buf);
8092                         pin_path = buf;
8093                 } else if (!map->pin_path) {
8094                         continue;
8095                 }
8096
8097                 err = bpf_map__unpin(map, pin_path);
8098                 if (err)
8099                         return libbpf_err(err);
8100         }
8101
8102         return 0;
8103 }
8104
8105 int bpf_object__pin_programs(struct bpf_object *obj, const char *path)
8106 {
8107         struct bpf_program *prog;
8108         char buf[PATH_MAX];
8109         int err;
8110
8111         if (!obj)
8112                 return libbpf_err(-ENOENT);
8113
8114         if (!obj->loaded) {
8115                 pr_warn("object not yet loaded; load it first\n");
8116                 return libbpf_err(-ENOENT);
8117         }
8118
8119         bpf_object__for_each_program(prog, obj) {
8120                 err = pathname_concat(buf, sizeof(buf), path, prog->name);
8121                 if (err)
8122                         goto err_unpin_programs;
8123
8124                 err = bpf_program__pin(prog, buf);
8125                 if (err)
8126                         goto err_unpin_programs;
8127         }
8128
8129         return 0;
8130
8131 err_unpin_programs:
8132         while ((prog = bpf_object__prev_program(obj, prog))) {
8133                 if (pathname_concat(buf, sizeof(buf), path, prog->name))
8134                         continue;
8135
8136                 bpf_program__unpin(prog, buf);
8137         }
8138
8139         return libbpf_err(err);
8140 }
8141
8142 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path)
8143 {
8144         struct bpf_program *prog;
8145         int err;
8146
8147         if (!obj)
8148                 return libbpf_err(-ENOENT);
8149
8150         bpf_object__for_each_program(prog, obj) {
8151                 char buf[PATH_MAX];
8152
8153                 err = pathname_concat(buf, sizeof(buf), path, prog->name);
8154                 if (err)
8155                         return libbpf_err(err);
8156
8157                 err = bpf_program__unpin(prog, buf);
8158                 if (err)
8159                         return libbpf_err(err);
8160         }
8161
8162         return 0;
8163 }
8164
8165 int bpf_object__pin(struct bpf_object *obj, const char *path)
8166 {
8167         int err;
8168
8169         err = bpf_object__pin_maps(obj, path);
8170         if (err)
8171                 return libbpf_err(err);
8172
8173         err = bpf_object__pin_programs(obj, path);
8174         if (err) {
8175                 bpf_object__unpin_maps(obj, path);
8176                 return libbpf_err(err);
8177         }
8178
8179         return 0;
8180 }
8181
8182 static void bpf_map__destroy(struct bpf_map *map)
8183 {
8184         if (map->inner_map) {
8185                 bpf_map__destroy(map->inner_map);
8186                 zfree(&map->inner_map);
8187         }
8188
8189         zfree(&map->init_slots);
8190         map->init_slots_sz = 0;
8191
8192         if (map->mmaped) {
8193                 munmap(map->mmaped, bpf_map_mmap_sz(map));
8194                 map->mmaped = NULL;
8195         }
8196
8197         if (map->st_ops) {
8198                 zfree(&map->st_ops->data);
8199                 zfree(&map->st_ops->progs);
8200                 zfree(&map->st_ops->kern_func_off);
8201                 zfree(&map->st_ops);
8202         }
8203
8204         zfree(&map->name);
8205         zfree(&map->real_name);
8206         zfree(&map->pin_path);
8207
8208         if (map->fd >= 0)
8209                 zclose(map->fd);
8210 }
8211
8212 void bpf_object__close(struct bpf_object *obj)
8213 {
8214         size_t i;
8215
8216         if (IS_ERR_OR_NULL(obj))
8217                 return;
8218
8219         usdt_manager_free(obj->usdt_man);
8220         obj->usdt_man = NULL;
8221
8222         bpf_gen__free(obj->gen_loader);
8223         bpf_object__elf_finish(obj);
8224         bpf_object_unload(obj);
8225         btf__free(obj->btf);
8226         btf_ext__free(obj->btf_ext);
8227
8228         for (i = 0; i < obj->nr_maps; i++)
8229                 bpf_map__destroy(&obj->maps[i]);
8230
8231         zfree(&obj->btf_custom_path);
8232         zfree(&obj->kconfig);
8233         zfree(&obj->externs);
8234         obj->nr_extern = 0;
8235
8236         zfree(&obj->maps);
8237         obj->nr_maps = 0;
8238
8239         if (obj->programs && obj->nr_programs) {
8240                 for (i = 0; i < obj->nr_programs; i++)
8241                         bpf_program__exit(&obj->programs[i]);
8242         }
8243         zfree(&obj->programs);
8244
8245         free(obj);
8246 }
8247
8248 const char *bpf_object__name(const struct bpf_object *obj)
8249 {
8250         return obj ? obj->name : libbpf_err_ptr(-EINVAL);
8251 }
8252
8253 unsigned int bpf_object__kversion(const struct bpf_object *obj)
8254 {
8255         return obj ? obj->kern_version : 0;
8256 }
8257
8258 struct btf *bpf_object__btf(const struct bpf_object *obj)
8259 {
8260         return obj ? obj->btf : NULL;
8261 }
8262
8263 int bpf_object__btf_fd(const struct bpf_object *obj)
8264 {
8265         return obj->btf ? btf__fd(obj->btf) : -1;
8266 }
8267
8268 int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version)
8269 {
8270         if (obj->loaded)
8271                 return libbpf_err(-EINVAL);
8272
8273         obj->kern_version = kern_version;
8274
8275         return 0;
8276 }
8277
8278 int bpf_object__gen_loader(struct bpf_object *obj, struct gen_loader_opts *opts)
8279 {
8280         struct bpf_gen *gen;
8281
8282         if (!opts)
8283                 return -EFAULT;
8284         if (!OPTS_VALID(opts, gen_loader_opts))
8285                 return -EINVAL;
8286         gen = calloc(sizeof(*gen), 1);
8287         if (!gen)
8288                 return -ENOMEM;
8289         gen->opts = opts;
8290         obj->gen_loader = gen;
8291         return 0;
8292 }
8293
8294 static struct bpf_program *
8295 __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj,
8296                     bool forward)
8297 {
8298         size_t nr_programs = obj->nr_programs;
8299         ssize_t idx;
8300
8301         if (!nr_programs)
8302                 return NULL;
8303
8304         if (!p)
8305                 /* Iter from the beginning */
8306                 return forward ? &obj->programs[0] :
8307                         &obj->programs[nr_programs - 1];
8308
8309         if (p->obj != obj) {
8310                 pr_warn("error: program handler doesn't match object\n");
8311                 return errno = EINVAL, NULL;
8312         }
8313
8314         idx = (p - obj->programs) + (forward ? 1 : -1);
8315         if (idx >= obj->nr_programs || idx < 0)
8316                 return NULL;
8317         return &obj->programs[idx];
8318 }
8319
8320 struct bpf_program *
8321 bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prev)
8322 {
8323         struct bpf_program *prog = prev;
8324
8325         do {
8326                 prog = __bpf_program__iter(prog, obj, true);
8327         } while (prog && prog_is_subprog(obj, prog));
8328
8329         return prog;
8330 }
8331
8332 struct bpf_program *
8333 bpf_object__prev_program(const struct bpf_object *obj, struct bpf_program *next)
8334 {
8335         struct bpf_program *prog = next;
8336
8337         do {
8338                 prog = __bpf_program__iter(prog, obj, false);
8339         } while (prog && prog_is_subprog(obj, prog));
8340
8341         return prog;
8342 }
8343
8344 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
8345 {
8346         prog->prog_ifindex = ifindex;
8347 }
8348
8349 const char *bpf_program__name(const struct bpf_program *prog)
8350 {
8351         return prog->name;
8352 }
8353
8354 const char *bpf_program__section_name(const struct bpf_program *prog)
8355 {
8356         return prog->sec_name;
8357 }
8358
8359 bool bpf_program__autoload(const struct bpf_program *prog)
8360 {
8361         return prog->autoload;
8362 }
8363
8364 int bpf_program__set_autoload(struct bpf_program *prog, bool autoload)
8365 {
8366         if (prog->obj->loaded)
8367                 return libbpf_err(-EINVAL);
8368
8369         prog->autoload = autoload;
8370         return 0;
8371 }
8372
8373 bool bpf_program__autoattach(const struct bpf_program *prog)
8374 {
8375         return prog->autoattach;
8376 }
8377
8378 void bpf_program__set_autoattach(struct bpf_program *prog, bool autoattach)
8379 {
8380         prog->autoattach = autoattach;
8381 }
8382
8383 const struct bpf_insn *bpf_program__insns(const struct bpf_program *prog)
8384 {
8385         return prog->insns;
8386 }
8387
8388 size_t bpf_program__insn_cnt(const struct bpf_program *prog)
8389 {
8390         return prog->insns_cnt;
8391 }
8392
8393 int bpf_program__set_insns(struct bpf_program *prog,
8394                            struct bpf_insn *new_insns, size_t new_insn_cnt)
8395 {
8396         struct bpf_insn *insns;
8397
8398         if (prog->obj->loaded)
8399                 return -EBUSY;
8400
8401         insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns));
8402         if (!insns) {
8403                 pr_warn("prog '%s': failed to realloc prog code\n", prog->name);
8404                 return -ENOMEM;
8405         }
8406         memcpy(insns, new_insns, new_insn_cnt * sizeof(*insns));
8407
8408         prog->insns = insns;
8409         prog->insns_cnt = new_insn_cnt;
8410         return 0;
8411 }
8412
8413 int bpf_program__fd(const struct bpf_program *prog)
8414 {
8415         if (!prog)
8416                 return libbpf_err(-EINVAL);
8417
8418         if (prog->fd < 0)
8419                 return libbpf_err(-ENOENT);
8420
8421         return prog->fd;
8422 }
8423
8424 __alias(bpf_program__type)
8425 enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog);
8426
8427 enum bpf_prog_type bpf_program__type(const struct bpf_program *prog)
8428 {
8429         return prog->type;
8430 }
8431
8432 int bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
8433 {
8434         if (prog->obj->loaded)
8435                 return libbpf_err(-EBUSY);
8436
8437         prog->type = type;
8438         return 0;
8439 }
8440
8441 __alias(bpf_program__expected_attach_type)
8442 enum bpf_attach_type bpf_program__get_expected_attach_type(const struct bpf_program *prog);
8443
8444 enum bpf_attach_type bpf_program__expected_attach_type(const struct bpf_program *prog)
8445 {
8446         return prog->expected_attach_type;
8447 }
8448
8449 int bpf_program__set_expected_attach_type(struct bpf_program *prog,
8450                                            enum bpf_attach_type type)
8451 {
8452         if (prog->obj->loaded)
8453                 return libbpf_err(-EBUSY);
8454
8455         prog->expected_attach_type = type;
8456         return 0;
8457 }
8458
8459 __u32 bpf_program__flags(const struct bpf_program *prog)
8460 {
8461         return prog->prog_flags;
8462 }
8463
8464 int bpf_program__set_flags(struct bpf_program *prog, __u32 flags)
8465 {
8466         if (prog->obj->loaded)
8467                 return libbpf_err(-EBUSY);
8468
8469         prog->prog_flags = flags;
8470         return 0;
8471 }
8472
8473 __u32 bpf_program__log_level(const struct bpf_program *prog)
8474 {
8475         return prog->log_level;
8476 }
8477
8478 int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level)
8479 {
8480         if (prog->obj->loaded)
8481                 return libbpf_err(-EBUSY);
8482
8483         prog->log_level = log_level;
8484         return 0;
8485 }
8486
8487 const char *bpf_program__log_buf(const struct bpf_program *prog, size_t *log_size)
8488 {
8489         *log_size = prog->log_size;
8490         return prog->log_buf;
8491 }
8492
8493 int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size)
8494 {
8495         if (log_size && !log_buf)
8496                 return -EINVAL;
8497         if (prog->log_size > UINT_MAX)
8498                 return -EINVAL;
8499         if (prog->obj->loaded)
8500                 return -EBUSY;
8501
8502         prog->log_buf = log_buf;
8503         prog->log_size = log_size;
8504         return 0;
8505 }
8506
8507 #define SEC_DEF(sec_pfx, ptype, atype, flags, ...) {                        \
8508         .sec = (char *)sec_pfx,                                             \
8509         .prog_type = BPF_PROG_TYPE_##ptype,                                 \
8510         .expected_attach_type = atype,                                      \
8511         .cookie = (long)(flags),                                            \
8512         .prog_prepare_load_fn = libbpf_prepare_prog_load,                   \
8513         __VA_ARGS__                                                         \
8514 }
8515
8516 static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8517 static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8518 static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8519 static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8520 static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8521 static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8522 static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8523 static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8524 static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8525 static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8526
8527 static const struct bpf_sec_def section_defs[] = {
8528         SEC_DEF("socket",               SOCKET_FILTER, 0, SEC_NONE),
8529         SEC_DEF("sk_reuseport/migrate", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT_OR_MIGRATE, SEC_ATTACHABLE),
8530         SEC_DEF("sk_reuseport",         SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT, SEC_ATTACHABLE),
8531         SEC_DEF("kprobe+",              KPROBE, 0, SEC_NONE, attach_kprobe),
8532         SEC_DEF("uprobe+",              KPROBE, 0, SEC_NONE, attach_uprobe),
8533         SEC_DEF("uprobe.s+",            KPROBE, 0, SEC_SLEEPABLE, attach_uprobe),
8534         SEC_DEF("kretprobe+",           KPROBE, 0, SEC_NONE, attach_kprobe),
8535         SEC_DEF("uretprobe+",           KPROBE, 0, SEC_NONE, attach_uprobe),
8536         SEC_DEF("uretprobe.s+",         KPROBE, 0, SEC_SLEEPABLE, attach_uprobe),
8537         SEC_DEF("kprobe.multi+",        KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi),
8538         SEC_DEF("kretprobe.multi+",     KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi),
8539         SEC_DEF("ksyscall+",            KPROBE, 0, SEC_NONE, attach_ksyscall),
8540         SEC_DEF("kretsyscall+",         KPROBE, 0, SEC_NONE, attach_ksyscall),
8541         SEC_DEF("usdt+",                KPROBE, 0, SEC_NONE, attach_usdt),
8542         SEC_DEF("tc",                   SCHED_CLS, 0, SEC_NONE),
8543         SEC_DEF("classifier",           SCHED_CLS, 0, SEC_NONE),
8544         SEC_DEF("action",               SCHED_ACT, 0, SEC_NONE),
8545         SEC_DEF("tracepoint+",          TRACEPOINT, 0, SEC_NONE, attach_tp),
8546         SEC_DEF("tp+",                  TRACEPOINT, 0, SEC_NONE, attach_tp),
8547         SEC_DEF("raw_tracepoint+",      RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp),
8548         SEC_DEF("raw_tp+",              RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp),
8549         SEC_DEF("raw_tracepoint.w+",    RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp),
8550         SEC_DEF("raw_tp.w+",            RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp),
8551         SEC_DEF("tp_btf+",              TRACING, BPF_TRACE_RAW_TP, SEC_ATTACH_BTF, attach_trace),
8552         SEC_DEF("fentry+",              TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF, attach_trace),
8553         SEC_DEF("fmod_ret+",            TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF, attach_trace),
8554         SEC_DEF("fexit+",               TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF, attach_trace),
8555         SEC_DEF("fentry.s+",            TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace),
8556         SEC_DEF("fmod_ret.s+",          TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace),
8557         SEC_DEF("fexit.s+",             TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace),
8558         SEC_DEF("freplace+",            EXT, 0, SEC_ATTACH_BTF, attach_trace),
8559         SEC_DEF("lsm+",                 LSM, BPF_LSM_MAC, SEC_ATTACH_BTF, attach_lsm),
8560         SEC_DEF("lsm.s+",               LSM, BPF_LSM_MAC, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_lsm),
8561         SEC_DEF("lsm_cgroup+",          LSM, BPF_LSM_CGROUP, SEC_ATTACH_BTF),
8562         SEC_DEF("iter+",                TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF, attach_iter),
8563         SEC_DEF("iter.s+",              TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_iter),
8564         SEC_DEF("syscall",              SYSCALL, 0, SEC_SLEEPABLE),
8565         SEC_DEF("xdp.frags/devmap",     XDP, BPF_XDP_DEVMAP, SEC_XDP_FRAGS),
8566         SEC_DEF("xdp/devmap",           XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE),
8567         SEC_DEF("xdp.frags/cpumap",     XDP, BPF_XDP_CPUMAP, SEC_XDP_FRAGS),
8568         SEC_DEF("xdp/cpumap",           XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE),
8569         SEC_DEF("xdp.frags",            XDP, BPF_XDP, SEC_XDP_FRAGS),
8570         SEC_DEF("xdp",                  XDP, BPF_XDP, SEC_ATTACHABLE_OPT),
8571         SEC_DEF("perf_event",           PERF_EVENT, 0, SEC_NONE),
8572         SEC_DEF("lwt_in",               LWT_IN, 0, SEC_NONE),
8573         SEC_DEF("lwt_out",              LWT_OUT, 0, SEC_NONE),
8574         SEC_DEF("lwt_xmit",             LWT_XMIT, 0, SEC_NONE),
8575         SEC_DEF("lwt_seg6local",        LWT_SEG6LOCAL, 0, SEC_NONE),
8576         SEC_DEF("sockops",              SOCK_OPS, BPF_CGROUP_SOCK_OPS, SEC_ATTACHABLE_OPT),
8577         SEC_DEF("sk_skb/stream_parser", SK_SKB, BPF_SK_SKB_STREAM_PARSER, SEC_ATTACHABLE_OPT),
8578         SEC_DEF("sk_skb/stream_verdict",SK_SKB, BPF_SK_SKB_STREAM_VERDICT, SEC_ATTACHABLE_OPT),
8579         SEC_DEF("sk_skb",               SK_SKB, 0, SEC_NONE),
8580         SEC_DEF("sk_msg",               SK_MSG, BPF_SK_MSG_VERDICT, SEC_ATTACHABLE_OPT),
8581         SEC_DEF("lirc_mode2",           LIRC_MODE2, BPF_LIRC_MODE2, SEC_ATTACHABLE_OPT),
8582         SEC_DEF("flow_dissector",       FLOW_DISSECTOR, BPF_FLOW_DISSECTOR, SEC_ATTACHABLE_OPT),
8583         SEC_DEF("cgroup_skb/ingress",   CGROUP_SKB, BPF_CGROUP_INET_INGRESS, SEC_ATTACHABLE_OPT),
8584         SEC_DEF("cgroup_skb/egress",    CGROUP_SKB, BPF_CGROUP_INET_EGRESS, SEC_ATTACHABLE_OPT),
8585         SEC_DEF("cgroup/skb",           CGROUP_SKB, 0, SEC_NONE),
8586         SEC_DEF("cgroup/sock_create",   CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE),
8587         SEC_DEF("cgroup/sock_release",  CGROUP_SOCK, BPF_CGROUP_INET_SOCK_RELEASE, SEC_ATTACHABLE),
8588         SEC_DEF("cgroup/sock",          CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE_OPT),
8589         SEC_DEF("cgroup/post_bind4",    CGROUP_SOCK, BPF_CGROUP_INET4_POST_BIND, SEC_ATTACHABLE),
8590         SEC_DEF("cgroup/post_bind6",    CGROUP_SOCK, BPF_CGROUP_INET6_POST_BIND, SEC_ATTACHABLE),
8591         SEC_DEF("cgroup/bind4",         CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_BIND, SEC_ATTACHABLE),
8592         SEC_DEF("cgroup/bind6",         CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_BIND, SEC_ATTACHABLE),
8593         SEC_DEF("cgroup/connect4",      CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_CONNECT, SEC_ATTACHABLE),
8594         SEC_DEF("cgroup/connect6",      CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_CONNECT, SEC_ATTACHABLE),
8595         SEC_DEF("cgroup/sendmsg4",      CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_SENDMSG, SEC_ATTACHABLE),
8596         SEC_DEF("cgroup/sendmsg6",      CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_SENDMSG, SEC_ATTACHABLE),
8597         SEC_DEF("cgroup/recvmsg4",      CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_RECVMSG, SEC_ATTACHABLE),
8598         SEC_DEF("cgroup/recvmsg6",      CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_RECVMSG, SEC_ATTACHABLE),
8599         SEC_DEF("cgroup/getpeername4",  CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETPEERNAME, SEC_ATTACHABLE),
8600         SEC_DEF("cgroup/getpeername6",  CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETPEERNAME, SEC_ATTACHABLE),
8601         SEC_DEF("cgroup/getsockname4",  CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETSOCKNAME, SEC_ATTACHABLE),
8602         SEC_DEF("cgroup/getsockname6",  CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETSOCKNAME, SEC_ATTACHABLE),
8603         SEC_DEF("cgroup/sysctl",        CGROUP_SYSCTL, BPF_CGROUP_SYSCTL, SEC_ATTACHABLE),
8604         SEC_DEF("cgroup/getsockopt",    CGROUP_SOCKOPT, BPF_CGROUP_GETSOCKOPT, SEC_ATTACHABLE),
8605         SEC_DEF("cgroup/setsockopt",    CGROUP_SOCKOPT, BPF_CGROUP_SETSOCKOPT, SEC_ATTACHABLE),
8606         SEC_DEF("cgroup/dev",           CGROUP_DEVICE, BPF_CGROUP_DEVICE, SEC_ATTACHABLE_OPT),
8607         SEC_DEF("struct_ops+",          STRUCT_OPS, 0, SEC_NONE),
8608         SEC_DEF("sk_lookup",            SK_LOOKUP, BPF_SK_LOOKUP, SEC_ATTACHABLE),
8609 };
8610
8611 static size_t custom_sec_def_cnt;
8612 static struct bpf_sec_def *custom_sec_defs;
8613 static struct bpf_sec_def custom_fallback_def;
8614 static bool has_custom_fallback_def;
8615
8616 static int last_custom_sec_def_handler_id;
8617
8618 int libbpf_register_prog_handler(const char *sec,
8619                                  enum bpf_prog_type prog_type,
8620                                  enum bpf_attach_type exp_attach_type,
8621                                  const struct libbpf_prog_handler_opts *opts)
8622 {
8623         struct bpf_sec_def *sec_def;
8624
8625         if (!OPTS_VALID(opts, libbpf_prog_handler_opts))
8626                 return libbpf_err(-EINVAL);
8627
8628         if (last_custom_sec_def_handler_id == INT_MAX) /* prevent overflow */
8629                 return libbpf_err(-E2BIG);
8630
8631         if (sec) {
8632                 sec_def = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt + 1,
8633                                               sizeof(*sec_def));
8634                 if (!sec_def)
8635                         return libbpf_err(-ENOMEM);
8636
8637                 custom_sec_defs = sec_def;
8638                 sec_def = &custom_sec_defs[custom_sec_def_cnt];
8639         } else {
8640                 if (has_custom_fallback_def)
8641                         return libbpf_err(-EBUSY);
8642
8643                 sec_def = &custom_fallback_def;
8644         }
8645
8646         sec_def->sec = sec ? strdup(sec) : NULL;
8647         if (sec && !sec_def->sec)
8648                 return libbpf_err(-ENOMEM);
8649
8650         sec_def->prog_type = prog_type;
8651         sec_def->expected_attach_type = exp_attach_type;
8652         sec_def->cookie = OPTS_GET(opts, cookie, 0);
8653
8654         sec_def->prog_setup_fn = OPTS_GET(opts, prog_setup_fn, NULL);
8655         sec_def->prog_prepare_load_fn = OPTS_GET(opts, prog_prepare_load_fn, NULL);
8656         sec_def->prog_attach_fn = OPTS_GET(opts, prog_attach_fn, NULL);
8657
8658         sec_def->handler_id = ++last_custom_sec_def_handler_id;
8659
8660         if (sec)
8661                 custom_sec_def_cnt++;
8662         else
8663                 has_custom_fallback_def = true;
8664
8665         return sec_def->handler_id;
8666 }
8667
8668 int libbpf_unregister_prog_handler(int handler_id)
8669 {
8670         struct bpf_sec_def *sec_defs;
8671         int i;
8672
8673         if (handler_id <= 0)
8674                 return libbpf_err(-EINVAL);
8675
8676         if (has_custom_fallback_def && custom_fallback_def.handler_id == handler_id) {
8677                 memset(&custom_fallback_def, 0, sizeof(custom_fallback_def));
8678                 has_custom_fallback_def = false;
8679                 return 0;
8680         }
8681
8682         for (i = 0; i < custom_sec_def_cnt; i++) {
8683                 if (custom_sec_defs[i].handler_id == handler_id)
8684                         break;
8685         }
8686
8687         if (i == custom_sec_def_cnt)
8688                 return libbpf_err(-ENOENT);
8689
8690         free(custom_sec_defs[i].sec);
8691         for (i = i + 1; i < custom_sec_def_cnt; i++)
8692                 custom_sec_defs[i - 1] = custom_sec_defs[i];
8693         custom_sec_def_cnt--;
8694
8695         /* try to shrink the array, but it's ok if we couldn't */
8696         sec_defs = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt, sizeof(*sec_defs));
8697         if (sec_defs)
8698                 custom_sec_defs = sec_defs;
8699
8700         return 0;
8701 }
8702
8703 static bool sec_def_matches(const struct bpf_sec_def *sec_def, const char *sec_name)
8704 {
8705         size_t len = strlen(sec_def->sec);
8706
8707         /* "type/" always has to have proper SEC("type/extras") form */
8708         if (sec_def->sec[len - 1] == '/') {
8709                 if (str_has_pfx(sec_name, sec_def->sec))
8710                         return true;
8711                 return false;
8712         }
8713
8714         /* "type+" means it can be either exact SEC("type") or
8715          * well-formed SEC("type/extras") with proper '/' separator
8716          */
8717         if (sec_def->sec[len - 1] == '+') {
8718                 len--;
8719                 /* not even a prefix */
8720                 if (strncmp(sec_name, sec_def->sec, len) != 0)
8721                         return false;
8722                 /* exact match or has '/' separator */
8723                 if (sec_name[len] == '\0' || sec_name[len] == '/')
8724                         return true;
8725                 return false;
8726         }
8727
8728         return strcmp(sec_name, sec_def->sec) == 0;
8729 }
8730
8731 static const struct bpf_sec_def *find_sec_def(const char *sec_name)
8732 {
8733         const struct bpf_sec_def *sec_def;
8734         int i, n;
8735
8736         n = custom_sec_def_cnt;
8737         for (i = 0; i < n; i++) {
8738                 sec_def = &custom_sec_defs[i];
8739                 if (sec_def_matches(sec_def, sec_name))
8740                         return sec_def;
8741         }
8742
8743         n = ARRAY_SIZE(section_defs);
8744         for (i = 0; i < n; i++) {
8745                 sec_def = &section_defs[i];
8746                 if (sec_def_matches(sec_def, sec_name))
8747                         return sec_def;
8748         }
8749
8750         if (has_custom_fallback_def)
8751                 return &custom_fallback_def;
8752
8753         return NULL;
8754 }
8755
8756 #define MAX_TYPE_NAME_SIZE 32
8757
8758 static char *libbpf_get_type_names(bool attach_type)
8759 {
8760         int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE;
8761         char *buf;
8762
8763         buf = malloc(len);
8764         if (!buf)
8765                 return NULL;
8766
8767         buf[0] = '\0';
8768         /* Forge string buf with all available names */
8769         for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
8770                 const struct bpf_sec_def *sec_def = &section_defs[i];
8771
8772                 if (attach_type) {
8773                         if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)
8774                                 continue;
8775
8776                         if (!(sec_def->cookie & SEC_ATTACHABLE))
8777                                 continue;
8778                 }
8779
8780                 if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) {
8781                         free(buf);
8782                         return NULL;
8783                 }
8784                 strcat(buf, " ");
8785                 strcat(buf, section_defs[i].sec);
8786         }
8787
8788         return buf;
8789 }
8790
8791 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
8792                              enum bpf_attach_type *expected_attach_type)
8793 {
8794         const struct bpf_sec_def *sec_def;
8795         char *type_names;
8796
8797         if (!name)
8798                 return libbpf_err(-EINVAL);
8799
8800         sec_def = find_sec_def(name);
8801         if (sec_def) {
8802                 *prog_type = sec_def->prog_type;
8803                 *expected_attach_type = sec_def->expected_attach_type;
8804                 return 0;
8805         }
8806
8807         pr_debug("failed to guess program type from ELF section '%s'\n", name);
8808         type_names = libbpf_get_type_names(false);
8809         if (type_names != NULL) {
8810                 pr_debug("supported section(type) names are:%s\n", type_names);
8811                 free(type_names);
8812         }
8813
8814         return libbpf_err(-ESRCH);
8815 }
8816
8817 const char *libbpf_bpf_attach_type_str(enum bpf_attach_type t)
8818 {
8819         if (t < 0 || t >= ARRAY_SIZE(attach_type_name))
8820                 return NULL;
8821
8822         return attach_type_name[t];
8823 }
8824
8825 const char *libbpf_bpf_link_type_str(enum bpf_link_type t)
8826 {
8827         if (t < 0 || t >= ARRAY_SIZE(link_type_name))
8828                 return NULL;
8829
8830         return link_type_name[t];
8831 }
8832
8833 const char *libbpf_bpf_map_type_str(enum bpf_map_type t)
8834 {
8835         if (t < 0 || t >= ARRAY_SIZE(map_type_name))
8836                 return NULL;
8837
8838         return map_type_name[t];
8839 }
8840
8841 const char *libbpf_bpf_prog_type_str(enum bpf_prog_type t)
8842 {
8843         if (t < 0 || t >= ARRAY_SIZE(prog_type_name))
8844                 return NULL;
8845
8846         return prog_type_name[t];
8847 }
8848
8849 static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj,
8850                                                      size_t offset)
8851 {
8852         struct bpf_map *map;
8853         size_t i;
8854
8855         for (i = 0; i < obj->nr_maps; i++) {
8856                 map = &obj->maps[i];
8857                 if (!bpf_map__is_struct_ops(map))
8858                         continue;
8859                 if (map->sec_offset <= offset &&
8860                     offset - map->sec_offset < map->def.value_size)
8861                         return map;
8862         }
8863
8864         return NULL;
8865 }
8866
8867 /* Collect the reloc from ELF and populate the st_ops->progs[] */
8868 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
8869                                             Elf64_Shdr *shdr, Elf_Data *data)
8870 {
8871         const struct btf_member *member;
8872         struct bpf_struct_ops *st_ops;
8873         struct bpf_program *prog;
8874         unsigned int shdr_idx;
8875         const struct btf *btf;
8876         struct bpf_map *map;
8877         unsigned int moff, insn_idx;
8878         const char *name;
8879         __u32 member_idx;
8880         Elf64_Sym *sym;
8881         Elf64_Rel *rel;
8882         int i, nrels;
8883
8884         btf = obj->btf;
8885         nrels = shdr->sh_size / shdr->sh_entsize;
8886         for (i = 0; i < nrels; i++) {
8887                 rel = elf_rel_by_idx(data, i);
8888                 if (!rel) {
8889                         pr_warn("struct_ops reloc: failed to get %d reloc\n", i);
8890                         return -LIBBPF_ERRNO__FORMAT;
8891                 }
8892
8893                 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
8894                 if (!sym) {
8895                         pr_warn("struct_ops reloc: symbol %zx not found\n",
8896                                 (size_t)ELF64_R_SYM(rel->r_info));
8897                         return -LIBBPF_ERRNO__FORMAT;
8898                 }
8899
8900                 name = elf_sym_str(obj, sym->st_name) ?: "<?>";
8901                 map = find_struct_ops_map_by_offset(obj, rel->r_offset);
8902                 if (!map) {
8903                         pr_warn("struct_ops reloc: cannot find map at rel->r_offset %zu\n",
8904                                 (size_t)rel->r_offset);
8905                         return -EINVAL;
8906                 }
8907
8908                 moff = rel->r_offset - map->sec_offset;
8909                 shdr_idx = sym->st_shndx;
8910                 st_ops = map->st_ops;
8911                 pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %zu name %d (\'%s\')\n",
8912                          map->name,
8913                          (long long)(rel->r_info >> 32),
8914                          (long long)sym->st_value,
8915                          shdr_idx, (size_t)rel->r_offset,
8916                          map->sec_offset, sym->st_name, name);
8917
8918                 if (shdr_idx >= SHN_LORESERVE) {
8919                         pr_warn("struct_ops reloc %s: rel->r_offset %zu shdr_idx %u unsupported non-static function\n",
8920                                 map->name, (size_t)rel->r_offset, shdr_idx);
8921                         return -LIBBPF_ERRNO__RELOC;
8922                 }
8923                 if (sym->st_value % BPF_INSN_SZ) {
8924                         pr_warn("struct_ops reloc %s: invalid target program offset %llu\n",
8925                                 map->name, (unsigned long long)sym->st_value);
8926                         return -LIBBPF_ERRNO__FORMAT;
8927                 }
8928                 insn_idx = sym->st_value / BPF_INSN_SZ;
8929
8930                 member = find_member_by_offset(st_ops->type, moff * 8);
8931                 if (!member) {
8932                         pr_warn("struct_ops reloc %s: cannot find member at moff %u\n",
8933                                 map->name, moff);
8934                         return -EINVAL;
8935                 }
8936                 member_idx = member - btf_members(st_ops->type);
8937                 name = btf__name_by_offset(btf, member->name_off);
8938
8939                 if (!resolve_func_ptr(btf, member->type, NULL)) {
8940                         pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n",
8941                                 map->name, name);
8942                         return -EINVAL;
8943                 }
8944
8945                 prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx);
8946                 if (!prog) {
8947                         pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n",
8948                                 map->name, shdr_idx, name);
8949                         return -EINVAL;
8950                 }
8951
8952                 /* prevent the use of BPF prog with invalid type */
8953                 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) {
8954                         pr_warn("struct_ops reloc %s: prog %s is not struct_ops BPF program\n",
8955                                 map->name, prog->name);
8956                         return -EINVAL;
8957                 }
8958
8959                 /* if we haven't yet processed this BPF program, record proper
8960                  * attach_btf_id and member_idx
8961                  */
8962                 if (!prog->attach_btf_id) {
8963                         prog->attach_btf_id = st_ops->type_id;
8964                         prog->expected_attach_type = member_idx;
8965                 }
8966
8967                 /* struct_ops BPF prog can be re-used between multiple
8968                  * .struct_ops as long as it's the same struct_ops struct
8969                  * definition and the same function pointer field
8970                  */
8971                 if (prog->attach_btf_id != st_ops->type_id ||
8972                     prog->expected_attach_type != member_idx) {
8973                         pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n",
8974                                 map->name, prog->name, prog->sec_name, prog->type,
8975                                 prog->attach_btf_id, prog->expected_attach_type, name);
8976                         return -EINVAL;
8977                 }
8978
8979                 st_ops->progs[member_idx] = prog;
8980         }
8981
8982         return 0;
8983 }
8984
8985 #define BTF_TRACE_PREFIX "btf_trace_"
8986 #define BTF_LSM_PREFIX "bpf_lsm_"
8987 #define BTF_ITER_PREFIX "bpf_iter_"
8988 #define BTF_MAX_NAME_SIZE 128
8989
8990 void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type,
8991                                 const char **prefix, int *kind)
8992 {
8993         switch (attach_type) {
8994         case BPF_TRACE_RAW_TP:
8995                 *prefix = BTF_TRACE_PREFIX;
8996                 *kind = BTF_KIND_TYPEDEF;
8997                 break;
8998         case BPF_LSM_MAC:
8999         case BPF_LSM_CGROUP:
9000                 *prefix = BTF_LSM_PREFIX;
9001                 *kind = BTF_KIND_FUNC;
9002                 break;
9003         case BPF_TRACE_ITER:
9004                 *prefix = BTF_ITER_PREFIX;
9005                 *kind = BTF_KIND_FUNC;
9006                 break;
9007         default:
9008                 *prefix = "";
9009                 *kind = BTF_KIND_FUNC;
9010         }
9011 }
9012
9013 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
9014                                    const char *name, __u32 kind)
9015 {
9016         char btf_type_name[BTF_MAX_NAME_SIZE];
9017         int ret;
9018
9019         ret = snprintf(btf_type_name, sizeof(btf_type_name),
9020                        "%s%s", prefix, name);
9021         /* snprintf returns the number of characters written excluding the
9022          * terminating null. So, if >= BTF_MAX_NAME_SIZE are written, it
9023          * indicates truncation.
9024          */
9025         if (ret < 0 || ret >= sizeof(btf_type_name))
9026                 return -ENAMETOOLONG;
9027         return btf__find_by_name_kind(btf, btf_type_name, kind);
9028 }
9029
9030 static inline int find_attach_btf_id(struct btf *btf, const char *name,
9031                                      enum bpf_attach_type attach_type)
9032 {
9033         const char *prefix;
9034         int kind;
9035
9036         btf_get_kernel_prefix_kind(attach_type, &prefix, &kind);
9037         return find_btf_by_prefix_kind(btf, prefix, name, kind);
9038 }
9039
9040 int libbpf_find_vmlinux_btf_id(const char *name,
9041                                enum bpf_attach_type attach_type)
9042 {
9043         struct btf *btf;
9044         int err;
9045
9046         btf = btf__load_vmlinux_btf();
9047         err = libbpf_get_error(btf);
9048         if (err) {
9049                 pr_warn("vmlinux BTF is not found\n");
9050                 return libbpf_err(err);
9051         }
9052
9053         err = find_attach_btf_id(btf, name, attach_type);
9054         if (err <= 0)
9055                 pr_warn("%s is not found in vmlinux BTF\n", name);
9056
9057         btf__free(btf);
9058         return libbpf_err(err);
9059 }
9060
9061 static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd)
9062 {
9063         struct bpf_prog_info info;
9064         __u32 info_len = sizeof(info);
9065         struct btf *btf;
9066         int err;
9067
9068         memset(&info, 0, info_len);
9069         err = bpf_obj_get_info_by_fd(attach_prog_fd, &info, &info_len);
9070         if (err) {
9071                 pr_warn("failed bpf_obj_get_info_by_fd for FD %d: %d\n",
9072                         attach_prog_fd, err);
9073                 return err;
9074         }
9075
9076         err = -EINVAL;
9077         if (!info.btf_id) {
9078                 pr_warn("The target program doesn't have BTF\n");
9079                 goto out;
9080         }
9081         btf = btf__load_from_kernel_by_id(info.btf_id);
9082         err = libbpf_get_error(btf);
9083         if (err) {
9084                 pr_warn("Failed to get BTF %d of the program: %d\n", info.btf_id, err);
9085                 goto out;
9086         }
9087         err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
9088         btf__free(btf);
9089         if (err <= 0) {
9090                 pr_warn("%s is not found in prog's BTF\n", name);
9091                 goto out;
9092         }
9093 out:
9094         return err;
9095 }
9096
9097 static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name,
9098                               enum bpf_attach_type attach_type,
9099                               int *btf_obj_fd, int *btf_type_id)
9100 {
9101         int ret, i;
9102
9103         ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type);
9104         if (ret > 0) {
9105                 *btf_obj_fd = 0; /* vmlinux BTF */
9106                 *btf_type_id = ret;
9107                 return 0;
9108         }
9109         if (ret != -ENOENT)
9110                 return ret;
9111
9112         ret = load_module_btfs(obj);
9113         if (ret)
9114                 return ret;
9115
9116         for (i = 0; i < obj->btf_module_cnt; i++) {
9117                 const struct module_btf *mod = &obj->btf_modules[i];
9118
9119                 ret = find_attach_btf_id(mod->btf, attach_name, attach_type);
9120                 if (ret > 0) {
9121                         *btf_obj_fd = mod->fd;
9122                         *btf_type_id = ret;
9123                         return 0;
9124                 }
9125                 if (ret == -ENOENT)
9126                         continue;
9127
9128                 return ret;
9129         }
9130
9131         return -ESRCH;
9132 }
9133
9134 static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name,
9135                                      int *btf_obj_fd, int *btf_type_id)
9136 {
9137         enum bpf_attach_type attach_type = prog->expected_attach_type;
9138         __u32 attach_prog_fd = prog->attach_prog_fd;
9139         int err = 0;
9140
9141         /* BPF program's BTF ID */
9142         if (prog->type == BPF_PROG_TYPE_EXT || attach_prog_fd) {
9143                 if (!attach_prog_fd) {
9144                         pr_warn("prog '%s': attach program FD is not set\n", prog->name);
9145                         return -EINVAL;
9146                 }
9147                 err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd);
9148                 if (err < 0) {
9149                         pr_warn("prog '%s': failed to find BPF program (FD %d) BTF ID for '%s': %d\n",
9150                                  prog->name, attach_prog_fd, attach_name, err);
9151                         return err;
9152                 }
9153                 *btf_obj_fd = 0;
9154                 *btf_type_id = err;
9155                 return 0;
9156         }
9157
9158         /* kernel/module BTF ID */
9159         if (prog->obj->gen_loader) {
9160                 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type);
9161                 *btf_obj_fd = 0;
9162                 *btf_type_id = 1;
9163         } else {
9164                 err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id);
9165         }
9166         if (err) {
9167                 pr_warn("prog '%s': failed to find kernel BTF type ID of '%s': %d\n",
9168                         prog->name, attach_name, err);
9169                 return err;
9170         }
9171         return 0;
9172 }
9173
9174 int libbpf_attach_type_by_name(const char *name,
9175                                enum bpf_attach_type *attach_type)
9176 {
9177         char *type_names;
9178         const struct bpf_sec_def *sec_def;
9179
9180         if (!name)
9181                 return libbpf_err(-EINVAL);
9182
9183         sec_def = find_sec_def(name);
9184         if (!sec_def) {
9185                 pr_debug("failed to guess attach type based on ELF section name '%s'\n", name);
9186                 type_names = libbpf_get_type_names(true);
9187                 if (type_names != NULL) {
9188                         pr_debug("attachable section(type) names are:%s\n", type_names);
9189                         free(type_names);
9190                 }
9191
9192                 return libbpf_err(-EINVAL);
9193         }
9194
9195         if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)
9196                 return libbpf_err(-EINVAL);
9197         if (!(sec_def->cookie & SEC_ATTACHABLE))
9198                 return libbpf_err(-EINVAL);
9199
9200         *attach_type = sec_def->expected_attach_type;
9201         return 0;
9202 }
9203
9204 int bpf_map__fd(const struct bpf_map *map)
9205 {
9206         return map ? map->fd : libbpf_err(-EINVAL);
9207 }
9208
9209 static bool map_uses_real_name(const struct bpf_map *map)
9210 {
9211         /* Since libbpf started to support custom .data.* and .rodata.* maps,
9212          * their user-visible name differs from kernel-visible name. Users see
9213          * such map's corresponding ELF section name as a map name.
9214          * This check distinguishes .data/.rodata from .data.* and .rodata.*
9215          * maps to know which name has to be returned to the user.
9216          */
9217         if (map->libbpf_type == LIBBPF_MAP_DATA && strcmp(map->real_name, DATA_SEC) != 0)
9218                 return true;
9219         if (map->libbpf_type == LIBBPF_MAP_RODATA && strcmp(map->real_name, RODATA_SEC) != 0)
9220                 return true;
9221         return false;
9222 }
9223
9224 const char *bpf_map__name(const struct bpf_map *map)
9225 {
9226         if (!map)
9227                 return NULL;
9228
9229         if (map_uses_real_name(map))
9230                 return map->real_name;
9231
9232         return map->name;
9233 }
9234
9235 enum bpf_map_type bpf_map__type(const struct bpf_map *map)
9236 {
9237         return map->def.type;
9238 }
9239
9240 int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type)
9241 {
9242         if (map->fd >= 0)
9243                 return libbpf_err(-EBUSY);
9244         map->def.type = type;
9245         return 0;
9246 }
9247
9248 __u32 bpf_map__map_flags(const struct bpf_map *map)
9249 {
9250         return map->def.map_flags;
9251 }
9252
9253 int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags)
9254 {
9255         if (map->fd >= 0)
9256                 return libbpf_err(-EBUSY);
9257         map->def.map_flags = flags;
9258         return 0;
9259 }
9260
9261 __u64 bpf_map__map_extra(const struct bpf_map *map)
9262 {
9263         return map->map_extra;
9264 }
9265
9266 int bpf_map__set_map_extra(struct bpf_map *map, __u64 map_extra)
9267 {
9268         if (map->fd >= 0)
9269                 return libbpf_err(-EBUSY);
9270         map->map_extra = map_extra;
9271         return 0;
9272 }
9273
9274 __u32 bpf_map__numa_node(const struct bpf_map *map)
9275 {
9276         return map->numa_node;
9277 }
9278
9279 int bpf_map__set_numa_node(struct bpf_map *map, __u32 numa_node)
9280 {
9281         if (map->fd >= 0)
9282                 return libbpf_err(-EBUSY);
9283         map->numa_node = numa_node;
9284         return 0;
9285 }
9286
9287 __u32 bpf_map__key_size(const struct bpf_map *map)
9288 {
9289         return map->def.key_size;
9290 }
9291
9292 int bpf_map__set_key_size(struct bpf_map *map, __u32 size)
9293 {
9294         if (map->fd >= 0)
9295                 return libbpf_err(-EBUSY);
9296         map->def.key_size = size;
9297         return 0;
9298 }
9299
9300 __u32 bpf_map__value_size(const struct bpf_map *map)
9301 {
9302         return map->def.value_size;
9303 }
9304
9305 int bpf_map__set_value_size(struct bpf_map *map, __u32 size)
9306 {
9307         if (map->fd >= 0)
9308                 return libbpf_err(-EBUSY);
9309         map->def.value_size = size;
9310         return 0;
9311 }
9312
9313 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map)
9314 {
9315         return map ? map->btf_key_type_id : 0;
9316 }
9317
9318 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map)
9319 {
9320         return map ? map->btf_value_type_id : 0;
9321 }
9322
9323 int bpf_map__set_initial_value(struct bpf_map *map,
9324                                const void *data, size_t size)
9325 {
9326         if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG ||
9327             size != map->def.value_size || map->fd >= 0)
9328                 return libbpf_err(-EINVAL);
9329
9330         memcpy(map->mmaped, data, size);
9331         return 0;
9332 }
9333
9334 const void *bpf_map__initial_value(struct bpf_map *map, size_t *psize)
9335 {
9336         if (!map->mmaped)
9337                 return NULL;
9338         *psize = map->def.value_size;
9339         return map->mmaped;
9340 }
9341
9342 bool bpf_map__is_internal(const struct bpf_map *map)
9343 {
9344         return map->libbpf_type != LIBBPF_MAP_UNSPEC;
9345 }
9346
9347 __u32 bpf_map__ifindex(const struct bpf_map *map)
9348 {
9349         return map->map_ifindex;
9350 }
9351
9352 int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex)
9353 {
9354         if (map->fd >= 0)
9355                 return libbpf_err(-EBUSY);
9356         map->map_ifindex = ifindex;
9357         return 0;
9358 }
9359
9360 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd)
9361 {
9362         if (!bpf_map_type__is_map_in_map(map->def.type)) {
9363                 pr_warn("error: unsupported map type\n");
9364                 return libbpf_err(-EINVAL);
9365         }
9366         if (map->inner_map_fd != -1) {
9367                 pr_warn("error: inner_map_fd already specified\n");
9368                 return libbpf_err(-EINVAL);
9369         }
9370         if (map->inner_map) {
9371                 bpf_map__destroy(map->inner_map);
9372                 zfree(&map->inner_map);
9373         }
9374         map->inner_map_fd = fd;
9375         return 0;
9376 }
9377
9378 static struct bpf_map *
9379 __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i)
9380 {
9381         ssize_t idx;
9382         struct bpf_map *s, *e;
9383
9384         if (!obj || !obj->maps)
9385                 return errno = EINVAL, NULL;
9386
9387         s = obj->maps;
9388         e = obj->maps + obj->nr_maps;
9389
9390         if ((m < s) || (m >= e)) {
9391                 pr_warn("error in %s: map handler doesn't belong to object\n",
9392                          __func__);
9393                 return errno = EINVAL, NULL;
9394         }
9395
9396         idx = (m - obj->maps) + i;
9397         if (idx >= obj->nr_maps || idx < 0)
9398                 return NULL;
9399         return &obj->maps[idx];
9400 }
9401
9402 struct bpf_map *
9403 bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *prev)
9404 {
9405         if (prev == NULL)
9406                 return obj->maps;
9407
9408         return __bpf_map__iter(prev, obj, 1);
9409 }
9410
9411 struct bpf_map *
9412 bpf_object__prev_map(const struct bpf_object *obj, const struct bpf_map *next)
9413 {
9414         if (next == NULL) {
9415                 if (!obj->nr_maps)
9416                         return NULL;
9417                 return obj->maps + obj->nr_maps - 1;
9418         }
9419
9420         return __bpf_map__iter(next, obj, -1);
9421 }
9422
9423 struct bpf_map *
9424 bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name)
9425 {
9426         struct bpf_map *pos;
9427
9428         bpf_object__for_each_map(pos, obj) {
9429                 /* if it's a special internal map name (which always starts
9430                  * with dot) then check if that special name matches the
9431                  * real map name (ELF section name)
9432                  */
9433                 if (name[0] == '.') {
9434                         if (pos->real_name && strcmp(pos->real_name, name) == 0)
9435                                 return pos;
9436                         continue;
9437                 }
9438                 /* otherwise map name has to be an exact match */
9439                 if (map_uses_real_name(pos)) {
9440                         if (strcmp(pos->real_name, name) == 0)
9441                                 return pos;
9442                         continue;
9443                 }
9444                 if (strcmp(pos->name, name) == 0)
9445                         return pos;
9446         }
9447         return errno = ENOENT, NULL;
9448 }
9449
9450 int
9451 bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name)
9452 {
9453         return bpf_map__fd(bpf_object__find_map_by_name(obj, name));
9454 }
9455
9456 static int validate_map_op(const struct bpf_map *map, size_t key_sz,
9457                            size_t value_sz, bool check_value_sz)
9458 {
9459         if (map->fd <= 0)
9460                 return -ENOENT;
9461
9462         if (map->def.key_size != key_sz) {
9463                 pr_warn("map '%s': unexpected key size %zu provided, expected %u\n",
9464                         map->name, key_sz, map->def.key_size);
9465                 return -EINVAL;
9466         }
9467
9468         if (!check_value_sz)
9469                 return 0;
9470
9471         switch (map->def.type) {
9472         case BPF_MAP_TYPE_PERCPU_ARRAY:
9473         case BPF_MAP_TYPE_PERCPU_HASH:
9474         case BPF_MAP_TYPE_LRU_PERCPU_HASH:
9475         case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: {
9476                 int num_cpu = libbpf_num_possible_cpus();
9477                 size_t elem_sz = roundup(map->def.value_size, 8);
9478
9479                 if (value_sz != num_cpu * elem_sz) {
9480                         pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zd\n",
9481                                 map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz);
9482                         return -EINVAL;
9483                 }
9484                 break;
9485         }
9486         default:
9487                 if (map->def.value_size != value_sz) {
9488                         pr_warn("map '%s': unexpected value size %zu provided, expected %u\n",
9489                                 map->name, value_sz, map->def.value_size);
9490                         return -EINVAL;
9491                 }
9492                 break;
9493         }
9494         return 0;
9495 }
9496
9497 int bpf_map__lookup_elem(const struct bpf_map *map,
9498                          const void *key, size_t key_sz,
9499                          void *value, size_t value_sz, __u64 flags)
9500 {
9501         int err;
9502
9503         err = validate_map_op(map, key_sz, value_sz, true);
9504         if (err)
9505                 return libbpf_err(err);
9506
9507         return bpf_map_lookup_elem_flags(map->fd, key, value, flags);
9508 }
9509
9510 int bpf_map__update_elem(const struct bpf_map *map,
9511                          const void *key, size_t key_sz,
9512                          const void *value, size_t value_sz, __u64 flags)
9513 {
9514         int err;
9515
9516         err = validate_map_op(map, key_sz, value_sz, true);
9517         if (err)
9518                 return libbpf_err(err);
9519
9520         return bpf_map_update_elem(map->fd, key, value, flags);
9521 }
9522
9523 int bpf_map__delete_elem(const struct bpf_map *map,
9524                          const void *key, size_t key_sz, __u64 flags)
9525 {
9526         int err;
9527
9528         err = validate_map_op(map, key_sz, 0, false /* check_value_sz */);
9529         if (err)
9530                 return libbpf_err(err);
9531
9532         return bpf_map_delete_elem_flags(map->fd, key, flags);
9533 }
9534
9535 int bpf_map__lookup_and_delete_elem(const struct bpf_map *map,
9536                                     const void *key, size_t key_sz,
9537                                     void *value, size_t value_sz, __u64 flags)
9538 {
9539         int err;
9540
9541         err = validate_map_op(map, key_sz, value_sz, true);
9542         if (err)
9543                 return libbpf_err(err);
9544
9545         return bpf_map_lookup_and_delete_elem_flags(map->fd, key, value, flags);
9546 }
9547
9548 int bpf_map__get_next_key(const struct bpf_map *map,
9549                           const void *cur_key, void *next_key, size_t key_sz)
9550 {
9551         int err;
9552
9553         err = validate_map_op(map, key_sz, 0, false /* check_value_sz */);
9554         if (err)
9555                 return libbpf_err(err);
9556
9557         return bpf_map_get_next_key(map->fd, cur_key, next_key);
9558 }
9559
9560 long libbpf_get_error(const void *ptr)
9561 {
9562         if (!IS_ERR_OR_NULL(ptr))
9563                 return 0;
9564
9565         if (IS_ERR(ptr))
9566                 errno = -PTR_ERR(ptr);
9567
9568         /* If ptr == NULL, then errno should be already set by the failing
9569          * API, because libbpf never returns NULL on success and it now always
9570          * sets errno on error. So no extra errno handling for ptr == NULL
9571          * case.
9572          */
9573         return -errno;
9574 }
9575
9576 /* Replace link's underlying BPF program with the new one */
9577 int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog)
9578 {
9579         int ret;
9580
9581         ret = bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL);
9582         return libbpf_err_errno(ret);
9583 }
9584
9585 /* Release "ownership" of underlying BPF resource (typically, BPF program
9586  * attached to some BPF hook, e.g., tracepoint, kprobe, etc). Disconnected
9587  * link, when destructed through bpf_link__destroy() call won't attempt to
9588  * detach/unregisted that BPF resource. This is useful in situations where,
9589  * say, attached BPF program has to outlive userspace program that attached it
9590  * in the system. Depending on type of BPF program, though, there might be
9591  * additional steps (like pinning BPF program in BPF FS) necessary to ensure
9592  * exit of userspace program doesn't trigger automatic detachment and clean up
9593  * inside the kernel.
9594  */
9595 void bpf_link__disconnect(struct bpf_link *link)
9596 {
9597         link->disconnected = true;
9598 }
9599
9600 int bpf_link__destroy(struct bpf_link *link)
9601 {
9602         int err = 0;
9603
9604         if (IS_ERR_OR_NULL(link))
9605                 return 0;
9606
9607         if (!link->disconnected && link->detach)
9608                 err = link->detach(link);
9609         if (link->pin_path)
9610                 free(link->pin_path);
9611         if (link->dealloc)
9612                 link->dealloc(link);
9613         else
9614                 free(link);
9615
9616         return libbpf_err(err);
9617 }
9618
9619 int bpf_link__fd(const struct bpf_link *link)
9620 {
9621         return link->fd;
9622 }
9623
9624 const char *bpf_link__pin_path(const struct bpf_link *link)
9625 {
9626         return link->pin_path;
9627 }
9628
9629 static int bpf_link__detach_fd(struct bpf_link *link)
9630 {
9631         return libbpf_err_errno(close(link->fd));
9632 }
9633
9634 struct bpf_link *bpf_link__open(const char *path)
9635 {
9636         struct bpf_link *link;
9637         int fd;
9638
9639         fd = bpf_obj_get(path);
9640         if (fd < 0) {
9641                 fd = -errno;
9642                 pr_warn("failed to open link at %s: %d\n", path, fd);
9643                 return libbpf_err_ptr(fd);
9644         }
9645
9646         link = calloc(1, sizeof(*link));
9647         if (!link) {
9648                 close(fd);
9649                 return libbpf_err_ptr(-ENOMEM);
9650         }
9651         link->detach = &bpf_link__detach_fd;
9652         link->fd = fd;
9653
9654         link->pin_path = strdup(path);
9655         if (!link->pin_path) {
9656                 bpf_link__destroy(link);
9657                 return libbpf_err_ptr(-ENOMEM);
9658         }
9659
9660         return link;
9661 }
9662
9663 int bpf_link__detach(struct bpf_link *link)
9664 {
9665         return bpf_link_detach(link->fd) ? -errno : 0;
9666 }
9667
9668 int bpf_link__pin(struct bpf_link *link, const char *path)
9669 {
9670         int err;
9671
9672         if (link->pin_path)
9673                 return libbpf_err(-EBUSY);
9674         err = make_parent_dir(path);
9675         if (err)
9676                 return libbpf_err(err);
9677         err = check_path(path);
9678         if (err)
9679                 return libbpf_err(err);
9680
9681         link->pin_path = strdup(path);
9682         if (!link->pin_path)
9683                 return libbpf_err(-ENOMEM);
9684
9685         if (bpf_obj_pin(link->fd, link->pin_path)) {
9686                 err = -errno;
9687                 zfree(&link->pin_path);
9688                 return libbpf_err(err);
9689         }
9690
9691         pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path);
9692         return 0;
9693 }
9694
9695 int bpf_link__unpin(struct bpf_link *link)
9696 {
9697         int err;
9698
9699         if (!link->pin_path)
9700                 return libbpf_err(-EINVAL);
9701
9702         err = unlink(link->pin_path);
9703         if (err != 0)
9704                 return -errno;
9705
9706         pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path);
9707         zfree(&link->pin_path);
9708         return 0;
9709 }
9710
9711 struct bpf_link_perf {
9712         struct bpf_link link;
9713         int perf_event_fd;
9714         /* legacy kprobe support: keep track of probe identifier and type */
9715         char *legacy_probe_name;
9716         bool legacy_is_kprobe;
9717         bool legacy_is_retprobe;
9718 };
9719
9720 static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe);
9721 static int remove_uprobe_event_legacy(const char *probe_name, bool retprobe);
9722
9723 static int bpf_link_perf_detach(struct bpf_link *link)
9724 {
9725         struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
9726         int err = 0;
9727
9728         if (ioctl(perf_link->perf_event_fd, PERF_EVENT_IOC_DISABLE, 0) < 0)
9729                 err = -errno;
9730
9731         if (perf_link->perf_event_fd != link->fd)
9732                 close(perf_link->perf_event_fd);
9733         close(link->fd);
9734
9735         /* legacy uprobe/kprobe needs to be removed after perf event fd closure */
9736         if (perf_link->legacy_probe_name) {
9737                 if (perf_link->legacy_is_kprobe) {
9738                         err = remove_kprobe_event_legacy(perf_link->legacy_probe_name,
9739                                                          perf_link->legacy_is_retprobe);
9740                 } else {
9741                         err = remove_uprobe_event_legacy(perf_link->legacy_probe_name,
9742                                                          perf_link->legacy_is_retprobe);
9743                 }
9744         }
9745
9746         return err;
9747 }
9748
9749 static void bpf_link_perf_dealloc(struct bpf_link *link)
9750 {
9751         struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
9752
9753         free(perf_link->legacy_probe_name);
9754         free(perf_link);
9755 }
9756
9757 struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *prog, int pfd,
9758                                                      const struct bpf_perf_event_opts *opts)
9759 {
9760         char errmsg[STRERR_BUFSIZE];
9761         struct bpf_link_perf *link;
9762         int prog_fd, link_fd = -1, err;
9763
9764         if (!OPTS_VALID(opts, bpf_perf_event_opts))
9765                 return libbpf_err_ptr(-EINVAL);
9766
9767         if (pfd < 0) {
9768                 pr_warn("prog '%s': invalid perf event FD %d\n",
9769                         prog->name, pfd);
9770                 return libbpf_err_ptr(-EINVAL);
9771         }
9772         prog_fd = bpf_program__fd(prog);
9773         if (prog_fd < 0) {
9774                 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n",
9775                         prog->name);
9776                 return libbpf_err_ptr(-EINVAL);
9777         }
9778
9779         link = calloc(1, sizeof(*link));
9780         if (!link)
9781                 return libbpf_err_ptr(-ENOMEM);
9782         link->link.detach = &bpf_link_perf_detach;
9783         link->link.dealloc = &bpf_link_perf_dealloc;
9784         link->perf_event_fd = pfd;
9785
9786         if (kernel_supports(prog->obj, FEAT_PERF_LINK)) {
9787                 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_opts,
9788                         .perf_event.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0));
9789
9790                 link_fd = bpf_link_create(prog_fd, pfd, BPF_PERF_EVENT, &link_opts);
9791                 if (link_fd < 0) {
9792                         err = -errno;
9793                         pr_warn("prog '%s': failed to create BPF link for perf_event FD %d: %d (%s)\n",
9794                                 prog->name, pfd,
9795                                 err, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9796                         goto err_out;
9797                 }
9798                 link->link.fd = link_fd;
9799         } else {
9800                 if (OPTS_GET(opts, bpf_cookie, 0)) {
9801                         pr_warn("prog '%s': user context value is not supported\n", prog->name);
9802                         err = -EOPNOTSUPP;
9803                         goto err_out;
9804                 }
9805
9806                 if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) {
9807                         err = -errno;
9808                         pr_warn("prog '%s': failed to attach to perf_event FD %d: %s\n",
9809                                 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9810                         if (err == -EPROTO)
9811                                 pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n",
9812                                         prog->name, pfd);
9813                         goto err_out;
9814                 }
9815                 link->link.fd = pfd;
9816         }
9817         if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
9818                 err = -errno;
9819                 pr_warn("prog '%s': failed to enable perf_event FD %d: %s\n",
9820                         prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9821                 goto err_out;
9822         }
9823
9824         return &link->link;
9825 err_out:
9826         if (link_fd >= 0)
9827                 close(link_fd);
9828         free(link);
9829         return libbpf_err_ptr(err);
9830 }
9831
9832 struct bpf_link *bpf_program__attach_perf_event(const struct bpf_program *prog, int pfd)
9833 {
9834         return bpf_program__attach_perf_event_opts(prog, pfd, NULL);
9835 }
9836
9837 /*
9838  * this function is expected to parse integer in the range of [0, 2^31-1] from
9839  * given file using scanf format string fmt. If actual parsed value is
9840  * negative, the result might be indistinguishable from error
9841  */
9842 static int parse_uint_from_file(const char *file, const char *fmt)
9843 {
9844         char buf[STRERR_BUFSIZE];
9845         int err, ret;
9846         FILE *f;
9847
9848         f = fopen(file, "r");
9849         if (!f) {
9850                 err = -errno;
9851                 pr_debug("failed to open '%s': %s\n", file,
9852                          libbpf_strerror_r(err, buf, sizeof(buf)));
9853                 return err;
9854         }
9855         err = fscanf(f, fmt, &ret);
9856         if (err != 1) {
9857                 err = err == EOF ? -EIO : -errno;
9858                 pr_debug("failed to parse '%s': %s\n", file,
9859                         libbpf_strerror_r(err, buf, sizeof(buf)));
9860                 fclose(f);
9861                 return err;
9862         }
9863         fclose(f);
9864         return ret;
9865 }
9866
9867 static int determine_kprobe_perf_type(void)
9868 {
9869         const char *file = "/sys/bus/event_source/devices/kprobe/type";
9870
9871         return parse_uint_from_file(file, "%d\n");
9872 }
9873
9874 static int determine_uprobe_perf_type(void)
9875 {
9876         const char *file = "/sys/bus/event_source/devices/uprobe/type";
9877
9878         return parse_uint_from_file(file, "%d\n");
9879 }
9880
9881 static int determine_kprobe_retprobe_bit(void)
9882 {
9883         const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe";
9884
9885         return parse_uint_from_file(file, "config:%d\n");
9886 }
9887
9888 static int determine_uprobe_retprobe_bit(void)
9889 {
9890         const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe";
9891
9892         return parse_uint_from_file(file, "config:%d\n");
9893 }
9894
9895 #define PERF_UPROBE_REF_CTR_OFFSET_BITS 32
9896 #define PERF_UPROBE_REF_CTR_OFFSET_SHIFT 32
9897
9898 static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name,
9899                                  uint64_t offset, int pid, size_t ref_ctr_off)
9900 {
9901         const size_t attr_sz = sizeof(struct perf_event_attr);
9902         struct perf_event_attr attr;
9903         char errmsg[STRERR_BUFSIZE];
9904         int type, pfd;
9905
9906         if (ref_ctr_off >= (1ULL << PERF_UPROBE_REF_CTR_OFFSET_BITS))
9907                 return -EINVAL;
9908
9909         memset(&attr, 0, attr_sz);
9910
9911         type = uprobe ? determine_uprobe_perf_type()
9912                       : determine_kprobe_perf_type();
9913         if (type < 0) {
9914                 pr_warn("failed to determine %s perf type: %s\n",
9915                         uprobe ? "uprobe" : "kprobe",
9916                         libbpf_strerror_r(type, errmsg, sizeof(errmsg)));
9917                 return type;
9918         }
9919         if (retprobe) {
9920                 int bit = uprobe ? determine_uprobe_retprobe_bit()
9921                                  : determine_kprobe_retprobe_bit();
9922
9923                 if (bit < 0) {
9924                         pr_warn("failed to determine %s retprobe bit: %s\n",
9925                                 uprobe ? "uprobe" : "kprobe",
9926                                 libbpf_strerror_r(bit, errmsg, sizeof(errmsg)));
9927                         return bit;
9928                 }
9929                 attr.config |= 1 << bit;
9930         }
9931         attr.size = attr_sz;
9932         attr.type = type;
9933         attr.config |= (__u64)ref_ctr_off << PERF_UPROBE_REF_CTR_OFFSET_SHIFT;
9934         attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */
9935         attr.config2 = offset;           /* kprobe_addr or probe_offset */
9936
9937         /* pid filter is meaningful only for uprobes */
9938         pfd = syscall(__NR_perf_event_open, &attr,
9939                       pid < 0 ? -1 : pid /* pid */,
9940                       pid == -1 ? 0 : -1 /* cpu */,
9941                       -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
9942         return pfd >= 0 ? pfd : -errno;
9943 }
9944
9945 static int append_to_file(const char *file, const char *fmt, ...)
9946 {
9947         int fd, n, err = 0;
9948         va_list ap;
9949
9950         fd = open(file, O_WRONLY | O_APPEND | O_CLOEXEC, 0);
9951         if (fd < 0)
9952                 return -errno;
9953
9954         va_start(ap, fmt);
9955         n = vdprintf(fd, fmt, ap);
9956         va_end(ap);
9957
9958         if (n < 0)
9959                 err = -errno;
9960
9961         close(fd);
9962         return err;
9963 }
9964
9965 #define DEBUGFS "/sys/kernel/debug/tracing"
9966 #define TRACEFS "/sys/kernel/tracing"
9967
9968 static bool use_debugfs(void)
9969 {
9970         static int has_debugfs = -1;
9971
9972         if (has_debugfs < 0)
9973                 has_debugfs = faccessat(AT_FDCWD, DEBUGFS, F_OK, AT_EACCESS) == 0;
9974
9975         return has_debugfs == 1;
9976 }
9977
9978 static const char *tracefs_path(void)
9979 {
9980         return use_debugfs() ? DEBUGFS : TRACEFS;
9981 }
9982
9983 static const char *tracefs_kprobe_events(void)
9984 {
9985         return use_debugfs() ? DEBUGFS"/kprobe_events" : TRACEFS"/kprobe_events";
9986 }
9987
9988 static const char *tracefs_uprobe_events(void)
9989 {
9990         return use_debugfs() ? DEBUGFS"/uprobe_events" : TRACEFS"/uprobe_events";
9991 }
9992
9993 static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz,
9994                                          const char *kfunc_name, size_t offset)
9995 {
9996         static int index = 0;
9997
9998         snprintf(buf, buf_sz, "libbpf_%u_%s_0x%zx_%d", getpid(), kfunc_name, offset,
9999                  __sync_fetch_and_add(&index, 1));
10000 }
10001
10002 static int add_kprobe_event_legacy(const char *probe_name, bool retprobe,
10003                                    const char *kfunc_name, size_t offset)
10004 {
10005         return append_to_file(tracefs_kprobe_events(), "%c:%s/%s %s+0x%zx",
10006                               retprobe ? 'r' : 'p',
10007                               retprobe ? "kretprobes" : "kprobes",
10008                               probe_name, kfunc_name, offset);
10009 }
10010
10011 static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe)
10012 {
10013         return append_to_file(tracefs_kprobe_events(), "-:%s/%s",
10014                               retprobe ? "kretprobes" : "kprobes", probe_name);
10015 }
10016
10017 static int determine_kprobe_perf_type_legacy(const char *probe_name, bool retprobe)
10018 {
10019         char file[256];
10020
10021         snprintf(file, sizeof(file), "%s/events/%s/%s/id",
10022                  tracefs_path(), retprobe ? "kretprobes" : "kprobes", probe_name);
10023
10024         return parse_uint_from_file(file, "%d\n");
10025 }
10026
10027 static int perf_event_kprobe_open_legacy(const char *probe_name, bool retprobe,
10028                                          const char *kfunc_name, size_t offset, int pid)
10029 {
10030         const size_t attr_sz = sizeof(struct perf_event_attr);
10031         struct perf_event_attr attr;
10032         char errmsg[STRERR_BUFSIZE];
10033         int type, pfd, err;
10034
10035         err = add_kprobe_event_legacy(probe_name, retprobe, kfunc_name, offset);
10036         if (err < 0) {
10037                 pr_warn("failed to add legacy kprobe event for '%s+0x%zx': %s\n",
10038                         kfunc_name, offset,
10039                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10040                 return err;
10041         }
10042         type = determine_kprobe_perf_type_legacy(probe_name, retprobe);
10043         if (type < 0) {
10044                 err = type;
10045                 pr_warn("failed to determine legacy kprobe event id for '%s+0x%zx': %s\n",
10046                         kfunc_name, offset,
10047                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10048                 goto err_clean_legacy;
10049         }
10050
10051         memset(&attr, 0, attr_sz);
10052         attr.size = attr_sz;
10053         attr.config = type;
10054         attr.type = PERF_TYPE_TRACEPOINT;
10055
10056         pfd = syscall(__NR_perf_event_open, &attr,
10057                       pid < 0 ? -1 : pid, /* pid */
10058                       pid == -1 ? 0 : -1, /* cpu */
10059                       -1 /* group_fd */,  PERF_FLAG_FD_CLOEXEC);
10060         if (pfd < 0) {
10061                 err = -errno;
10062                 pr_warn("legacy kprobe perf_event_open() failed: %s\n",
10063                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10064                 goto err_clean_legacy;
10065         }
10066         return pfd;
10067
10068 err_clean_legacy:
10069         /* Clear the newly added legacy kprobe_event */
10070         remove_kprobe_event_legacy(probe_name, retprobe);
10071         return err;
10072 }
10073
10074 static const char *arch_specific_syscall_pfx(void)
10075 {
10076 #if defined(__x86_64__)
10077         return "x64";
10078 #elif defined(__i386__)
10079         return "ia32";
10080 #elif defined(__s390x__)
10081         return "s390x";
10082 #elif defined(__s390__)
10083         return "s390";
10084 #elif defined(__arm__)
10085         return "arm";
10086 #elif defined(__aarch64__)
10087         return "arm64";
10088 #elif defined(__mips__)
10089         return "mips";
10090 #elif defined(__riscv)
10091         return "riscv";
10092 #elif defined(__powerpc__)
10093         return "powerpc";
10094 #elif defined(__powerpc64__)
10095         return "powerpc64";
10096 #else
10097         return NULL;
10098 #endif
10099 }
10100
10101 static int probe_kern_syscall_wrapper(void)
10102 {
10103         char syscall_name[64];
10104         const char *ksys_pfx;
10105
10106         ksys_pfx = arch_specific_syscall_pfx();
10107         if (!ksys_pfx)
10108                 return 0;
10109
10110         snprintf(syscall_name, sizeof(syscall_name), "__%s_sys_bpf", ksys_pfx);
10111
10112         if (determine_kprobe_perf_type() >= 0) {
10113                 int pfd;
10114
10115                 pfd = perf_event_open_probe(false, false, syscall_name, 0, getpid(), 0);
10116                 if (pfd >= 0)
10117                         close(pfd);
10118
10119                 return pfd >= 0 ? 1 : 0;
10120         } else { /* legacy mode */
10121                 char probe_name[128];
10122
10123                 gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name), syscall_name, 0);
10124                 if (add_kprobe_event_legacy(probe_name, false, syscall_name, 0) < 0)
10125                         return 0;
10126
10127                 (void)remove_kprobe_event_legacy(probe_name, false);
10128                 return 1;
10129         }
10130 }
10131
10132 struct bpf_link *
10133 bpf_program__attach_kprobe_opts(const struct bpf_program *prog,
10134                                 const char *func_name,
10135                                 const struct bpf_kprobe_opts *opts)
10136 {
10137         DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
10138         char errmsg[STRERR_BUFSIZE];
10139         char *legacy_probe = NULL;
10140         struct bpf_link *link;
10141         size_t offset;
10142         bool retprobe, legacy;
10143         int pfd, err;
10144
10145         if (!OPTS_VALID(opts, bpf_kprobe_opts))
10146                 return libbpf_err_ptr(-EINVAL);
10147
10148         retprobe = OPTS_GET(opts, retprobe, false);
10149         offset = OPTS_GET(opts, offset, 0);
10150         pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
10151
10152         legacy = determine_kprobe_perf_type() < 0;
10153         if (!legacy) {
10154                 pfd = perf_event_open_probe(false /* uprobe */, retprobe,
10155                                             func_name, offset,
10156                                             -1 /* pid */, 0 /* ref_ctr_off */);
10157         } else {
10158                 char probe_name[256];
10159
10160                 gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name),
10161                                              func_name, offset);
10162
10163                 legacy_probe = strdup(probe_name);
10164                 if (!legacy_probe)
10165                         return libbpf_err_ptr(-ENOMEM);
10166
10167                 pfd = perf_event_kprobe_open_legacy(legacy_probe, retprobe, func_name,
10168                                                     offset, -1 /* pid */);
10169         }
10170         if (pfd < 0) {
10171                 err = -errno;
10172                 pr_warn("prog '%s': failed to create %s '%s+0x%zx' perf event: %s\n",
10173                         prog->name, retprobe ? "kretprobe" : "kprobe",
10174                         func_name, offset,
10175                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10176                 goto err_out;
10177         }
10178         link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
10179         err = libbpf_get_error(link);
10180         if (err) {
10181                 close(pfd);
10182                 pr_warn("prog '%s': failed to attach to %s '%s+0x%zx': %s\n",
10183                         prog->name, retprobe ? "kretprobe" : "kprobe",
10184                         func_name, offset,
10185                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10186                 goto err_clean_legacy;
10187         }
10188         if (legacy) {
10189                 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
10190
10191                 perf_link->legacy_probe_name = legacy_probe;
10192                 perf_link->legacy_is_kprobe = true;
10193                 perf_link->legacy_is_retprobe = retprobe;
10194         }
10195
10196         return link;
10197
10198 err_clean_legacy:
10199         if (legacy)
10200                 remove_kprobe_event_legacy(legacy_probe, retprobe);
10201 err_out:
10202         free(legacy_probe);
10203         return libbpf_err_ptr(err);
10204 }
10205
10206 struct bpf_link *bpf_program__attach_kprobe(const struct bpf_program *prog,
10207                                             bool retprobe,
10208                                             const char *func_name)
10209 {
10210         DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts,
10211                 .retprobe = retprobe,
10212         );
10213
10214         return bpf_program__attach_kprobe_opts(prog, func_name, &opts);
10215 }
10216
10217 struct bpf_link *bpf_program__attach_ksyscall(const struct bpf_program *prog,
10218                                               const char *syscall_name,
10219                                               const struct bpf_ksyscall_opts *opts)
10220 {
10221         LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts);
10222         char func_name[128];
10223
10224         if (!OPTS_VALID(opts, bpf_ksyscall_opts))
10225                 return libbpf_err_ptr(-EINVAL);
10226
10227         if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) {
10228                 /* arch_specific_syscall_pfx() should never return NULL here
10229                  * because it is guarded by kernel_supports(). However, since
10230                  * compiler does not know that we have an explicit conditional
10231                  * as well.
10232                  */
10233                 snprintf(func_name, sizeof(func_name), "__%s_sys_%s",
10234                          arch_specific_syscall_pfx() ? : "", syscall_name);
10235         } else {
10236                 snprintf(func_name, sizeof(func_name), "__se_sys_%s", syscall_name);
10237         }
10238
10239         kprobe_opts.retprobe = OPTS_GET(opts, retprobe, false);
10240         kprobe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
10241
10242         return bpf_program__attach_kprobe_opts(prog, func_name, &kprobe_opts);
10243 }
10244
10245 /* Adapted from perf/util/string.c */
10246 static bool glob_match(const char *str, const char *pat)
10247 {
10248         while (*str && *pat && *pat != '*') {
10249                 if (*pat == '?') {      /* Matches any single character */
10250                         str++;
10251                         pat++;
10252                         continue;
10253                 }
10254                 if (*str != *pat)
10255                         return false;
10256                 str++;
10257                 pat++;
10258         }
10259         /* Check wild card */
10260         if (*pat == '*') {
10261                 while (*pat == '*')
10262                         pat++;
10263                 if (!*pat) /* Tail wild card matches all */
10264                         return true;
10265                 while (*str)
10266                         if (glob_match(str++, pat))
10267                                 return true;
10268         }
10269         return !*str && !*pat;
10270 }
10271
10272 struct kprobe_multi_resolve {
10273         const char *pattern;
10274         unsigned long *addrs;
10275         size_t cap;
10276         size_t cnt;
10277 };
10278
10279 static int
10280 resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type,
10281                         const char *sym_name, void *ctx)
10282 {
10283         struct kprobe_multi_resolve *res = ctx;
10284         int err;
10285
10286         if (!glob_match(sym_name, res->pattern))
10287                 return 0;
10288
10289         err = libbpf_ensure_mem((void **) &res->addrs, &res->cap, sizeof(unsigned long),
10290                                 res->cnt + 1);
10291         if (err)
10292                 return err;
10293
10294         res->addrs[res->cnt++] = (unsigned long) sym_addr;
10295         return 0;
10296 }
10297
10298 struct bpf_link *
10299 bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
10300                                       const char *pattern,
10301                                       const struct bpf_kprobe_multi_opts *opts)
10302 {
10303         LIBBPF_OPTS(bpf_link_create_opts, lopts);
10304         struct kprobe_multi_resolve res = {
10305                 .pattern = pattern,
10306         };
10307         struct bpf_link *link = NULL;
10308         char errmsg[STRERR_BUFSIZE];
10309         const unsigned long *addrs;
10310         int err, link_fd, prog_fd;
10311         const __u64 *cookies;
10312         const char **syms;
10313         bool retprobe;
10314         size_t cnt;
10315
10316         if (!OPTS_VALID(opts, bpf_kprobe_multi_opts))
10317                 return libbpf_err_ptr(-EINVAL);
10318
10319         syms    = OPTS_GET(opts, syms, false);
10320         addrs   = OPTS_GET(opts, addrs, false);
10321         cnt     = OPTS_GET(opts, cnt, false);
10322         cookies = OPTS_GET(opts, cookies, false);
10323
10324         if (!pattern && !addrs && !syms)
10325                 return libbpf_err_ptr(-EINVAL);
10326         if (pattern && (addrs || syms || cookies || cnt))
10327                 return libbpf_err_ptr(-EINVAL);
10328         if (!pattern && !cnt)
10329                 return libbpf_err_ptr(-EINVAL);
10330         if (addrs && syms)
10331                 return libbpf_err_ptr(-EINVAL);
10332
10333         if (pattern) {
10334                 err = libbpf_kallsyms_parse(resolve_kprobe_multi_cb, &res);
10335                 if (err)
10336                         goto error;
10337                 if (!res.cnt) {
10338                         err = -ENOENT;
10339                         goto error;
10340                 }
10341                 addrs = res.addrs;
10342                 cnt = res.cnt;
10343         }
10344
10345         retprobe = OPTS_GET(opts, retprobe, false);
10346
10347         lopts.kprobe_multi.syms = syms;
10348         lopts.kprobe_multi.addrs = addrs;
10349         lopts.kprobe_multi.cookies = cookies;
10350         lopts.kprobe_multi.cnt = cnt;
10351         lopts.kprobe_multi.flags = retprobe ? BPF_F_KPROBE_MULTI_RETURN : 0;
10352
10353         link = calloc(1, sizeof(*link));
10354         if (!link) {
10355                 err = -ENOMEM;
10356                 goto error;
10357         }
10358         link->detach = &bpf_link__detach_fd;
10359
10360         prog_fd = bpf_program__fd(prog);
10361         link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, &lopts);
10362         if (link_fd < 0) {
10363                 err = -errno;
10364                 pr_warn("prog '%s': failed to attach: %s\n",
10365                         prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10366                 goto error;
10367         }
10368         link->fd = link_fd;
10369         free(res.addrs);
10370         return link;
10371
10372 error:
10373         free(link);
10374         free(res.addrs);
10375         return libbpf_err_ptr(err);
10376 }
10377
10378 static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link)
10379 {
10380         DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts);
10381         unsigned long offset = 0;
10382         const char *func_name;
10383         char *func;
10384         int n;
10385
10386         *link = NULL;
10387
10388         /* no auto-attach for SEC("kprobe") and SEC("kretprobe") */
10389         if (strcmp(prog->sec_name, "kprobe") == 0 || strcmp(prog->sec_name, "kretprobe") == 0)
10390                 return 0;
10391
10392         opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/");
10393         if (opts.retprobe)
10394                 func_name = prog->sec_name + sizeof("kretprobe/") - 1;
10395         else
10396                 func_name = prog->sec_name + sizeof("kprobe/") - 1;
10397
10398         n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset);
10399         if (n < 1) {
10400                 pr_warn("kprobe name is invalid: %s\n", func_name);
10401                 return -EINVAL;
10402         }
10403         if (opts.retprobe && offset != 0) {
10404                 free(func);
10405                 pr_warn("kretprobes do not support offset specification\n");
10406                 return -EINVAL;
10407         }
10408
10409         opts.offset = offset;
10410         *link = bpf_program__attach_kprobe_opts(prog, func, &opts);
10411         free(func);
10412         return libbpf_get_error(*link);
10413 }
10414
10415 static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link)
10416 {
10417         LIBBPF_OPTS(bpf_ksyscall_opts, opts);
10418         const char *syscall_name;
10419
10420         *link = NULL;
10421
10422         /* no auto-attach for SEC("ksyscall") and SEC("kretsyscall") */
10423         if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0)
10424                 return 0;
10425
10426         opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/");
10427         if (opts.retprobe)
10428                 syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1;
10429         else
10430                 syscall_name = prog->sec_name + sizeof("ksyscall/") - 1;
10431
10432         *link = bpf_program__attach_ksyscall(prog, syscall_name, &opts);
10433         return *link ? 0 : -errno;
10434 }
10435
10436 static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link)
10437 {
10438         LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
10439         const char *spec;
10440         char *pattern;
10441         int n;
10442
10443         *link = NULL;
10444
10445         /* no auto-attach for SEC("kprobe.multi") and SEC("kretprobe.multi") */
10446         if (strcmp(prog->sec_name, "kprobe.multi") == 0 ||
10447             strcmp(prog->sec_name, "kretprobe.multi") == 0)
10448                 return 0;
10449
10450         opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/");
10451         if (opts.retprobe)
10452                 spec = prog->sec_name + sizeof("kretprobe.multi/") - 1;
10453         else
10454                 spec = prog->sec_name + sizeof("kprobe.multi/") - 1;
10455
10456         n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern);
10457         if (n < 1) {
10458                 pr_warn("kprobe multi pattern is invalid: %s\n", pattern);
10459                 return -EINVAL;
10460         }
10461
10462         *link = bpf_program__attach_kprobe_multi_opts(prog, pattern, &opts);
10463         free(pattern);
10464         return libbpf_get_error(*link);
10465 }
10466
10467 static void gen_uprobe_legacy_event_name(char *buf, size_t buf_sz,
10468                                          const char *binary_path, uint64_t offset)
10469 {
10470         int i;
10471
10472         snprintf(buf, buf_sz, "libbpf_%u_%s_0x%zx", getpid(), binary_path, (size_t)offset);
10473
10474         /* sanitize binary_path in the probe name */
10475         for (i = 0; buf[i]; i++) {
10476                 if (!isalnum(buf[i]))
10477                         buf[i] = '_';
10478         }
10479 }
10480
10481 static inline int add_uprobe_event_legacy(const char *probe_name, bool retprobe,
10482                                           const char *binary_path, size_t offset)
10483 {
10484         return append_to_file(tracefs_uprobe_events(), "%c:%s/%s %s:0x%zx",
10485                               retprobe ? 'r' : 'p',
10486                               retprobe ? "uretprobes" : "uprobes",
10487                               probe_name, binary_path, offset);
10488 }
10489
10490 static inline int remove_uprobe_event_legacy(const char *probe_name, bool retprobe)
10491 {
10492         return append_to_file(tracefs_uprobe_events(), "-:%s/%s",
10493                               retprobe ? "uretprobes" : "uprobes", probe_name);
10494 }
10495
10496 static int determine_uprobe_perf_type_legacy(const char *probe_name, bool retprobe)
10497 {
10498         char file[512];
10499
10500         snprintf(file, sizeof(file), "%s/events/%s/%s/id",
10501                  tracefs_path(), retprobe ? "uretprobes" : "uprobes", probe_name);
10502
10503         return parse_uint_from_file(file, "%d\n");
10504 }
10505
10506 static int perf_event_uprobe_open_legacy(const char *probe_name, bool retprobe,
10507                                          const char *binary_path, size_t offset, int pid)
10508 {
10509         const size_t attr_sz = sizeof(struct perf_event_attr);
10510         struct perf_event_attr attr;
10511         int type, pfd, err;
10512
10513         err = add_uprobe_event_legacy(probe_name, retprobe, binary_path, offset);
10514         if (err < 0) {
10515                 pr_warn("failed to add legacy uprobe event for %s:0x%zx: %d\n",
10516                         binary_path, (size_t)offset, err);
10517                 return err;
10518         }
10519         type = determine_uprobe_perf_type_legacy(probe_name, retprobe);
10520         if (type < 0) {
10521                 err = type;
10522                 pr_warn("failed to determine legacy uprobe event id for %s:0x%zx: %d\n",
10523                         binary_path, offset, err);
10524                 goto err_clean_legacy;
10525         }
10526
10527         memset(&attr, 0, attr_sz);
10528         attr.size = attr_sz;
10529         attr.config = type;
10530         attr.type = PERF_TYPE_TRACEPOINT;
10531
10532         pfd = syscall(__NR_perf_event_open, &attr,
10533                       pid < 0 ? -1 : pid, /* pid */
10534                       pid == -1 ? 0 : -1, /* cpu */
10535                       -1 /* group_fd */,  PERF_FLAG_FD_CLOEXEC);
10536         if (pfd < 0) {
10537                 err = -errno;
10538                 pr_warn("legacy uprobe perf_event_open() failed: %d\n", err);
10539                 goto err_clean_legacy;
10540         }
10541         return pfd;
10542
10543 err_clean_legacy:
10544         /* Clear the newly added legacy uprobe_event */
10545         remove_uprobe_event_legacy(probe_name, retprobe);
10546         return err;
10547 }
10548
10549 /* Return next ELF section of sh_type after scn, or first of that type if scn is NULL. */
10550 static Elf_Scn *elf_find_next_scn_by_type(Elf *elf, int sh_type, Elf_Scn *scn)
10551 {
10552         while ((scn = elf_nextscn(elf, scn)) != NULL) {
10553                 GElf_Shdr sh;
10554
10555                 if (!gelf_getshdr(scn, &sh))
10556                         continue;
10557                 if (sh.sh_type == sh_type)
10558                         return scn;
10559         }
10560         return NULL;
10561 }
10562
10563 /* Find offset of function name in object specified by path.  "name" matches
10564  * symbol name or name@@LIB for library functions.
10565  */
10566 static long elf_find_func_offset(const char *binary_path, const char *name)
10567 {
10568         int fd, i, sh_types[2] = { SHT_DYNSYM, SHT_SYMTAB };
10569         bool is_shared_lib, is_name_qualified;
10570         char errmsg[STRERR_BUFSIZE];
10571         long ret = -ENOENT;
10572         size_t name_len;
10573         GElf_Ehdr ehdr;
10574         Elf *elf;
10575
10576         fd = open(binary_path, O_RDONLY | O_CLOEXEC);
10577         if (fd < 0) {
10578                 ret = -errno;
10579                 pr_warn("failed to open %s: %s\n", binary_path,
10580                         libbpf_strerror_r(ret, errmsg, sizeof(errmsg)));
10581                 return ret;
10582         }
10583         elf = elf_begin(fd, ELF_C_READ_MMAP, NULL);
10584         if (!elf) {
10585                 pr_warn("elf: could not read elf from %s: %s\n", binary_path, elf_errmsg(-1));
10586                 close(fd);
10587                 return -LIBBPF_ERRNO__FORMAT;
10588         }
10589         if (!gelf_getehdr(elf, &ehdr)) {
10590                 pr_warn("elf: failed to get ehdr from %s: %s\n", binary_path, elf_errmsg(-1));
10591                 ret = -LIBBPF_ERRNO__FORMAT;
10592                 goto out;
10593         }
10594         /* for shared lib case, we do not need to calculate relative offset */
10595         is_shared_lib = ehdr.e_type == ET_DYN;
10596
10597         name_len = strlen(name);
10598         /* Does name specify "@@LIB"? */
10599         is_name_qualified = strstr(name, "@@") != NULL;
10600
10601         /* Search SHT_DYNSYM, SHT_SYMTAB for symbol.  This search order is used because if
10602          * a binary is stripped, it may only have SHT_DYNSYM, and a fully-statically
10603          * linked binary may not have SHT_DYMSYM, so absence of a section should not be
10604          * reported as a warning/error.
10605          */
10606         for (i = 0; i < ARRAY_SIZE(sh_types); i++) {
10607                 size_t nr_syms, strtabidx, idx;
10608                 Elf_Data *symbols = NULL;
10609                 Elf_Scn *scn = NULL;
10610                 int last_bind = -1;
10611                 const char *sname;
10612                 GElf_Shdr sh;
10613
10614                 scn = elf_find_next_scn_by_type(elf, sh_types[i], NULL);
10615                 if (!scn) {
10616                         pr_debug("elf: failed to find symbol table ELF sections in '%s'\n",
10617                                  binary_path);
10618                         continue;
10619                 }
10620                 if (!gelf_getshdr(scn, &sh))
10621                         continue;
10622                 strtabidx = sh.sh_link;
10623                 symbols = elf_getdata(scn, 0);
10624                 if (!symbols) {
10625                         pr_warn("elf: failed to get symbols for symtab section in '%s': %s\n",
10626                                 binary_path, elf_errmsg(-1));
10627                         ret = -LIBBPF_ERRNO__FORMAT;
10628                         goto out;
10629                 }
10630                 nr_syms = symbols->d_size / sh.sh_entsize;
10631
10632                 for (idx = 0; idx < nr_syms; idx++) {
10633                         int curr_bind;
10634                         GElf_Sym sym;
10635                         Elf_Scn *sym_scn;
10636                         GElf_Shdr sym_sh;
10637
10638                         if (!gelf_getsym(symbols, idx, &sym))
10639                                 continue;
10640
10641                         if (GELF_ST_TYPE(sym.st_info) != STT_FUNC)
10642                                 continue;
10643
10644                         sname = elf_strptr(elf, strtabidx, sym.st_name);
10645                         if (!sname)
10646                                 continue;
10647
10648                         curr_bind = GELF_ST_BIND(sym.st_info);
10649
10650                         /* User can specify func, func@@LIB or func@@LIB_VERSION. */
10651                         if (strncmp(sname, name, name_len) != 0)
10652                                 continue;
10653                         /* ...but we don't want a search for "foo" to match 'foo2" also, so any
10654                          * additional characters in sname should be of the form "@@LIB".
10655                          */
10656                         if (!is_name_qualified && sname[name_len] != '\0' && sname[name_len] != '@')
10657                                 continue;
10658
10659                         if (ret >= 0) {
10660                                 /* handle multiple matches */
10661                                 if (last_bind != STB_WEAK && curr_bind != STB_WEAK) {
10662                                         /* Only accept one non-weak bind. */
10663                                         pr_warn("elf: ambiguous match for '%s', '%s' in '%s'\n",
10664                                                 sname, name, binary_path);
10665                                         ret = -LIBBPF_ERRNO__FORMAT;
10666                                         goto out;
10667                                 } else if (curr_bind == STB_WEAK) {
10668                                         /* already have a non-weak bind, and
10669                                          * this is a weak bind, so ignore.
10670                                          */
10671                                         continue;
10672                                 }
10673                         }
10674
10675                         /* Transform symbol's virtual address (absolute for
10676                          * binaries and relative for shared libs) into file
10677                          * offset, which is what kernel is expecting for
10678                          * uprobe/uretprobe attachment.
10679                          * See Documentation/trace/uprobetracer.rst for more
10680                          * details.
10681                          * This is done by looking up symbol's containing
10682                          * section's header and using it's virtual address
10683                          * (sh_addr) and corresponding file offset (sh_offset)
10684                          * to transform sym.st_value (virtual address) into
10685                          * desired final file offset.
10686                          */
10687                         sym_scn = elf_getscn(elf, sym.st_shndx);
10688                         if (!sym_scn)
10689                                 continue;
10690                         if (!gelf_getshdr(sym_scn, &sym_sh))
10691                                 continue;
10692
10693                         ret = sym.st_value - sym_sh.sh_addr + sym_sh.sh_offset;
10694                         last_bind = curr_bind;
10695                 }
10696                 if (ret > 0)
10697                         break;
10698         }
10699
10700         if (ret > 0) {
10701                 pr_debug("elf: symbol address match for '%s' in '%s': 0x%lx\n", name, binary_path,
10702                          ret);
10703         } else {
10704                 if (ret == 0) {
10705                         pr_warn("elf: '%s' is 0 in symtab for '%s': %s\n", name, binary_path,
10706                                 is_shared_lib ? "should not be 0 in a shared library" :
10707                                                 "try using shared library path instead");
10708                         ret = -ENOENT;
10709                 } else {
10710                         pr_warn("elf: failed to find symbol '%s' in '%s'\n", name, binary_path);
10711                 }
10712         }
10713 out:
10714         elf_end(elf);
10715         close(fd);
10716         return ret;
10717 }
10718
10719 static const char *arch_specific_lib_paths(void)
10720 {
10721         /*
10722          * Based on https://packages.debian.org/sid/libc6.
10723          *
10724          * Assume that the traced program is built for the same architecture
10725          * as libbpf, which should cover the vast majority of cases.
10726          */
10727 #if defined(__x86_64__)
10728         return "/lib/x86_64-linux-gnu";
10729 #elif defined(__i386__)
10730         return "/lib/i386-linux-gnu";
10731 #elif defined(__s390x__)
10732         return "/lib/s390x-linux-gnu";
10733 #elif defined(__s390__)
10734         return "/lib/s390-linux-gnu";
10735 #elif defined(__arm__) && defined(__SOFTFP__)
10736         return "/lib/arm-linux-gnueabi";
10737 #elif defined(__arm__) && !defined(__SOFTFP__)
10738         return "/lib/arm-linux-gnueabihf";
10739 #elif defined(__aarch64__)
10740         return "/lib/aarch64-linux-gnu";
10741 #elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 64
10742         return "/lib/mips64el-linux-gnuabi64";
10743 #elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 32
10744         return "/lib/mipsel-linux-gnu";
10745 #elif defined(__powerpc64__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
10746         return "/lib/powerpc64le-linux-gnu";
10747 #elif defined(__sparc__) && defined(__arch64__)
10748         return "/lib/sparc64-linux-gnu";
10749 #elif defined(__riscv) && __riscv_xlen == 64
10750         return "/lib/riscv64-linux-gnu";
10751 #else
10752         return NULL;
10753 #endif
10754 }
10755
10756 /* Get full path to program/shared library. */
10757 static int resolve_full_path(const char *file, char *result, size_t result_sz)
10758 {
10759         const char *search_paths[3] = {};
10760         int i, perm;
10761
10762         if (str_has_sfx(file, ".so") || strstr(file, ".so.")) {
10763                 search_paths[0] = getenv("LD_LIBRARY_PATH");
10764                 search_paths[1] = "/usr/lib64:/usr/lib";
10765                 search_paths[2] = arch_specific_lib_paths();
10766                 perm = R_OK;
10767         } else {
10768                 search_paths[0] = getenv("PATH");
10769                 search_paths[1] = "/usr/bin:/usr/sbin";
10770                 perm = R_OK | X_OK;
10771         }
10772
10773         for (i = 0; i < ARRAY_SIZE(search_paths); i++) {
10774                 const char *s;
10775
10776                 if (!search_paths[i])
10777                         continue;
10778                 for (s = search_paths[i]; s != NULL; s = strchr(s, ':')) {
10779                         char *next_path;
10780                         int seg_len;
10781
10782                         if (s[0] == ':')
10783                                 s++;
10784                         next_path = strchr(s, ':');
10785                         seg_len = next_path ? next_path - s : strlen(s);
10786                         if (!seg_len)
10787                                 continue;
10788                         snprintf(result, result_sz, "%.*s/%s", seg_len, s, file);
10789                         /* ensure it has required permissions */
10790                         if (faccessat(AT_FDCWD, result, perm, AT_EACCESS) < 0)
10791                                 continue;
10792                         pr_debug("resolved '%s' to '%s'\n", file, result);
10793                         return 0;
10794                 }
10795         }
10796         return -ENOENT;
10797 }
10798
10799 LIBBPF_API struct bpf_link *
10800 bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid,
10801                                 const char *binary_path, size_t func_offset,
10802                                 const struct bpf_uprobe_opts *opts)
10803 {
10804         DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
10805         char errmsg[STRERR_BUFSIZE], *legacy_probe = NULL;
10806         char full_binary_path[PATH_MAX];
10807         struct bpf_link *link;
10808         size_t ref_ctr_off;
10809         int pfd, err;
10810         bool retprobe, legacy;
10811         const char *func_name;
10812
10813         if (!OPTS_VALID(opts, bpf_uprobe_opts))
10814                 return libbpf_err_ptr(-EINVAL);
10815
10816         retprobe = OPTS_GET(opts, retprobe, false);
10817         ref_ctr_off = OPTS_GET(opts, ref_ctr_offset, 0);
10818         pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
10819
10820         if (!binary_path)
10821                 return libbpf_err_ptr(-EINVAL);
10822
10823         if (!strchr(binary_path, '/')) {
10824                 err = resolve_full_path(binary_path, full_binary_path,
10825                                         sizeof(full_binary_path));
10826                 if (err) {
10827                         pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",
10828                                 prog->name, binary_path, err);
10829                         return libbpf_err_ptr(err);
10830                 }
10831                 binary_path = full_binary_path;
10832         }
10833         func_name = OPTS_GET(opts, func_name, NULL);
10834         if (func_name) {
10835                 long sym_off;
10836
10837                 sym_off = elf_find_func_offset(binary_path, func_name);
10838                 if (sym_off < 0)
10839                         return libbpf_err_ptr(sym_off);
10840                 func_offset += sym_off;
10841         }
10842
10843         legacy = determine_uprobe_perf_type() < 0;
10844         if (!legacy) {
10845                 pfd = perf_event_open_probe(true /* uprobe */, retprobe, binary_path,
10846                                             func_offset, pid, ref_ctr_off);
10847         } else {
10848                 char probe_name[PATH_MAX + 64];
10849
10850                 if (ref_ctr_off)
10851                         return libbpf_err_ptr(-EINVAL);
10852
10853                 gen_uprobe_legacy_event_name(probe_name, sizeof(probe_name),
10854                                              binary_path, func_offset);
10855
10856                 legacy_probe = strdup(probe_name);
10857                 if (!legacy_probe)
10858                         return libbpf_err_ptr(-ENOMEM);
10859
10860                 pfd = perf_event_uprobe_open_legacy(legacy_probe, retprobe,
10861                                                     binary_path, func_offset, pid);
10862         }
10863         if (pfd < 0) {
10864                 err = -errno;
10865                 pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n",
10866                         prog->name, retprobe ? "uretprobe" : "uprobe",
10867                         binary_path, func_offset,
10868                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10869                 goto err_out;
10870         }
10871
10872         link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
10873         err = libbpf_get_error(link);
10874         if (err) {
10875                 close(pfd);
10876                 pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n",
10877                         prog->name, retprobe ? "uretprobe" : "uprobe",
10878                         binary_path, func_offset,
10879                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10880                 goto err_clean_legacy;
10881         }
10882         if (legacy) {
10883                 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
10884
10885                 perf_link->legacy_probe_name = legacy_probe;
10886                 perf_link->legacy_is_kprobe = false;
10887                 perf_link->legacy_is_retprobe = retprobe;
10888         }
10889         return link;
10890
10891 err_clean_legacy:
10892         if (legacy)
10893                 remove_uprobe_event_legacy(legacy_probe, retprobe);
10894 err_out:
10895         free(legacy_probe);
10896         return libbpf_err_ptr(err);
10897 }
10898
10899 /* Format of u[ret]probe section definition supporting auto-attach:
10900  * u[ret]probe/binary:function[+offset]
10901  *
10902  * binary can be an absolute/relative path or a filename; the latter is resolved to a
10903  * full binary path via bpf_program__attach_uprobe_opts.
10904  *
10905  * Specifying uprobe+ ensures we carry out strict matching; either "uprobe" must be
10906  * specified (and auto-attach is not possible) or the above format is specified for
10907  * auto-attach.
10908  */
10909 static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link)
10910 {
10911         DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts);
10912         char *probe_type = NULL, *binary_path = NULL, *func_name = NULL;
10913         int n, ret = -EINVAL;
10914         long offset = 0;
10915
10916         *link = NULL;
10917
10918         n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[a-zA-Z0-9_.]+%li",
10919                    &probe_type, &binary_path, &func_name, &offset);
10920         switch (n) {
10921         case 1:
10922                 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */
10923                 ret = 0;
10924                 break;
10925         case 2:
10926                 pr_warn("prog '%s': section '%s' missing ':function[+offset]' specification\n",
10927                         prog->name, prog->sec_name);
10928                 break;
10929         case 3:
10930         case 4:
10931                 opts.retprobe = strcmp(probe_type, "uretprobe") == 0 ||
10932                                 strcmp(probe_type, "uretprobe.s") == 0;
10933                 if (opts.retprobe && offset != 0) {
10934                         pr_warn("prog '%s': uretprobes do not support offset specification\n",
10935                                 prog->name);
10936                         break;
10937                 }
10938                 opts.func_name = func_name;
10939                 *link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts);
10940                 ret = libbpf_get_error(*link);
10941                 break;
10942         default:
10943                 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,
10944                         prog->sec_name);
10945                 break;
10946         }
10947         free(probe_type);
10948         free(binary_path);
10949         free(func_name);
10950
10951         return ret;
10952 }
10953
10954 struct bpf_link *bpf_program__attach_uprobe(const struct bpf_program *prog,
10955                                             bool retprobe, pid_t pid,
10956                                             const char *binary_path,
10957                                             size_t func_offset)
10958 {
10959         DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts, .retprobe = retprobe);
10960
10961         return bpf_program__attach_uprobe_opts(prog, pid, binary_path, func_offset, &opts);
10962 }
10963
10964 struct bpf_link *bpf_program__attach_usdt(const struct bpf_program *prog,
10965                                           pid_t pid, const char *binary_path,
10966                                           const char *usdt_provider, const char *usdt_name,
10967                                           const struct bpf_usdt_opts *opts)
10968 {
10969         char resolved_path[512];
10970         struct bpf_object *obj = prog->obj;
10971         struct bpf_link *link;
10972         __u64 usdt_cookie;
10973         int err;
10974
10975         if (!OPTS_VALID(opts, bpf_uprobe_opts))
10976                 return libbpf_err_ptr(-EINVAL);
10977
10978         if (bpf_program__fd(prog) < 0) {
10979                 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n",
10980                         prog->name);
10981                 return libbpf_err_ptr(-EINVAL);
10982         }
10983
10984         if (!binary_path)
10985                 return libbpf_err_ptr(-EINVAL);
10986
10987         if (!strchr(binary_path, '/')) {
10988                 err = resolve_full_path(binary_path, resolved_path, sizeof(resolved_path));
10989                 if (err) {
10990                         pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",
10991                                 prog->name, binary_path, err);
10992                         return libbpf_err_ptr(err);
10993                 }
10994                 binary_path = resolved_path;
10995         }
10996
10997         /* USDT manager is instantiated lazily on first USDT attach. It will
10998          * be destroyed together with BPF object in bpf_object__close().
10999          */
11000         if (IS_ERR(obj->usdt_man))
11001                 return libbpf_ptr(obj->usdt_man);
11002         if (!obj->usdt_man) {
11003                 obj->usdt_man = usdt_manager_new(obj);
11004                 if (IS_ERR(obj->usdt_man))
11005                         return libbpf_ptr(obj->usdt_man);
11006         }
11007
11008         usdt_cookie = OPTS_GET(opts, usdt_cookie, 0);
11009         link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path,
11010                                         usdt_provider, usdt_name, usdt_cookie);
11011         err = libbpf_get_error(link);
11012         if (err)
11013                 return libbpf_err_ptr(err);
11014         return link;
11015 }
11016
11017 static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11018 {
11019         char *path = NULL, *provider = NULL, *name = NULL;
11020         const char *sec_name;
11021         int n, err;
11022
11023         sec_name = bpf_program__section_name(prog);
11024         if (strcmp(sec_name, "usdt") == 0) {
11025                 /* no auto-attach for just SEC("usdt") */
11026                 *link = NULL;
11027                 return 0;
11028         }
11029
11030         n = sscanf(sec_name, "usdt/%m[^:]:%m[^:]:%m[^:]", &path, &provider, &name);
11031         if (n != 3) {
11032                 pr_warn("invalid section '%s', expected SEC(\"usdt/<path>:<provider>:<name>\")\n",
11033                         sec_name);
11034                 err = -EINVAL;
11035         } else {
11036                 *link = bpf_program__attach_usdt(prog, -1 /* any process */, path,
11037                                                  provider, name, NULL);
11038                 err = libbpf_get_error(*link);
11039         }
11040         free(path);
11041         free(provider);
11042         free(name);
11043         return err;
11044 }
11045
11046 static int determine_tracepoint_id(const char *tp_category,
11047                                    const char *tp_name)
11048 {
11049         char file[PATH_MAX];
11050         int ret;
11051
11052         ret = snprintf(file, sizeof(file), "%s/events/%s/%s/id",
11053                        tracefs_path(), tp_category, tp_name);
11054         if (ret < 0)
11055                 return -errno;
11056         if (ret >= sizeof(file)) {
11057                 pr_debug("tracepoint %s/%s path is too long\n",
11058                          tp_category, tp_name);
11059                 return -E2BIG;
11060         }
11061         return parse_uint_from_file(file, "%d\n");
11062 }
11063
11064 static int perf_event_open_tracepoint(const char *tp_category,
11065                                       const char *tp_name)
11066 {
11067         const size_t attr_sz = sizeof(struct perf_event_attr);
11068         struct perf_event_attr attr;
11069         char errmsg[STRERR_BUFSIZE];
11070         int tp_id, pfd, err;
11071
11072         tp_id = determine_tracepoint_id(tp_category, tp_name);
11073         if (tp_id < 0) {
11074                 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n",
11075                         tp_category, tp_name,
11076                         libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg)));
11077                 return tp_id;
11078         }
11079
11080         memset(&attr, 0, attr_sz);
11081         attr.type = PERF_TYPE_TRACEPOINT;
11082         attr.size = attr_sz;
11083         attr.config = tp_id;
11084
11085         pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */,
11086                       -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
11087         if (pfd < 0) {
11088                 err = -errno;
11089                 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n",
11090                         tp_category, tp_name,
11091                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11092                 return err;
11093         }
11094         return pfd;
11095 }
11096
11097 struct bpf_link *bpf_program__attach_tracepoint_opts(const struct bpf_program *prog,
11098                                                      const char *tp_category,
11099                                                      const char *tp_name,
11100                                                      const struct bpf_tracepoint_opts *opts)
11101 {
11102         DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
11103         char errmsg[STRERR_BUFSIZE];
11104         struct bpf_link *link;
11105         int pfd, err;
11106
11107         if (!OPTS_VALID(opts, bpf_tracepoint_opts))
11108                 return libbpf_err_ptr(-EINVAL);
11109
11110         pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
11111
11112         pfd = perf_event_open_tracepoint(tp_category, tp_name);
11113         if (pfd < 0) {
11114                 pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n",
11115                         prog->name, tp_category, tp_name,
11116                         libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
11117                 return libbpf_err_ptr(pfd);
11118         }
11119         link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
11120         err = libbpf_get_error(link);
11121         if (err) {
11122                 close(pfd);
11123                 pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n",
11124                         prog->name, tp_category, tp_name,
11125                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11126                 return libbpf_err_ptr(err);
11127         }
11128         return link;
11129 }
11130
11131 struct bpf_link *bpf_program__attach_tracepoint(const struct bpf_program *prog,
11132                                                 const char *tp_category,
11133                                                 const char *tp_name)
11134 {
11135         return bpf_program__attach_tracepoint_opts(prog, tp_category, tp_name, NULL);
11136 }
11137
11138 static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11139 {
11140         char *sec_name, *tp_cat, *tp_name;
11141
11142         *link = NULL;
11143
11144         /* no auto-attach for SEC("tp") or SEC("tracepoint") */
11145         if (strcmp(prog->sec_name, "tp") == 0 || strcmp(prog->sec_name, "tracepoint") == 0)
11146                 return 0;
11147
11148         sec_name = strdup(prog->sec_name);
11149         if (!sec_name)
11150                 return -ENOMEM;
11151
11152         /* extract "tp/<category>/<name>" or "tracepoint/<category>/<name>" */
11153         if (str_has_pfx(prog->sec_name, "tp/"))
11154                 tp_cat = sec_name + sizeof("tp/") - 1;
11155         else
11156                 tp_cat = sec_name + sizeof("tracepoint/") - 1;
11157         tp_name = strchr(tp_cat, '/');
11158         if (!tp_name) {
11159                 free(sec_name);
11160                 return -EINVAL;
11161         }
11162         *tp_name = '\0';
11163         tp_name++;
11164
11165         *link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name);
11166         free(sec_name);
11167         return libbpf_get_error(*link);
11168 }
11169
11170 struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *prog,
11171                                                     const char *tp_name)
11172 {
11173         char errmsg[STRERR_BUFSIZE];
11174         struct bpf_link *link;
11175         int prog_fd, pfd;
11176
11177         prog_fd = bpf_program__fd(prog);
11178         if (prog_fd < 0) {
11179                 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
11180                 return libbpf_err_ptr(-EINVAL);
11181         }
11182
11183         link = calloc(1, sizeof(*link));
11184         if (!link)
11185                 return libbpf_err_ptr(-ENOMEM);
11186         link->detach = &bpf_link__detach_fd;
11187
11188         pfd = bpf_raw_tracepoint_open(tp_name, prog_fd);
11189         if (pfd < 0) {
11190                 pfd = -errno;
11191                 free(link);
11192                 pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n",
11193                         prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
11194                 return libbpf_err_ptr(pfd);
11195         }
11196         link->fd = pfd;
11197         return link;
11198 }
11199
11200 static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11201 {
11202         static const char *const prefixes[] = {
11203                 "raw_tp",
11204                 "raw_tracepoint",
11205                 "raw_tp.w",
11206                 "raw_tracepoint.w",
11207         };
11208         size_t i;
11209         const char *tp_name = NULL;
11210
11211         *link = NULL;
11212
11213         for (i = 0; i < ARRAY_SIZE(prefixes); i++) {
11214                 size_t pfx_len;
11215
11216                 if (!str_has_pfx(prog->sec_name, prefixes[i]))
11217                         continue;
11218
11219                 pfx_len = strlen(prefixes[i]);
11220                 /* no auto-attach case of, e.g., SEC("raw_tp") */
11221                 if (prog->sec_name[pfx_len] == '\0')
11222                         return 0;
11223
11224                 if (prog->sec_name[pfx_len] != '/')
11225                         continue;
11226
11227                 tp_name = prog->sec_name + pfx_len + 1;
11228                 break;
11229         }
11230
11231         if (!tp_name) {
11232                 pr_warn("prog '%s': invalid section name '%s'\n",
11233                         prog->name, prog->sec_name);
11234                 return -EINVAL;
11235         }
11236
11237         *link = bpf_program__attach_raw_tracepoint(prog, tp_name);
11238         return libbpf_get_error(*link);
11239 }
11240
11241 /* Common logic for all BPF program types that attach to a btf_id */
11242 static struct bpf_link *bpf_program__attach_btf_id(const struct bpf_program *prog,
11243                                                    const struct bpf_trace_opts *opts)
11244 {
11245         LIBBPF_OPTS(bpf_link_create_opts, link_opts);
11246         char errmsg[STRERR_BUFSIZE];
11247         struct bpf_link *link;
11248         int prog_fd, pfd;
11249
11250         if (!OPTS_VALID(opts, bpf_trace_opts))
11251                 return libbpf_err_ptr(-EINVAL);
11252
11253         prog_fd = bpf_program__fd(prog);
11254         if (prog_fd < 0) {
11255                 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
11256                 return libbpf_err_ptr(-EINVAL);
11257         }
11258
11259         link = calloc(1, sizeof(*link));
11260         if (!link)
11261                 return libbpf_err_ptr(-ENOMEM);
11262         link->detach = &bpf_link__detach_fd;
11263
11264         /* libbpf is smart enough to redirect to BPF_RAW_TRACEPOINT_OPEN on old kernels */
11265         link_opts.tracing.cookie = OPTS_GET(opts, cookie, 0);
11266         pfd = bpf_link_create(prog_fd, 0, bpf_program__expected_attach_type(prog), &link_opts);
11267         if (pfd < 0) {
11268                 pfd = -errno;
11269                 free(link);
11270                 pr_warn("prog '%s': failed to attach: %s\n",
11271                         prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
11272                 return libbpf_err_ptr(pfd);
11273         }
11274         link->fd = pfd;
11275         return link;
11276 }
11277
11278 struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog)
11279 {
11280         return bpf_program__attach_btf_id(prog, NULL);
11281 }
11282
11283 struct bpf_link *bpf_program__attach_trace_opts(const struct bpf_program *prog,
11284                                                 const struct bpf_trace_opts *opts)
11285 {
11286         return bpf_program__attach_btf_id(prog, opts);
11287 }
11288
11289 struct bpf_link *bpf_program__attach_lsm(const struct bpf_program *prog)
11290 {
11291         return bpf_program__attach_btf_id(prog, NULL);
11292 }
11293
11294 static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11295 {
11296         *link = bpf_program__attach_trace(prog);
11297         return libbpf_get_error(*link);
11298 }
11299
11300 static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11301 {
11302         *link = bpf_program__attach_lsm(prog);
11303         return libbpf_get_error(*link);
11304 }
11305
11306 static struct bpf_link *
11307 bpf_program__attach_fd(const struct bpf_program *prog, int target_fd, int btf_id,
11308                        const char *target_name)
11309 {
11310         DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts,
11311                             .target_btf_id = btf_id);
11312         enum bpf_attach_type attach_type;
11313         char errmsg[STRERR_BUFSIZE];
11314         struct bpf_link *link;
11315         int prog_fd, link_fd;
11316
11317         prog_fd = bpf_program__fd(prog);
11318         if (prog_fd < 0) {
11319                 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
11320                 return libbpf_err_ptr(-EINVAL);
11321         }
11322
11323         link = calloc(1, sizeof(*link));
11324         if (!link)
11325                 return libbpf_err_ptr(-ENOMEM);
11326         link->detach = &bpf_link__detach_fd;
11327
11328         attach_type = bpf_program__expected_attach_type(prog);
11329         link_fd = bpf_link_create(prog_fd, target_fd, attach_type, &opts);
11330         if (link_fd < 0) {
11331                 link_fd = -errno;
11332                 free(link);
11333                 pr_warn("prog '%s': failed to attach to %s: %s\n",
11334                         prog->name, target_name,
11335                         libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
11336                 return libbpf_err_ptr(link_fd);
11337         }
11338         link->fd = link_fd;
11339         return link;
11340 }
11341
11342 struct bpf_link *
11343 bpf_program__attach_cgroup(const struct bpf_program *prog, int cgroup_fd)
11344 {
11345         return bpf_program__attach_fd(prog, cgroup_fd, 0, "cgroup");
11346 }
11347
11348 struct bpf_link *
11349 bpf_program__attach_netns(const struct bpf_program *prog, int netns_fd)
11350 {
11351         return bpf_program__attach_fd(prog, netns_fd, 0, "netns");
11352 }
11353
11354 struct bpf_link *bpf_program__attach_xdp(const struct bpf_program *prog, int ifindex)
11355 {
11356         /* target_fd/target_ifindex use the same field in LINK_CREATE */
11357         return bpf_program__attach_fd(prog, ifindex, 0, "xdp");
11358 }
11359
11360 struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog,
11361                                               int target_fd,
11362                                               const char *attach_func_name)
11363 {
11364         int btf_id;
11365
11366         if (!!target_fd != !!attach_func_name) {
11367                 pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n",
11368                         prog->name);
11369                 return libbpf_err_ptr(-EINVAL);
11370         }
11371
11372         if (prog->type != BPF_PROG_TYPE_EXT) {
11373                 pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace",
11374                         prog->name);
11375                 return libbpf_err_ptr(-EINVAL);
11376         }
11377
11378         if (target_fd) {
11379                 btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd);
11380                 if (btf_id < 0)
11381                         return libbpf_err_ptr(btf_id);
11382
11383                 return bpf_program__attach_fd(prog, target_fd, btf_id, "freplace");
11384         } else {
11385                 /* no target, so use raw_tracepoint_open for compatibility
11386                  * with old kernels
11387                  */
11388                 return bpf_program__attach_trace(prog);
11389         }
11390 }
11391
11392 struct bpf_link *
11393 bpf_program__attach_iter(const struct bpf_program *prog,
11394                          const struct bpf_iter_attach_opts *opts)
11395 {
11396         DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);
11397         char errmsg[STRERR_BUFSIZE];
11398         struct bpf_link *link;
11399         int prog_fd, link_fd;
11400         __u32 target_fd = 0;
11401
11402         if (!OPTS_VALID(opts, bpf_iter_attach_opts))
11403                 return libbpf_err_ptr(-EINVAL);
11404
11405         link_create_opts.iter_info = OPTS_GET(opts, link_info, (void *)0);
11406         link_create_opts.iter_info_len = OPTS_GET(opts, link_info_len, 0);
11407
11408         prog_fd = bpf_program__fd(prog);
11409         if (prog_fd < 0) {
11410                 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
11411                 return libbpf_err_ptr(-EINVAL);
11412         }
11413
11414         link = calloc(1, sizeof(*link));
11415         if (!link)
11416                 return libbpf_err_ptr(-ENOMEM);
11417         link->detach = &bpf_link__detach_fd;
11418
11419         link_fd = bpf_link_create(prog_fd, target_fd, BPF_TRACE_ITER,
11420                                   &link_create_opts);
11421         if (link_fd < 0) {
11422                 link_fd = -errno;
11423                 free(link);
11424                 pr_warn("prog '%s': failed to attach to iterator: %s\n",
11425                         prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
11426                 return libbpf_err_ptr(link_fd);
11427         }
11428         link->fd = link_fd;
11429         return link;
11430 }
11431
11432 static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11433 {
11434         *link = bpf_program__attach_iter(prog, NULL);
11435         return libbpf_get_error(*link);
11436 }
11437
11438 struct bpf_link *bpf_program__attach(const struct bpf_program *prog)
11439 {
11440         struct bpf_link *link = NULL;
11441         int err;
11442
11443         if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
11444                 return libbpf_err_ptr(-EOPNOTSUPP);
11445
11446         err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link);
11447         if (err)
11448                 return libbpf_err_ptr(err);
11449
11450         /* When calling bpf_program__attach() explicitly, auto-attach support
11451          * is expected to work, so NULL returned link is considered an error.
11452          * This is different for skeleton's attach, see comment in
11453          * bpf_object__attach_skeleton().
11454          */
11455         if (!link)
11456                 return libbpf_err_ptr(-EOPNOTSUPP);
11457
11458         return link;
11459 }
11460
11461 static int bpf_link__detach_struct_ops(struct bpf_link *link)
11462 {
11463         __u32 zero = 0;
11464
11465         if (bpf_map_delete_elem(link->fd, &zero))
11466                 return -errno;
11467
11468         return 0;
11469 }
11470
11471 struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map)
11472 {
11473         struct bpf_struct_ops *st_ops;
11474         struct bpf_link *link;
11475         __u32 i, zero = 0;
11476         int err;
11477
11478         if (!bpf_map__is_struct_ops(map) || map->fd == -1)
11479                 return libbpf_err_ptr(-EINVAL);
11480
11481         link = calloc(1, sizeof(*link));
11482         if (!link)
11483                 return libbpf_err_ptr(-EINVAL);
11484
11485         st_ops = map->st_ops;
11486         for (i = 0; i < btf_vlen(st_ops->type); i++) {
11487                 struct bpf_program *prog = st_ops->progs[i];
11488                 void *kern_data;
11489                 int prog_fd;
11490
11491                 if (!prog)
11492                         continue;
11493
11494                 prog_fd = bpf_program__fd(prog);
11495                 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i];
11496                 *(unsigned long *)kern_data = prog_fd;
11497         }
11498
11499         err = bpf_map_update_elem(map->fd, &zero, st_ops->kern_vdata, 0);
11500         if (err) {
11501                 err = -errno;
11502                 free(link);
11503                 return libbpf_err_ptr(err);
11504         }
11505
11506         link->detach = bpf_link__detach_struct_ops;
11507         link->fd = map->fd;
11508
11509         return link;
11510 }
11511
11512 typedef enum bpf_perf_event_ret (*bpf_perf_event_print_t)(struct perf_event_header *hdr,
11513                                                           void *private_data);
11514
11515 static enum bpf_perf_event_ret
11516 perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
11517                        void **copy_mem, size_t *copy_size,
11518                        bpf_perf_event_print_t fn, void *private_data)
11519 {
11520         struct perf_event_mmap_page *header = mmap_mem;
11521         __u64 data_head = ring_buffer_read_head(header);
11522         __u64 data_tail = header->data_tail;
11523         void *base = ((__u8 *)header) + page_size;
11524         int ret = LIBBPF_PERF_EVENT_CONT;
11525         struct perf_event_header *ehdr;
11526         size_t ehdr_size;
11527
11528         while (data_head != data_tail) {
11529                 ehdr = base + (data_tail & (mmap_size - 1));
11530                 ehdr_size = ehdr->size;
11531
11532                 if (((void *)ehdr) + ehdr_size > base + mmap_size) {
11533                         void *copy_start = ehdr;
11534                         size_t len_first = base + mmap_size - copy_start;
11535                         size_t len_secnd = ehdr_size - len_first;
11536
11537                         if (*copy_size < ehdr_size) {
11538                                 free(*copy_mem);
11539                                 *copy_mem = malloc(ehdr_size);
11540                                 if (!*copy_mem) {
11541                                         *copy_size = 0;
11542                                         ret = LIBBPF_PERF_EVENT_ERROR;
11543                                         break;
11544                                 }
11545                                 *copy_size = ehdr_size;
11546                         }
11547
11548                         memcpy(*copy_mem, copy_start, len_first);
11549                         memcpy(*copy_mem + len_first, base, len_secnd);
11550                         ehdr = *copy_mem;
11551                 }
11552
11553                 ret = fn(ehdr, private_data);
11554                 data_tail += ehdr_size;
11555                 if (ret != LIBBPF_PERF_EVENT_CONT)
11556                         break;
11557         }
11558
11559         ring_buffer_write_tail(header, data_tail);
11560         return libbpf_err(ret);
11561 }
11562
11563 struct perf_buffer;
11564
11565 struct perf_buffer_params {
11566         struct perf_event_attr *attr;
11567         /* if event_cb is specified, it takes precendence */
11568         perf_buffer_event_fn event_cb;
11569         /* sample_cb and lost_cb are higher-level common-case callbacks */
11570         perf_buffer_sample_fn sample_cb;
11571         perf_buffer_lost_fn lost_cb;
11572         void *ctx;
11573         int cpu_cnt;
11574         int *cpus;
11575         int *map_keys;
11576 };
11577
11578 struct perf_cpu_buf {
11579         struct perf_buffer *pb;
11580         void *base; /* mmap()'ed memory */
11581         void *buf; /* for reconstructing segmented data */
11582         size_t buf_size;
11583         int fd;
11584         int cpu;
11585         int map_key;
11586 };
11587
11588 struct perf_buffer {
11589         perf_buffer_event_fn event_cb;
11590         perf_buffer_sample_fn sample_cb;
11591         perf_buffer_lost_fn lost_cb;
11592         void *ctx; /* passed into callbacks */
11593
11594         size_t page_size;
11595         size_t mmap_size;
11596         struct perf_cpu_buf **cpu_bufs;
11597         struct epoll_event *events;
11598         int cpu_cnt; /* number of allocated CPU buffers */
11599         int epoll_fd; /* perf event FD */
11600         int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */
11601 };
11602
11603 static void perf_buffer__free_cpu_buf(struct perf_buffer *pb,
11604                                       struct perf_cpu_buf *cpu_buf)
11605 {
11606         if (!cpu_buf)
11607                 return;
11608         if (cpu_buf->base &&
11609             munmap(cpu_buf->base, pb->mmap_size + pb->page_size))
11610                 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu);
11611         if (cpu_buf->fd >= 0) {
11612                 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0);
11613                 close(cpu_buf->fd);
11614         }
11615         free(cpu_buf->buf);
11616         free(cpu_buf);
11617 }
11618
11619 void perf_buffer__free(struct perf_buffer *pb)
11620 {
11621         int i;
11622
11623         if (IS_ERR_OR_NULL(pb))
11624                 return;
11625         if (pb->cpu_bufs) {
11626                 for (i = 0; i < pb->cpu_cnt; i++) {
11627                         struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
11628
11629                         if (!cpu_buf)
11630                                 continue;
11631
11632                         bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);
11633                         perf_buffer__free_cpu_buf(pb, cpu_buf);
11634                 }
11635                 free(pb->cpu_bufs);
11636         }
11637         if (pb->epoll_fd >= 0)
11638                 close(pb->epoll_fd);
11639         free(pb->events);
11640         free(pb);
11641 }
11642
11643 static struct perf_cpu_buf *
11644 perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr,
11645                           int cpu, int map_key)
11646 {
11647         struct perf_cpu_buf *cpu_buf;
11648         char msg[STRERR_BUFSIZE];
11649         int err;
11650
11651         cpu_buf = calloc(1, sizeof(*cpu_buf));
11652         if (!cpu_buf)
11653                 return ERR_PTR(-ENOMEM);
11654
11655         cpu_buf->pb = pb;
11656         cpu_buf->cpu = cpu;
11657         cpu_buf->map_key = map_key;
11658
11659         cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu,
11660                               -1, PERF_FLAG_FD_CLOEXEC);
11661         if (cpu_buf->fd < 0) {
11662                 err = -errno;
11663                 pr_warn("failed to open perf buffer event on cpu #%d: %s\n",
11664                         cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
11665                 goto error;
11666         }
11667
11668         cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,
11669                              PROT_READ | PROT_WRITE, MAP_SHARED,
11670                              cpu_buf->fd, 0);
11671         if (cpu_buf->base == MAP_FAILED) {
11672                 cpu_buf->base = NULL;
11673                 err = -errno;
11674                 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n",
11675                         cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
11676                 goto error;
11677         }
11678
11679         if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
11680                 err = -errno;
11681                 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n",
11682                         cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
11683                 goto error;
11684         }
11685
11686         return cpu_buf;
11687
11688 error:
11689         perf_buffer__free_cpu_buf(pb, cpu_buf);
11690         return (struct perf_cpu_buf *)ERR_PTR(err);
11691 }
11692
11693 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
11694                                               struct perf_buffer_params *p);
11695
11696 struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt,
11697                                      perf_buffer_sample_fn sample_cb,
11698                                      perf_buffer_lost_fn lost_cb,
11699                                      void *ctx,
11700                                      const struct perf_buffer_opts *opts)
11701 {
11702         const size_t attr_sz = sizeof(struct perf_event_attr);
11703         struct perf_buffer_params p = {};
11704         struct perf_event_attr attr;
11705
11706         if (!OPTS_VALID(opts, perf_buffer_opts))
11707                 return libbpf_err_ptr(-EINVAL);
11708
11709         memset(&attr, 0, attr_sz);
11710         attr.size = attr_sz;
11711         attr.config = PERF_COUNT_SW_BPF_OUTPUT;
11712         attr.type = PERF_TYPE_SOFTWARE;
11713         attr.sample_type = PERF_SAMPLE_RAW;
11714         attr.sample_period = 1;
11715         attr.wakeup_events = 1;
11716
11717         p.attr = &attr;
11718         p.sample_cb = sample_cb;
11719         p.lost_cb = lost_cb;
11720         p.ctx = ctx;
11721
11722         return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p));
11723 }
11724
11725 struct perf_buffer *perf_buffer__new_raw(int map_fd, size_t page_cnt,
11726                                          struct perf_event_attr *attr,
11727                                          perf_buffer_event_fn event_cb, void *ctx,
11728                                          const struct perf_buffer_raw_opts *opts)
11729 {
11730         struct perf_buffer_params p = {};
11731
11732         if (!attr)
11733                 return libbpf_err_ptr(-EINVAL);
11734
11735         if (!OPTS_VALID(opts, perf_buffer_raw_opts))
11736                 return libbpf_err_ptr(-EINVAL);
11737
11738         p.attr = attr;
11739         p.event_cb = event_cb;
11740         p.ctx = ctx;
11741         p.cpu_cnt = OPTS_GET(opts, cpu_cnt, 0);
11742         p.cpus = OPTS_GET(opts, cpus, NULL);
11743         p.map_keys = OPTS_GET(opts, map_keys, NULL);
11744
11745         return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p));
11746 }
11747
11748 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
11749                                               struct perf_buffer_params *p)
11750 {
11751         const char *online_cpus_file = "/sys/devices/system/cpu/online";
11752         struct bpf_map_info map;
11753         char msg[STRERR_BUFSIZE];
11754         struct perf_buffer *pb;
11755         bool *online = NULL;
11756         __u32 map_info_len;
11757         int err, i, j, n;
11758
11759         if (page_cnt == 0 || (page_cnt & (page_cnt - 1))) {
11760                 pr_warn("page count should be power of two, but is %zu\n",
11761                         page_cnt);
11762                 return ERR_PTR(-EINVAL);
11763         }
11764
11765         /* best-effort sanity checks */
11766         memset(&map, 0, sizeof(map));
11767         map_info_len = sizeof(map);
11768         err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len);
11769         if (err) {
11770                 err = -errno;
11771                 /* if BPF_OBJ_GET_INFO_BY_FD is supported, will return
11772                  * -EBADFD, -EFAULT, or -E2BIG on real error
11773                  */
11774                 if (err != -EINVAL) {
11775                         pr_warn("failed to get map info for map FD %d: %s\n",
11776                                 map_fd, libbpf_strerror_r(err, msg, sizeof(msg)));
11777                         return ERR_PTR(err);
11778                 }
11779                 pr_debug("failed to get map info for FD %d; API not supported? Ignoring...\n",
11780                          map_fd);
11781         } else {
11782                 if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) {
11783                         pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n",
11784                                 map.name);
11785                         return ERR_PTR(-EINVAL);
11786                 }
11787         }
11788
11789         pb = calloc(1, sizeof(*pb));
11790         if (!pb)
11791                 return ERR_PTR(-ENOMEM);
11792
11793         pb->event_cb = p->event_cb;
11794         pb->sample_cb = p->sample_cb;
11795         pb->lost_cb = p->lost_cb;
11796         pb->ctx = p->ctx;
11797
11798         pb->page_size = getpagesize();
11799         pb->mmap_size = pb->page_size * page_cnt;
11800         pb->map_fd = map_fd;
11801
11802         pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
11803         if (pb->epoll_fd < 0) {
11804                 err = -errno;
11805                 pr_warn("failed to create epoll instance: %s\n",
11806                         libbpf_strerror_r(err, msg, sizeof(msg)));
11807                 goto error;
11808         }
11809
11810         if (p->cpu_cnt > 0) {
11811                 pb->cpu_cnt = p->cpu_cnt;
11812         } else {
11813                 pb->cpu_cnt = libbpf_num_possible_cpus();
11814                 if (pb->cpu_cnt < 0) {
11815                         err = pb->cpu_cnt;
11816                         goto error;
11817                 }
11818                 if (map.max_entries && map.max_entries < pb->cpu_cnt)
11819                         pb->cpu_cnt = map.max_entries;
11820         }
11821
11822         pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));
11823         if (!pb->events) {
11824                 err = -ENOMEM;
11825                 pr_warn("failed to allocate events: out of memory\n");
11826                 goto error;
11827         }
11828         pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));
11829         if (!pb->cpu_bufs) {
11830                 err = -ENOMEM;
11831                 pr_warn("failed to allocate buffers: out of memory\n");
11832                 goto error;
11833         }
11834
11835         err = parse_cpu_mask_file(online_cpus_file, &online, &n);
11836         if (err) {
11837                 pr_warn("failed to get online CPU mask: %d\n", err);
11838                 goto error;
11839         }
11840
11841         for (i = 0, j = 0; i < pb->cpu_cnt; i++) {
11842                 struct perf_cpu_buf *cpu_buf;
11843                 int cpu, map_key;
11844
11845                 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i;
11846                 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i;
11847
11848                 /* in case user didn't explicitly requested particular CPUs to
11849                  * be attached to, skip offline/not present CPUs
11850                  */
11851                 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu]))
11852                         continue;
11853
11854                 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);
11855                 if (IS_ERR(cpu_buf)) {
11856                         err = PTR_ERR(cpu_buf);
11857                         goto error;
11858                 }
11859
11860                 pb->cpu_bufs[j] = cpu_buf;
11861
11862                 err = bpf_map_update_elem(pb->map_fd, &map_key,
11863                                           &cpu_buf->fd, 0);
11864                 if (err) {
11865                         err = -errno;
11866                         pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",
11867                                 cpu, map_key, cpu_buf->fd,
11868                                 libbpf_strerror_r(err, msg, sizeof(msg)));
11869                         goto error;
11870                 }
11871
11872                 pb->events[j].events = EPOLLIN;
11873                 pb->events[j].data.ptr = cpu_buf;
11874                 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,
11875                               &pb->events[j]) < 0) {
11876                         err = -errno;
11877                         pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n",
11878                                 cpu, cpu_buf->fd,
11879                                 libbpf_strerror_r(err, msg, sizeof(msg)));
11880                         goto error;
11881                 }
11882                 j++;
11883         }
11884         pb->cpu_cnt = j;
11885         free(online);
11886
11887         return pb;
11888
11889 error:
11890         free(online);
11891         if (pb)
11892                 perf_buffer__free(pb);
11893         return ERR_PTR(err);
11894 }
11895
11896 struct perf_sample_raw {
11897         struct perf_event_header header;
11898         uint32_t size;
11899         char data[];
11900 };
11901
11902 struct perf_sample_lost {
11903         struct perf_event_header header;
11904         uint64_t id;
11905         uint64_t lost;
11906         uint64_t sample_id;
11907 };
11908
11909 static enum bpf_perf_event_ret
11910 perf_buffer__process_record(struct perf_event_header *e, void *ctx)
11911 {
11912         struct perf_cpu_buf *cpu_buf = ctx;
11913         struct perf_buffer *pb = cpu_buf->pb;
11914         void *data = e;
11915
11916         /* user wants full control over parsing perf event */
11917         if (pb->event_cb)
11918                 return pb->event_cb(pb->ctx, cpu_buf->cpu, e);
11919
11920         switch (e->type) {
11921         case PERF_RECORD_SAMPLE: {
11922                 struct perf_sample_raw *s = data;
11923
11924                 if (pb->sample_cb)
11925                         pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);
11926                 break;
11927         }
11928         case PERF_RECORD_LOST: {
11929                 struct perf_sample_lost *s = data;
11930
11931                 if (pb->lost_cb)
11932                         pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);
11933                 break;
11934         }
11935         default:
11936                 pr_warn("unknown perf sample type %d\n", e->type);
11937                 return LIBBPF_PERF_EVENT_ERROR;
11938         }
11939         return LIBBPF_PERF_EVENT_CONT;
11940 }
11941
11942 static int perf_buffer__process_records(struct perf_buffer *pb,
11943                                         struct perf_cpu_buf *cpu_buf)
11944 {
11945         enum bpf_perf_event_ret ret;
11946
11947         ret = perf_event_read_simple(cpu_buf->base, pb->mmap_size,
11948                                      pb->page_size, &cpu_buf->buf,
11949                                      &cpu_buf->buf_size,
11950                                      perf_buffer__process_record, cpu_buf);
11951         if (ret != LIBBPF_PERF_EVENT_CONT)
11952                 return ret;
11953         return 0;
11954 }
11955
11956 int perf_buffer__epoll_fd(const struct perf_buffer *pb)
11957 {
11958         return pb->epoll_fd;
11959 }
11960
11961 int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms)
11962 {
11963         int i, cnt, err;
11964
11965         cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);
11966         if (cnt < 0)
11967                 return -errno;
11968
11969         for (i = 0; i < cnt; i++) {
11970                 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;
11971
11972                 err = perf_buffer__process_records(pb, cpu_buf);
11973                 if (err) {
11974                         pr_warn("error while processing records: %d\n", err);
11975                         return libbpf_err(err);
11976                 }
11977         }
11978         return cnt;
11979 }
11980
11981 /* Return number of PERF_EVENT_ARRAY map slots set up by this perf_buffer
11982  * manager.
11983  */
11984 size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb)
11985 {
11986         return pb->cpu_cnt;
11987 }
11988
11989 /*
11990  * Return perf_event FD of a ring buffer in *buf_idx* slot of
11991  * PERF_EVENT_ARRAY BPF map. This FD can be polled for new data using
11992  * select()/poll()/epoll() Linux syscalls.
11993  */
11994 int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx)
11995 {
11996         struct perf_cpu_buf *cpu_buf;
11997
11998         if (buf_idx >= pb->cpu_cnt)
11999                 return libbpf_err(-EINVAL);
12000
12001         cpu_buf = pb->cpu_bufs[buf_idx];
12002         if (!cpu_buf)
12003                 return libbpf_err(-ENOENT);
12004
12005         return cpu_buf->fd;
12006 }
12007
12008 int perf_buffer__buffer(struct perf_buffer *pb, int buf_idx, void **buf, size_t *buf_size)
12009 {
12010         struct perf_cpu_buf *cpu_buf;
12011
12012         if (buf_idx >= pb->cpu_cnt)
12013                 return libbpf_err(-EINVAL);
12014
12015         cpu_buf = pb->cpu_bufs[buf_idx];
12016         if (!cpu_buf)
12017                 return libbpf_err(-ENOENT);
12018
12019         *buf = cpu_buf->base;
12020         *buf_size = pb->mmap_size;
12021         return 0;
12022 }
12023
12024 /*
12025  * Consume data from perf ring buffer corresponding to slot *buf_idx* in
12026  * PERF_EVENT_ARRAY BPF map without waiting/polling. If there is no data to
12027  * consume, do nothing and return success.
12028  * Returns:
12029  *   - 0 on success;
12030  *   - <0 on failure.
12031  */
12032 int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx)
12033 {
12034         struct perf_cpu_buf *cpu_buf;
12035
12036         if (buf_idx >= pb->cpu_cnt)
12037                 return libbpf_err(-EINVAL);
12038
12039         cpu_buf = pb->cpu_bufs[buf_idx];
12040         if (!cpu_buf)
12041                 return libbpf_err(-ENOENT);
12042
12043         return perf_buffer__process_records(pb, cpu_buf);
12044 }
12045
12046 int perf_buffer__consume(struct perf_buffer *pb)
12047 {
12048         int i, err;
12049
12050         for (i = 0; i < pb->cpu_cnt; i++) {
12051                 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
12052
12053                 if (!cpu_buf)
12054                         continue;
12055
12056                 err = perf_buffer__process_records(pb, cpu_buf);
12057                 if (err) {
12058                         pr_warn("perf_buffer: failed to process records in buffer #%d: %d\n", i, err);
12059                         return libbpf_err(err);
12060                 }
12061         }
12062         return 0;
12063 }
12064
12065 int bpf_program__set_attach_target(struct bpf_program *prog,
12066                                    int attach_prog_fd,
12067                                    const char *attach_func_name)
12068 {
12069         int btf_obj_fd = 0, btf_id = 0, err;
12070
12071         if (!prog || attach_prog_fd < 0)
12072                 return libbpf_err(-EINVAL);
12073
12074         if (prog->obj->loaded)
12075                 return libbpf_err(-EINVAL);
12076
12077         if (attach_prog_fd && !attach_func_name) {
12078                 /* remember attach_prog_fd and let bpf_program__load() find
12079                  * BTF ID during the program load
12080                  */
12081                 prog->attach_prog_fd = attach_prog_fd;
12082                 return 0;
12083         }
12084
12085         if (attach_prog_fd) {
12086                 btf_id = libbpf_find_prog_btf_id(attach_func_name,
12087                                                  attach_prog_fd);
12088                 if (btf_id < 0)
12089                         return libbpf_err(btf_id);
12090         } else {
12091                 if (!attach_func_name)
12092                         return libbpf_err(-EINVAL);
12093
12094                 /* load btf_vmlinux, if not yet */
12095                 err = bpf_object__load_vmlinux_btf(prog->obj, true);
12096                 if (err)
12097                         return libbpf_err(err);
12098                 err = find_kernel_btf_id(prog->obj, attach_func_name,
12099                                          prog->expected_attach_type,
12100                                          &btf_obj_fd, &btf_id);
12101                 if (err)
12102                         return libbpf_err(err);
12103         }
12104
12105         prog->attach_btf_id = btf_id;
12106         prog->attach_btf_obj_fd = btf_obj_fd;
12107         prog->attach_prog_fd = attach_prog_fd;
12108         return 0;
12109 }
12110
12111 int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz)
12112 {
12113         int err = 0, n, len, start, end = -1;
12114         bool *tmp;
12115
12116         *mask = NULL;
12117         *mask_sz = 0;
12118
12119         /* Each sub string separated by ',' has format \d+-\d+ or \d+ */
12120         while (*s) {
12121                 if (*s == ',' || *s == '\n') {
12122                         s++;
12123                         continue;
12124                 }
12125                 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len);
12126                 if (n <= 0 || n > 2) {
12127                         pr_warn("Failed to get CPU range %s: %d\n", s, n);
12128                         err = -EINVAL;
12129                         goto cleanup;
12130                 } else if (n == 1) {
12131                         end = start;
12132                 }
12133                 if (start < 0 || start > end) {
12134                         pr_warn("Invalid CPU range [%d,%d] in %s\n",
12135                                 start, end, s);
12136                         err = -EINVAL;
12137                         goto cleanup;
12138                 }
12139                 tmp = realloc(*mask, end + 1);
12140                 if (!tmp) {
12141                         err = -ENOMEM;
12142                         goto cleanup;
12143                 }
12144                 *mask = tmp;
12145                 memset(tmp + *mask_sz, 0, start - *mask_sz);
12146                 memset(tmp + start, 1, end - start + 1);
12147                 *mask_sz = end + 1;
12148                 s += len;
12149         }
12150         if (!*mask_sz) {
12151                 pr_warn("Empty CPU range\n");
12152                 return -EINVAL;
12153         }
12154         return 0;
12155 cleanup:
12156         free(*mask);
12157         *mask = NULL;
12158         return err;
12159 }
12160
12161 int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz)
12162 {
12163         int fd, err = 0, len;
12164         char buf[128];
12165
12166         fd = open(fcpu, O_RDONLY | O_CLOEXEC);
12167         if (fd < 0) {
12168                 err = -errno;
12169                 pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err);
12170                 return err;
12171         }
12172         len = read(fd, buf, sizeof(buf));
12173         close(fd);
12174         if (len <= 0) {
12175                 err = len ? -errno : -EINVAL;
12176                 pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err);
12177                 return err;
12178         }
12179         if (len >= sizeof(buf)) {
12180                 pr_warn("CPU mask is too big in file %s\n", fcpu);
12181                 return -E2BIG;
12182         }
12183         buf[len] = '\0';
12184
12185         return parse_cpu_mask_str(buf, mask, mask_sz);
12186 }
12187
12188 int libbpf_num_possible_cpus(void)
12189 {
12190         static const char *fcpu = "/sys/devices/system/cpu/possible";
12191         static int cpus;
12192         int err, n, i, tmp_cpus;
12193         bool *mask;
12194
12195         tmp_cpus = READ_ONCE(cpus);
12196         if (tmp_cpus > 0)
12197                 return tmp_cpus;
12198
12199         err = parse_cpu_mask_file(fcpu, &mask, &n);
12200         if (err)
12201                 return libbpf_err(err);
12202
12203         tmp_cpus = 0;
12204         for (i = 0; i < n; i++) {
12205                 if (mask[i])
12206                         tmp_cpus++;
12207         }
12208         free(mask);
12209
12210         WRITE_ONCE(cpus, tmp_cpus);
12211         return tmp_cpus;
12212 }
12213
12214 static int populate_skeleton_maps(const struct bpf_object *obj,
12215                                   struct bpf_map_skeleton *maps,
12216                                   size_t map_cnt)
12217 {
12218         int i;
12219
12220         for (i = 0; i < map_cnt; i++) {
12221                 struct bpf_map **map = maps[i].map;
12222                 const char *name = maps[i].name;
12223                 void **mmaped = maps[i].mmaped;
12224
12225                 *map = bpf_object__find_map_by_name(obj, name);
12226                 if (!*map) {
12227                         pr_warn("failed to find skeleton map '%s'\n", name);
12228                         return -ESRCH;
12229                 }
12230
12231                 /* externs shouldn't be pre-setup from user code */
12232                 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG)
12233                         *mmaped = (*map)->mmaped;
12234         }
12235         return 0;
12236 }
12237
12238 static int populate_skeleton_progs(const struct bpf_object *obj,
12239                                    struct bpf_prog_skeleton *progs,
12240                                    size_t prog_cnt)
12241 {
12242         int i;
12243
12244         for (i = 0; i < prog_cnt; i++) {
12245                 struct bpf_program **prog = progs[i].prog;
12246                 const char *name = progs[i].name;
12247
12248                 *prog = bpf_object__find_program_by_name(obj, name);
12249                 if (!*prog) {
12250                         pr_warn("failed to find skeleton program '%s'\n", name);
12251                         return -ESRCH;
12252                 }
12253         }
12254         return 0;
12255 }
12256
12257 int bpf_object__open_skeleton(struct bpf_object_skeleton *s,
12258                               const struct bpf_object_open_opts *opts)
12259 {
12260         DECLARE_LIBBPF_OPTS(bpf_object_open_opts, skel_opts,
12261                 .object_name = s->name,
12262         );
12263         struct bpf_object *obj;
12264         int err;
12265
12266         /* Attempt to preserve opts->object_name, unless overriden by user
12267          * explicitly. Overwriting object name for skeletons is discouraged,
12268          * as it breaks global data maps, because they contain object name
12269          * prefix as their own map name prefix. When skeleton is generated,
12270          * bpftool is making an assumption that this name will stay the same.
12271          */
12272         if (opts) {
12273                 memcpy(&skel_opts, opts, sizeof(*opts));
12274                 if (!opts->object_name)
12275                         skel_opts.object_name = s->name;
12276         }
12277
12278         obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts);
12279         err = libbpf_get_error(obj);
12280         if (err) {
12281                 pr_warn("failed to initialize skeleton BPF object '%s': %d\n",
12282                         s->name, err);
12283                 return libbpf_err(err);
12284         }
12285
12286         *s->obj = obj;
12287         err = populate_skeleton_maps(obj, s->maps, s->map_cnt);
12288         if (err) {
12289                 pr_warn("failed to populate skeleton maps for '%s': %d\n", s->name, err);
12290                 return libbpf_err(err);
12291         }
12292
12293         err = populate_skeleton_progs(obj, s->progs, s->prog_cnt);
12294         if (err) {
12295                 pr_warn("failed to populate skeleton progs for '%s': %d\n", s->name, err);
12296                 return libbpf_err(err);
12297         }
12298
12299         return 0;
12300 }
12301
12302 int bpf_object__open_subskeleton(struct bpf_object_subskeleton *s)
12303 {
12304         int err, len, var_idx, i;
12305         const char *var_name;
12306         const struct bpf_map *map;
12307         struct btf *btf;
12308         __u32 map_type_id;
12309         const struct btf_type *map_type, *var_type;
12310         const struct bpf_var_skeleton *var_skel;
12311         struct btf_var_secinfo *var;
12312
12313         if (!s->obj)
12314                 return libbpf_err(-EINVAL);
12315
12316         btf = bpf_object__btf(s->obj);
12317         if (!btf) {
12318                 pr_warn("subskeletons require BTF at runtime (object %s)\n",
12319                         bpf_object__name(s->obj));
12320                 return libbpf_err(-errno);
12321         }
12322
12323         err = populate_skeleton_maps(s->obj, s->maps, s->map_cnt);
12324         if (err) {
12325                 pr_warn("failed to populate subskeleton maps: %d\n", err);
12326                 return libbpf_err(err);
12327         }
12328
12329         err = populate_skeleton_progs(s->obj, s->progs, s->prog_cnt);
12330         if (err) {
12331                 pr_warn("failed to populate subskeleton maps: %d\n", err);
12332                 return libbpf_err(err);
12333         }
12334
12335         for (var_idx = 0; var_idx < s->var_cnt; var_idx++) {
12336                 var_skel = &s->vars[var_idx];
12337                 map = *var_skel->map;
12338                 map_type_id = bpf_map__btf_value_type_id(map);
12339                 map_type = btf__type_by_id(btf, map_type_id);
12340
12341                 if (!btf_is_datasec(map_type)) {
12342                         pr_warn("type for map '%1$s' is not a datasec: %2$s",
12343                                 bpf_map__name(map),
12344                                 __btf_kind_str(btf_kind(map_type)));
12345                         return libbpf_err(-EINVAL);
12346                 }
12347
12348                 len = btf_vlen(map_type);
12349                 var = btf_var_secinfos(map_type);
12350                 for (i = 0; i < len; i++, var++) {
12351                         var_type = btf__type_by_id(btf, var->type);
12352                         var_name = btf__name_by_offset(btf, var_type->name_off);
12353                         if (strcmp(var_name, var_skel->name) == 0) {
12354                                 *var_skel->addr = map->mmaped + var->offset;
12355                                 break;
12356                         }
12357                 }
12358         }
12359         return 0;
12360 }
12361
12362 void bpf_object__destroy_subskeleton(struct bpf_object_subskeleton *s)
12363 {
12364         if (!s)
12365                 return;
12366         free(s->maps);
12367         free(s->progs);
12368         free(s->vars);
12369         free(s);
12370 }
12371
12372 int bpf_object__load_skeleton(struct bpf_object_skeleton *s)
12373 {
12374         int i, err;
12375
12376         err = bpf_object__load(*s->obj);
12377         if (err) {
12378                 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err);
12379                 return libbpf_err(err);
12380         }
12381
12382         for (i = 0; i < s->map_cnt; i++) {
12383                 struct bpf_map *map = *s->maps[i].map;
12384                 size_t mmap_sz = bpf_map_mmap_sz(map);
12385                 int prot, map_fd = bpf_map__fd(map);
12386                 void **mmaped = s->maps[i].mmaped;
12387
12388                 if (!mmaped)
12389                         continue;
12390
12391                 if (!(map->def.map_flags & BPF_F_MMAPABLE)) {
12392                         *mmaped = NULL;
12393                         continue;
12394                 }
12395
12396                 if (map->def.map_flags & BPF_F_RDONLY_PROG)
12397                         prot = PROT_READ;
12398                 else
12399                         prot = PROT_READ | PROT_WRITE;
12400
12401                 /* Remap anonymous mmap()-ed "map initialization image" as
12402                  * a BPF map-backed mmap()-ed memory, but preserving the same
12403                  * memory address. This will cause kernel to change process'
12404                  * page table to point to a different piece of kernel memory,
12405                  * but from userspace point of view memory address (and its
12406                  * contents, being identical at this point) will stay the
12407                  * same. This mapping will be released by bpf_object__close()
12408                  * as per normal clean up procedure, so we don't need to worry
12409                  * about it from skeleton's clean up perspective.
12410                  */
12411                 *mmaped = mmap(map->mmaped, mmap_sz, prot,
12412                                 MAP_SHARED | MAP_FIXED, map_fd, 0);
12413                 if (*mmaped == MAP_FAILED) {
12414                         err = -errno;
12415                         *mmaped = NULL;
12416                         pr_warn("failed to re-mmap() map '%s': %d\n",
12417                                  bpf_map__name(map), err);
12418                         return libbpf_err(err);
12419                 }
12420         }
12421
12422         return 0;
12423 }
12424
12425 int bpf_object__attach_skeleton(struct bpf_object_skeleton *s)
12426 {
12427         int i, err;
12428
12429         for (i = 0; i < s->prog_cnt; i++) {
12430                 struct bpf_program *prog = *s->progs[i].prog;
12431                 struct bpf_link **link = s->progs[i].link;
12432
12433                 if (!prog->autoload || !prog->autoattach)
12434                         continue;
12435
12436                 /* auto-attaching not supported for this program */
12437                 if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
12438                         continue;
12439
12440                 /* if user already set the link manually, don't attempt auto-attach */
12441                 if (*link)
12442                         continue;
12443
12444                 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link);
12445                 if (err) {
12446                         pr_warn("prog '%s': failed to auto-attach: %d\n",
12447                                 bpf_program__name(prog), err);
12448                         return libbpf_err(err);
12449                 }
12450
12451                 /* It's possible that for some SEC() definitions auto-attach
12452                  * is supported in some cases (e.g., if definition completely
12453                  * specifies target information), but is not in other cases.
12454                  * SEC("uprobe") is one such case. If user specified target
12455                  * binary and function name, such BPF program can be
12456                  * auto-attached. But if not, it shouldn't trigger skeleton's
12457                  * attach to fail. It should just be skipped.
12458                  * attach_fn signals such case with returning 0 (no error) and
12459                  * setting link to NULL.
12460                  */
12461         }
12462
12463         return 0;
12464 }
12465
12466 void bpf_object__detach_skeleton(struct bpf_object_skeleton *s)
12467 {
12468         int i;
12469
12470         for (i = 0; i < s->prog_cnt; i++) {
12471                 struct bpf_link **link = s->progs[i].link;
12472
12473                 bpf_link__destroy(*link);
12474                 *link = NULL;
12475         }
12476 }
12477
12478 void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s)
12479 {
12480         if (!s)
12481                 return;
12482
12483         if (s->progs)
12484                 bpf_object__detach_skeleton(s);
12485         if (s->obj)
12486                 bpf_object__close(*s->obj);
12487         free(s->maps);
12488         free(s->progs);
12489         free(s);
12490 }