bpf: Introduce bpf_func_info

This patch added interface to load a program with the following
additional information:
   . prog_btf_fd
   . func_info, func_info_rec_size and func_info_cnt
where func_info will provide function range and type_id
corresponding to each function.

The func_info_rec_size is introduced in the UAPI to specify
struct bpf_func_info size passed from user space. This
intends to make bpf_func_info structure growable in the future.
If the kernel gets a different bpf_func_info size from userspace,
it will try to handle user request with part of bpf_func_info
it can understand. In this patch, kernel can understand
  struct bpf_func_info {
       __u32   insn_offset;
       __u32   type_id;
  };
If user passed a bpf func_info record size of 16 bytes, the
kernel can still handle part of records with the above definition.

If verifier agrees with function range provided by the user,
the bpf_prog ksym for each function will use the func name
provided in the type_id, which is supposed to provide better
encoding as it is not limited by 16 bytes program name
limitation and this is better for bpf program which contains
multiple subprograms.

The bpf_prog_info interface is also extended to
return btf_id, func_info, func_info_rec_size and func_info_cnt
to userspace, so userspace can print out the function prototype
for each xlated function. The insn_offset in the returned
func_info corresponds to the insn offset for xlated functions.
With other jit related fields in bpf_prog_info, userspace can also
print out function prototypes for each jited function.

Signed-off-by: Yonghong Song <[email protected]>
Signed-off-by: Martin KaFai Lau <[email protected]>
Signed-off-by: Alexei Starovoitov <[email protected]>

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 987815152629a4932a927d2305495be71374472d..7f0e225bf630b6c36edecc0c366a869ca5e9cf99 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -316,6 +316,8 @@ struct bpf_prog_aux {
        void *security;
 #endif
        struct bpf_prog_offload *offload;
        void *security;
 #endif
        struct bpf_prog_offload *offload;

+       struct btf *btf;
+       u32 type_id; /* type id for this prog/func */

        union {
                struct work_struct work;
                struct rcu_head rcu;
        union {
                struct work_struct work;
                struct rcu_head rcu;


@@ -527,7 +529,8 @@ static inline void bpf_long_memcpy(void *dst, const void *src, u32 size)
 }
 
 /* verify correctness of eBPF program */
 }
 
 /* verify correctness of eBPF program */

-int bpf_check(struct bpf_prog **fp, union bpf_attr *attr);
+int bpf_check(struct bpf_prog **fp, union bpf_attr *attr,
+             union bpf_attr __user *uattr);

 void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
 
 /* Map specifics */
 void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
 
 /* Map specifics */

diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 11f5df1092d9baddeb8dc902fcee4e2f528a9a07..204382f46fd888113fd9043b92bef84e5da48cf5 100644 (file)
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -204,6 +204,7 @@ static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log)
 struct bpf_subprog_info {
        u32 start; /* insn idx of function entry point */
        u16 stack_depth; /* max. stack depth used by this function */
 struct bpf_subprog_info {
        u32 start; /* insn idx of function entry point */
        u16 stack_depth; /* max. stack depth used by this function */

+       u32 type_id; /* btf type_id for this subprog */

 };
 
 /* single container for all structs
 };
 
 /* single container for all structs

diff --git a/include/linux/btf.h b/include/linux/btf.h
index e076c4697049da1e62f39160ed2f1fcdf73c743c..7f2c0a4a45ea6cd42e18cce86abd60c407798d1c 100644 (file)
--- a/include/linux/btf.h
+++ b/include/linux/btf.h
@@ -46,5 +46,7 @@ void btf_type_seq_show(const struct btf *btf, u32 type_id, void *obj,
                       struct seq_file *m);
 int btf_get_fd_by_id(u32 id);
 u32 btf_id(const struct btf *btf);
                       struct seq_file *m);
 int btf_get_fd_by_id(u32 id);
 u32 btf_id(const struct btf *btf);

+const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id);
+const char *btf_name_by_offset(const struct btf *btf, u32 offset);

 
 #endif
 
 #endif

diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 05d95290b84867e6656ce10842063cfff9c12cb6..c1554aa0746594b73f8cbed93e3204f1b0c354de 100644 (file)
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -338,6 +338,10 @@ union bpf_attr {
                 * (context accesses, allowed helpers, etc).
                 */
                __u32           expected_attach_type;
                 * (context accesses, allowed helpers, etc).
                 */
                __u32           expected_attach_type;

+               __u32           prog_btf_fd;    /* fd pointing to BTF type data */
+               __u32           func_info_rec_size;     /* userspace bpf_func_info size */
+               __aligned_u64   func_info;      /* func info */
+               __u32           func_info_cnt;  /* number of bpf_func_info records */

        };
 
        struct { /* anonymous struct used by BPF_OBJ_* commands */
        };
 
        struct { /* anonymous struct used by BPF_OBJ_* commands */


@@ -2638,6 +2642,10 @@ struct bpf_prog_info {
        __u32 nr_jited_func_lens;
        __aligned_u64 jited_ksyms;
        __aligned_u64 jited_func_lens;
        __u32 nr_jited_func_lens;
        __aligned_u64 jited_ksyms;
        __aligned_u64 jited_func_lens;

+       __u32 btf_id;
+       __u32 func_info_rec_size;
+       __aligned_u64 func_info;
+       __u32 func_info_cnt;

 } __attribute__((aligned(8)));
 
 struct bpf_map_info {
 } __attribute__((aligned(8)));
 
 struct bpf_map_info {


@@ -2949,4 +2957,9 @@ struct bpf_flow_keys {
        };
 };
 
        };
 };
 

+struct bpf_func_info {
+       __u32   insn_offset;
+       __u32   type_id;
+};
+

 #endif /* _UAPI__LINUX_BPF_H__ */
 #endif /* _UAPI__LINUX_BPF_H__ */

diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index 6a2be79b73fc789e2fc333be8d7f1d12c90978d8..69da9169819adb3ace31c3a9a042ff6369e1d073 100644 (file)
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -474,7 +474,7 @@ static bool btf_name_valid_identifier(const struct btf *btf, u32 offset)
        return !*src;
 }
 
        return !*src;
 }
 

-static const char *btf_name_by_offset(const struct btf *btf, u32 offset)
+const char *btf_name_by_offset(const struct btf *btf, u32 offset)

 {
        if (!offset)
                return "(anon)";
 {
        if (!offset)
                return "(anon)";


@@ -484,7 +484,7 @@ static const char *btf_name_by_offset(const struct btf *btf, u32 offset)
                return "(invalid-name-offset)";
 }
 
                return "(invalid-name-offset)";
 }
 

-static const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id)
+const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id)

 {
        if (type_id > btf->nr_types)
                return NULL;
 {
        if (type_id > btf->nr_types)
                return NULL;

diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 1a796e0799ec4a524aee5c325734f0ab22a410ac..16d77012ad3efc9b996782636be6b006576b5300 100644 (file)
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -21,12 +21,14 @@
  * Kris Katterjohn - Added many additional checks in bpf_check_classic()
  */
 
  * Kris Katterjohn - Added many additional checks in bpf_check_classic()
  */
 

+#include <uapi/linux/btf.h>

 #include <linux/filter.h>
 #include <linux/skbuff.h>
 #include <linux/vmalloc.h>
 #include <linux/random.h>
 #include <linux/moduleloader.h>
 #include <linux/bpf.h>
 #include <linux/filter.h>
 #include <linux/skbuff.h>
 #include <linux/vmalloc.h>
 #include <linux/random.h>
 #include <linux/moduleloader.h>
 #include <linux/bpf.h>

+#include <linux/btf.h>

 #include <linux/frame.h>
 #include <linux/rbtree_latch.h>
 #include <linux/kallsyms.h>
 #include <linux/frame.h>
 #include <linux/rbtree_latch.h>
 #include <linux/kallsyms.h>


@@ -390,6 +392,8 @@ bpf_get_prog_addr_region(const struct bpf_prog *prog,
 static void bpf_get_prog_name(const struct bpf_prog *prog, char *sym)
 {
        const char *end = sym + KSYM_NAME_LEN;
 static void bpf_get_prog_name(const struct bpf_prog *prog, char *sym)
 {
        const char *end = sym + KSYM_NAME_LEN;

+       const struct btf_type *type;
+       const char *func_name;

 
        BUILD_BUG_ON(sizeof("bpf_prog_") +
                     sizeof(prog->tag) * 2 +
 
        BUILD_BUG_ON(sizeof("bpf_prog_") +
                     sizeof(prog->tag) * 2 +


@@ -404,6 +408,15 @@ static void bpf_get_prog_name(const struct bpf_prog *prog, char *sym)
 
        sym += snprintf(sym, KSYM_NAME_LEN, "bpf_prog_");
        sym  = bin2hex(sym, prog->tag, sizeof(prog->tag));
 
        sym += snprintf(sym, KSYM_NAME_LEN, "bpf_prog_");
        sym  = bin2hex(sym, prog->tag, sizeof(prog->tag));

+
+       /* prog->aux->name will be ignored if full btf name is available */
+       if (prog->aux->btf) {
+               type = btf_type_by_id(prog->aux->btf, prog->aux->type_id);
+               func_name = btf_name_by_offset(prog->aux->btf, type->name_off);
+               snprintf(sym, (size_t)(end - sym), "_%s", func_name);
+               return;
+       }
+

        if (prog->aux->name[0])
                snprintf(sym, (size_t)(end - sym), "_%s", prog->aux->name);
        else
        if (prog->aux->name[0])
                snprintf(sym, (size_t)(end - sym), "_%s", prog->aux->name);
        else

diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index cf5040fd54344dd798f73464eadb5b9684300f1c..9983778081021f21c918c823edeee8d7b25f62f3 100644 (file)
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -1213,6 +1213,7 @@ static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock)
                /* bpf_prog_free_id() must be called first */
                bpf_prog_free_id(prog, do_idr_lock);
                bpf_prog_kallsyms_del_all(prog);
                /* bpf_prog_free_id() must be called first */
                bpf_prog_free_id(prog, do_idr_lock);
                bpf_prog_kallsyms_del_all(prog);

+               btf_put(prog->aux->btf);

 
                call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
        }
 
                call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
        }


@@ -1437,9 +1438,9 @@ bpf_prog_load_check_attach_type(enum bpf_prog_type prog_type,
 }
 
 /* last field in 'union bpf_attr' used by this command */
 }
 
 /* last field in 'union bpf_attr' used by this command */

-#define        BPF_PROG_LOAD_LAST_FIELD expected_attach_type
+#define        BPF_PROG_LOAD_LAST_FIELD func_info_cnt

 
 

-static int bpf_prog_load(union bpf_attr *attr)
+static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr)

 {
        enum bpf_prog_type type = attr->prog_type;
        struct bpf_prog *prog;
 {
        enum bpf_prog_type type = attr->prog_type;
        struct bpf_prog *prog;


@@ -1525,7 +1526,7 @@ static int bpf_prog_load(union bpf_attr *attr)
                goto free_prog;
 
        /* run eBPF verifier */
                goto free_prog;
 
        /* run eBPF verifier */

-       err = bpf_check(&prog, attr);
+       err = bpf_check(&prog, attr, uattr);

        if (err < 0)
                goto free_used_maps;
 
        if (err < 0)
                goto free_used_maps;
 


@@ -2079,6 +2080,7 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
                info.xlated_prog_len = 0;
                info.nr_jited_ksyms = 0;
                info.nr_jited_func_lens = 0;
                info.xlated_prog_len = 0;
                info.nr_jited_ksyms = 0;
                info.nr_jited_func_lens = 0;

+               info.func_info_cnt = 0;

                goto done;
        }
 
                goto done;
        }
 


@@ -2216,6 +2218,55 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
                }
        }
 
                }
        }
 

+       if (prog->aux->btf) {
+               u32 ucnt, urec_size;
+
+               info.btf_id = btf_id(prog->aux->btf);
+
+               ucnt = info.func_info_cnt;
+               info.func_info_cnt = prog->aux->func_cnt ? : 1;
+               urec_size = info.func_info_rec_size;
+               info.func_info_rec_size = sizeof(struct bpf_func_info);
+               if (ucnt) {
+                       /* expect passed-in urec_size is what the kernel expects */
+                       if (urec_size != info.func_info_rec_size)
+                               return -EINVAL;
+
+                       if (bpf_dump_raw_ok()) {
+                               struct bpf_func_info kern_finfo;
+                               char __user *user_finfo;
+                               u32 i, insn_offset;
+
+                               user_finfo = u64_to_user_ptr(info.func_info);
+                               if (prog->aux->func_cnt) {
+                                       ucnt = min_t(u32, info.func_info_cnt, ucnt);
+                                       insn_offset = 0;
+                                       for (i = 0; i < ucnt; i++) {
+                                               kern_finfo.insn_offset = insn_offset;
+                                               kern_finfo.type_id = prog->aux->func[i]->aux->type_id;
+                                               if (copy_to_user(user_finfo, &kern_finfo,
+                                                                sizeof(kern_finfo)))
+                                                       return -EFAULT;
+
+                                               /* func[i]->len holds the prog len */
+                                               insn_offset += prog->aux->func[i]->len;
+                                               user_finfo += urec_size;
+                                       }
+                               } else {
+                                       kern_finfo.insn_offset = 0;
+                                       kern_finfo.type_id = prog->aux->type_id;
+                                       if (copy_to_user(user_finfo, &kern_finfo,
+                                                        sizeof(kern_finfo)))
+                                               return -EFAULT;
+                               }
+                       } else {
+                               info.func_info_cnt = 0;
+                       }
+               }
+       } else {
+               info.func_info_cnt = 0;
+       }
+

 done:
        if (copy_to_user(uinfo, &info, info_len) ||
            put_user(info_len, &uattr->info.info_len))
 done:
        if (copy_to_user(uinfo, &info, info_len) ||
            put_user(info_len, &uattr->info.info_len))


@@ -2501,7 +2552,7 @@ SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, siz
                err = map_get_next_key(&attr);
                break;
        case BPF_PROG_LOAD:
                err = map_get_next_key(&attr);
                break;
        case BPF_PROG_LOAD:

-               err = bpf_prog_load(&attr);
+               err = bpf_prog_load(&attr, uattr);

                break;
        case BPF_OBJ_PIN:
                err = bpf_obj_pin(&attr);
                break;
        case BPF_OBJ_PIN:
                err = bpf_obj_pin(&attr);

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index b5222aa61d54312b38caa1ee3294f8b7f6397dad..f102c4fd0c5afca78e55687bfd9409b76a813f85 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -11,10 +11,12 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  * General Public License for more details.
  */
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  * General Public License for more details.
  */

+#include <uapi/linux/btf.h>

 #include <linux/kernel.h>
 #include <linux/types.h>
 #include <linux/slab.h>
 #include <linux/bpf.h>
 #include <linux/kernel.h>
 #include <linux/types.h>
 #include <linux/slab.h>
 #include <linux/bpf.h>

+#include <linux/btf.h>

 #include <linux/bpf_verifier.h>
 #include <linux/filter.h>
 #include <net/netlink.h>
 #include <linux/bpf_verifier.h>
 #include <linux/filter.h>
 #include <net/netlink.h>


@@ -4639,6 +4641,114 @@ err_free:
        return ret;
 }
 
        return ret;
 }
 

+/* The minimum supported BTF func info size */
+#define MIN_BPF_FUNCINFO_SIZE  8
+#define MAX_FUNCINFO_REC_SIZE  252
+
+static int check_btf_func(struct bpf_prog *prog, struct bpf_verifier_env *env,
+                         union bpf_attr *attr, union bpf_attr __user *uattr)
+{
+       u32 i, nfuncs, urec_size, min_size, prev_offset;
+       u32 krec_size = sizeof(struct bpf_func_info);
+       struct bpf_func_info krecord = {};
+       const struct btf_type *type;
+       void __user *urecord;
+       struct btf *btf;
+       int ret = 0;
+
+       nfuncs = attr->func_info_cnt;
+       if (!nfuncs)
+               return 0;
+
+       if (nfuncs != env->subprog_cnt) {
+               verbose(env, "number of funcs in func_info doesn't match number of subprogs\n");
+               return -EINVAL;
+       }
+
+       urec_size = attr->func_info_rec_size;
+       if (urec_size < MIN_BPF_FUNCINFO_SIZE ||
+           urec_size > MAX_FUNCINFO_REC_SIZE ||
+           urec_size % sizeof(u32)) {
+               verbose(env, "invalid func info rec size %u\n", urec_size);
+               return -EINVAL;
+       }
+
+       btf = btf_get_by_fd(attr->prog_btf_fd);
+       if (IS_ERR(btf)) {
+               verbose(env, "unable to get btf from fd\n");
+               return PTR_ERR(btf);
+       }
+
+       urecord = u64_to_user_ptr(attr->func_info);
+       min_size = min_t(u32, krec_size, urec_size);
+
+       for (i = 0; i < nfuncs; i++) {
+               ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size);
+               if (ret) {
+                       if (ret == -E2BIG) {
+                               verbose(env, "nonzero tailing record in func info");
+                               /* set the size kernel expects so loader can zero
+                                * out the rest of the record.
+                                */
+                               if (put_user(min_size, &uattr->func_info_rec_size))
+                                       ret = -EFAULT;
+                       }
+                       goto free_btf;
+               }
+
+               if (copy_from_user(&krecord, urecord, min_size)) {
+                       ret = -EFAULT;
+                       goto free_btf;
+               }
+
+               /* check insn_offset */
+               if (i == 0) {
+                       if (krecord.insn_offset) {
+                               verbose(env,
+                                       "nonzero insn_offset %u for the first func info record",
+                                       krecord.insn_offset);
+                               ret = -EINVAL;
+                               goto free_btf;
+                       }
+               } else if (krecord.insn_offset <= prev_offset) {
+                       verbose(env,
+                               "same or smaller insn offset (%u) than previous func info record (%u)",
+                               krecord.insn_offset, prev_offset);
+                       ret = -EINVAL;
+                       goto free_btf;
+               }
+
+               if (env->subprog_info[i].start != krecord.insn_offset) {
+                       verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n");
+                       ret = -EINVAL;
+                       goto free_btf;
+               }
+
+               /* check type_id */
+               type = btf_type_by_id(btf, krecord.type_id);
+               if (!type || BTF_INFO_KIND(type->info) != BTF_KIND_FUNC) {
+                       verbose(env, "invalid type id %d in func info",
+                               krecord.type_id);
+                       ret = -EINVAL;
+                       goto free_btf;
+               }
+
+               if (i == 0)
+                       prog->aux->type_id = krecord.type_id;
+               env->subprog_info[i].type_id = krecord.type_id;
+
+               prev_offset = krecord.insn_offset;
+               urecord += urec_size;
+       }
+
+       prog->aux->btf = btf;
+       return 0;
+
+free_btf:
+       btf_put(btf);
+       return ret;
+}
+

 /* check %cur's range satisfies %old's */
 static bool range_within(struct bpf_reg_state *old,
                         struct bpf_reg_state *cur)
 /* check %cur's range satisfies %old's */
 static bool range_within(struct bpf_reg_state *old,
                         struct bpf_reg_state *cur)


@@ -5939,6 +6049,9 @@ static int jit_subprogs(struct bpf_verifier_env *env)
                func[i]->aux->name[0] = 'F';
                func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
                func[i]->jit_requested = 1;
                func[i]->aux->name[0] = 'F';
                func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
                func[i]->jit_requested = 1;

+               /* the btf will be freed only at prog->aux */
+               func[i]->aux->btf = prog->aux->btf;
+               func[i]->aux->type_id = env->subprog_info[i].type_id;

                func[i] = bpf_int_jit_compile(func[i]);
                if (!func[i]->jited) {
                        err = -ENOTSUPP;
                func[i] = bpf_int_jit_compile(func[i]);
                if (!func[i]->jited) {
                        err = -ENOTSUPP;


@@ -6325,7 +6438,8 @@ static void free_states(struct bpf_verifier_env *env)
        kfree(env->explored_states);
 }
 
        kfree(env->explored_states);
 }
 

-int bpf_check(struct bpf_prog **prog, union bpf_attr *attr)
+int bpf_check(struct bpf_prog **prog, union bpf_attr *attr,
+             union bpf_attr __user *uattr)

 {
        struct bpf_verifier_env *env;
        struct bpf_verifier_log *log;
 {
        struct bpf_verifier_env *env;
        struct bpf_verifier_log *log;


@@ -6397,6 +6511,10 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr)
        if (ret < 0)
                goto skip_full_check;
 
        if (ret < 0)
                goto skip_full_check;
 

+       ret = check_btf_func(env->prog, env, attr, uattr);
+       if (ret < 0)
+               goto skip_full_check;
+

        ret = do_check(env);
        if (env->cur_state) {
                free_verifier_state(env->cur_state, true);
        ret = do_check(env);
        if (env->cur_state) {
                free_verifier_state(env->cur_state, true);