kernel/bpf/bpf_inode_storage.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2019 Facebook
   4  * Copyright 2020 Google LLC.
   5  */
   6
   7 #include <linux/rculist.h>
   8 #include <linux/list.h>
   9 #include <linux/hash.h>
  10 #include <linux/types.h>
  11 #include <linux/spinlock.h>
  12 #include <linux/bpf.h>
  13 #include <linux/bpf_local_storage.h>
  14 #include <net/sock.h>
  15 #include <uapi/linux/sock_diag.h>
  16 #include <uapi/linux/btf.h>
  17 #include <linux/bpf_lsm.h>
  18 #include <linux/btf_ids.h>
  19 #include <linux/rcupdate_trace.h>
  20
  21 DEFINE_BPF_STORAGE_CACHE(inode_cache);
  22
  23 static struct bpf_local_storage __rcu **
  24 inode_storage_ptr(void *owner)
  25 {
  26         struct inode *inode = owner;
  27         struct bpf_storage_blob *bsb;
  28
  29         bsb = bpf_inode(inode);
  30         if (!bsb)
  31                 return NULL;
  32         return &bsb->storage;
  33 }
  34
  35 static struct bpf_local_storage_data *inode_storage_lookup(struct inode *inode,
  36                                                            struct bpf_map *map,
  37                                                            bool cacheit_lockit)
  38 {
  39         struct bpf_local_storage *inode_storage;
  40         struct bpf_local_storage_map *smap;
  41         struct bpf_storage_blob *bsb;
  42
  43         bsb = bpf_inode(inode);
  44         if (!bsb)
  45                 return NULL;
  46
  47         inode_storage =
  48                 rcu_dereference_check(bsb->storage, bpf_rcu_lock_held());
  49         if (!inode_storage)
  50                 return NULL;
  51
  52         smap = (struct bpf_local_storage_map *)map;
  53         return bpf_local_storage_lookup(inode_storage, smap, cacheit_lockit);
  54 }
  55
  56 void bpf_inode_storage_free(struct inode *inode)
  57 {
  58         struct bpf_local_storage *local_storage;
  59         struct bpf_storage_blob *bsb;
  60
  61         bsb = bpf_inode(inode);
  62         if (!bsb)
  63                 return;
  64
  65         rcu_read_lock();
  66
  67         local_storage = rcu_dereference(bsb->storage);
  68         if (!local_storage) {
  69                 rcu_read_unlock();
  70                 return;
  71         }
  72
  73         bpf_local_storage_destroy(local_storage);
  74         rcu_read_unlock();
  75 }
  76
  77 static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key)
  78 {
  79         struct bpf_local_storage_data *sdata;
  80         CLASS(fd_raw, f)(*(int *)key);
  81
  82         if (fd_empty(f))
  83                 return ERR_PTR(-EBADF);
  84
  85         sdata = inode_storage_lookup(file_inode(fd_file(f)), map, true);
  86         return sdata ? sdata->data : NULL;
  87 }
  88
  89 static long bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key,
  90                                              void *value, u64 map_flags)
  91 {
  92         struct bpf_local_storage_data *sdata;
  93         CLASS(fd_raw, f)(*(int *)key);
  94
  95         if (fd_empty(f))
  96                 return -EBADF;
  97         if (!inode_storage_ptr(file_inode(fd_file(f))))
  98                 return -EBADF;
  99
 100         sdata = bpf_local_storage_update(file_inode(fd_file(f)),
 101                                          (struct bpf_local_storage_map *)map,
 102                                          value, map_flags, false, GFP_ATOMIC);
 103         return PTR_ERR_OR_ZERO(sdata);
 104 }
 105
 106 static int inode_storage_delete(struct inode *inode, struct bpf_map *map)
 107 {
 108         struct bpf_local_storage_data *sdata;
 109
 110         sdata = inode_storage_lookup(inode, map, false);
 111         if (!sdata)
 112                 return -ENOENT;
 113
 114         bpf_selem_unlink(SELEM(sdata), false);
 115
 116         return 0;
 117 }
 118
 119 static long bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key)
 120 {
 121         CLASS(fd_raw, f)(*(int *)key);
 122
 123         if (fd_empty(f))
 124                 return -EBADF;
 125         return inode_storage_delete(file_inode(fd_file(f)), map);
 126 }
 127
 128 /* *gfp_flags* is a hidden argument provided by the verifier */
 129 BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode,
 130            void *, value, u64, flags, gfp_t, gfp_flags)
 131 {
 132         struct bpf_local_storage_data *sdata;
 133
 134         WARN_ON_ONCE(!bpf_rcu_lock_held());
 135         if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
 136                 return (unsigned long)NULL;
 137
 138         /* explicitly check that the inode_storage_ptr is not
 139          * NULL as inode_storage_lookup returns NULL in this case and
 140          * bpf_local_storage_update expects the owner to have a
 141          * valid storage pointer.
 142          */
 143         if (!inode || !inode_storage_ptr(inode))
 144                 return (unsigned long)NULL;
 145
 146         sdata = inode_storage_lookup(inode, map, true);
 147         if (sdata)
 148                 return (unsigned long)sdata->data;
 149
 150         /* This helper must only called from where the inode is guaranteed
 151          * to have a refcount and cannot be freed.
 152          */
 153         if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
 154                 sdata = bpf_local_storage_update(
 155                         inode, (struct bpf_local_storage_map *)map, value,
 156                         BPF_NOEXIST, false, gfp_flags);
 157                 return IS_ERR(sdata) ? (unsigned long)NULL :
 158                                              (unsigned long)sdata->data;
 159         }
 160
 161         return (unsigned long)NULL;
 162 }
 163
 164 BPF_CALL_2(bpf_inode_storage_delete,
 165            struct bpf_map *, map, struct inode *, inode)
 166 {
 167         WARN_ON_ONCE(!bpf_rcu_lock_held());
 168         if (!inode)
 169                 return -EINVAL;
 170
 171         /* This helper must only called from where the inode is guaranteed
 172          * to have a refcount and cannot be freed.
 173          */
 174         return inode_storage_delete(inode, map);
 175 }
 176
 177 static int notsupp_get_next_key(struct bpf_map *map, void *key,
 178                                 void *next_key)
 179 {
 180         return -ENOTSUPP;
 181 }
 182
 183 static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr)
 184 {
 185         return bpf_local_storage_map_alloc(attr, &inode_cache, false);
 186 }
 187
 188 static void inode_storage_map_free(struct bpf_map *map)
 189 {
 190         bpf_local_storage_map_free(map, &inode_cache, NULL);
 191 }
 192
 193 const struct bpf_map_ops inode_storage_map_ops = {
 194         .map_meta_equal = bpf_map_meta_equal,
 195         .map_alloc_check = bpf_local_storage_map_alloc_check,
 196         .map_alloc = inode_storage_map_alloc,
 197         .map_free = inode_storage_map_free,
 198         .map_get_next_key = notsupp_get_next_key,
 199         .map_lookup_elem = bpf_fd_inode_storage_lookup_elem,
 200         .map_update_elem = bpf_fd_inode_storage_update_elem,
 201         .map_delete_elem = bpf_fd_inode_storage_delete_elem,
 202         .map_check_btf = bpf_local_storage_map_check_btf,
 203         .map_mem_usage = bpf_local_storage_map_mem_usage,
 204         .map_btf_id = &bpf_local_storage_map_btf_id[0],
 205         .map_owner_storage_ptr = inode_storage_ptr,
 206 };
 207
 208 BTF_ID_LIST_SINGLE(bpf_inode_storage_btf_ids, struct, inode)
 209
 210 const struct bpf_func_proto bpf_inode_storage_get_proto = {
 211         .func           = bpf_inode_storage_get,
 212         .gpl_only       = false,
 213         .ret_type       = RET_PTR_TO_MAP_VALUE_OR_NULL,
 214         .arg1_type      = ARG_CONST_MAP_PTR,
 215         .arg2_type      = ARG_PTR_TO_BTF_ID_OR_NULL,
 216         .arg2_btf_id    = &bpf_inode_storage_btf_ids[0],
 217         .arg3_type      = ARG_PTR_TO_MAP_VALUE_OR_NULL,
 218         .arg4_type      = ARG_ANYTHING,
 219 };
 220
 221 const struct bpf_func_proto bpf_inode_storage_delete_proto = {
 222         .func           = bpf_inode_storage_delete,
 223         .gpl_only       = false,
 224         .ret_type       = RET_INTEGER,
 225         .arg1_type      = ARG_CONST_MAP_PTR,
 226         .arg2_type      = ARG_PTR_TO_BTF_ID_OR_NULL,
 227         .arg2_btf_id    = &bpf_inode_storage_btf_ids[0],
 228 };