bpf: Allow RCU-protected lookups to happen from bh context

XDP programs are called from a NAPI poll context, which means the RCU
reference liveness is ensured by local_bh_disable(). Add
rcu_read_lock_bh_held() as a condition to the RCU checks for map lookups so
lockdep understands that the dereferences are safe from inside *either* an
rcu_read_lock() section *or* a local_bh_disable() section. While both
bh_disabled and rcu_read_lock() provide RCU protection, they are
semantically distinct, so we need both conditions to prevent lockdep
complaints.

This change is done in preparation for removing the redundant
rcu_read_lock()s from drivers.

Signed-off-by: Toke Høiland-Jørgensen <[email protected]>
Signed-off-by: Daniel Borkmann <[email protected]>
Acked-by: Martin KaFai Lau <[email protected]>
Link: https://lore.kernel.org/bpf/[email protected]

diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c
index 6f6681b07364fd791f0d27e5393102683896917b..72c58cc516a397bb5e973a20f3dcd3a0066f6a44 100644 (file)
--- a/kernel/bpf/hashtab.c
+++ b/kernel/bpf/hashtab.c
@@ -596,7 +596,8 @@ static void *__htab_map_lookup_elem(struct bpf_map *map, void *key)
        struct htab_elem *l;
        u32 hash, key_size;
 
-       WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
+       WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
+                    !rcu_read_lock_bh_held());
 
        key_size = map->key_size;
 
@@ -989,7 +990,8 @@ static int htab_map_update_elem(struct bpf_map *map, void *key, void *value,
                /* unknown flags */
                return -EINVAL;
 
-       WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
+       WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
+                    !rcu_read_lock_bh_held());
 
        key_size = map->key_size;
 
@@ -1082,7 +1084,8 @@ static int htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value,
                /* unknown flags */
                return -EINVAL;
 
-       WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
+       WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
+                    !rcu_read_lock_bh_held());
 
        key_size = map->key_size;
 
@@ -1148,7 +1151,8 @@ static int __htab_percpu_map_update_elem(struct bpf_map *map, void *key,
                /* unknown flags */
                return -EINVAL;
 
-       WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
+       WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
+                    !rcu_read_lock_bh_held());
 
        key_size = map->key_size;
 
@@ -1202,7 +1206,8 @@ static int __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,
                /* unknown flags */
                return -EINVAL;
 
-       WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
+       WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
+                    !rcu_read_lock_bh_held());
 
        key_size = map->key_size;
 
@@ -1276,7 +1281,8 @@ static int htab_map_delete_elem(struct bpf_map *map, void *key)
        u32 hash, key_size;
        int ret;
 
-       WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
+       WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
+                    !rcu_read_lock_bh_held());
 
        key_size = map->key_size;
 
@@ -1311,7 +1317,8 @@ static int htab_lru_map_delete_elem(struct bpf_map *map, void *key)
        u32 hash, key_size;
        int ret;
 
-       WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
+       WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
+                    !rcu_read_lock_bh_held());
 
        key_size = map->key_size;
 

diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index a2f1f15ce43216de9b53ea7906aab7ed8791f980..62cf0038391040e20520b5a9bb04f20ae50db5f1 100644 (file)
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -29,7 +29,7 @@
  */
 BPF_CALL_2(bpf_map_lookup_elem, struct bpf_map *, map, void *, key)
 {
-       WARN_ON_ONCE(!rcu_read_lock_held());
+       WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
        return (unsigned long) map->ops->map_lookup_elem(map, key);
 }
 
@@ -45,7 +45,7 @@ const struct bpf_func_proto bpf_map_lookup_elem_proto = {
 BPF_CALL_4(bpf_map_update_elem, struct bpf_map *, map, void *, key,
           void *, value, u64, flags)
 {
-       WARN_ON_ONCE(!rcu_read_lock_held());
+       WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
        return map->ops->map_update_elem(map, key, value, flags);
 }
 
@@ -62,7 +62,7 @@ const struct bpf_func_proto bpf_map_update_elem_proto = {
 
 BPF_CALL_2(bpf_map_delete_elem, struct bpf_map *, map, void *, key)
 {
-       WARN_ON_ONCE(!rcu_read_lock_held());
+       WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
        return map->ops->map_delete_elem(map, key);
 }
 

diff --git a/kernel/bpf/lpm_trie.c b/kernel/bpf/lpm_trie.c
index 1b7b8a6f34ee278c313f7caa3ac0b4a259161889..423549d2c52e2e1315147ca0bfaeb54bfa5acb33 100644 (file)
--- a/kernel/bpf/lpm_trie.c
+++ b/kernel/bpf/lpm_trie.c
@@ -232,7 +232,8 @@ static void *trie_lookup_elem(struct bpf_map *map, void *_key)
 
        /* Start walking the trie from the root node ... */
 
-       for (node = rcu_dereference(trie->root); node;) {
+       for (node = rcu_dereference_check(trie->root, rcu_read_lock_bh_held());
+            node;) {
                unsigned int next_bit;
                size_t matchlen;
 
@@ -264,7 +265,8 @@ static void *trie_lookup_elem(struct bpf_map *map, void *_key)
                 * traverse down.
                 */
                next_bit = extract_bit(key->data, node->prefixlen);
-               node = rcu_dereference(node->child[next_bit]);
+               node = rcu_dereference_check(node->child[next_bit],
+                                            rcu_read_lock_bh_held());
        }
 
        if (!found)