xfrm: speed-up lookup of HW policies

[linux.git] / net / xfrm / xfrm_state.c

diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c
index 4d315e1a88faae3ff061cb4de99b364655caf4d7..2a190e85da806660ffee6b326f8f11897b651e27 100644 (file)
--- a/net/xfrm/xfrm_state.c
+++ b/net/xfrm/xfrm_state.c
@@ -84,6 +84,25 @@ static unsigned int xfrm_seq_hash(struct net *net, u32 seq)
        return __xfrm_seq_hash(seq, net->xfrm.state_hmask);
 }
 
+#define XFRM_STATE_INSERT(by, _n, _h, _type)                               \
+       {                                                                  \
+               struct xfrm_state *_x = NULL;                              \
+                                                                          \
+               if (_type != XFRM_DEV_OFFLOAD_PACKET) {                    \
+                       hlist_for_each_entry_rcu(_x, _h, by) {             \
+                               if (_x->xso.type == XFRM_DEV_OFFLOAD_PACKET) \
+                                       continue;                          \
+                               break;                                     \
+                       }                                                  \
+               }                                                          \
+                                                                          \
+               if (!_x || _x->xso.type == XFRM_DEV_OFFLOAD_PACKET)        \
+                       /* SAD is empty or consist from HW SAs only */     \
+                       hlist_add_head_rcu(_n, _h);                        \
+               else                                                       \
+                       hlist_add_before_rcu(_n, &_x->by);                 \
+       }
+
 static void xfrm_hash_transfer(struct hlist_head *list,
                               struct hlist_head *ndsttable,
                               struct hlist_head *nsrctable,
@@ -100,23 +119,25 @@ static void xfrm_hash_transfer(struct hlist_head *list,
                h = __xfrm_dst_hash(&x->id.daddr, &x->props.saddr,
                                    x->props.reqid, x->props.family,
                                    nhashmask);
-               hlist_add_head_rcu(&x->bydst, ndsttable + h);
+               XFRM_STATE_INSERT(bydst, &x->bydst, ndsttable + h, x->xso.type);
 
                h = __xfrm_src_hash(&x->id.daddr, &x->props.saddr,
                                    x->props.family,
                                    nhashmask);
-               hlist_add_head_rcu(&x->bysrc, nsrctable + h);
+               XFRM_STATE_INSERT(bysrc, &x->bysrc, nsrctable + h, x->xso.type);
 
                if (x->id.spi) {
                        h = __xfrm_spi_hash(&x->id.daddr, x->id.spi,
                                            x->id.proto, x->props.family,
                                            nhashmask);
-                       hlist_add_head_rcu(&x->byspi, nspitable + h);
+                       XFRM_STATE_INSERT(byspi, &x->byspi, nspitable + h,
+                                         x->xso.type);
                }
 
                if (x->km.seq) {
                        h = __xfrm_seq_hash(x->km.seq, nhashmask);
-                       hlist_add_head_rcu(&x->byseq, nseqtable + h);
+                       XFRM_STATE_INSERT(byseq, &x->byseq, nseqtable + h,
+                                         x->xso.type);
                }
        }
 }
@@ -1269,16 +1290,24 @@ found:
                        spin_lock_bh(&net->xfrm.xfrm_state_lock);
                        x->km.state = XFRM_STATE_ACQ;
                        list_add(&x->km.all, &net->xfrm.state_all);
-                       hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h);
+                       XFRM_STATE_INSERT(bydst, &x->bydst,
+                                         net->xfrm.state_bydst + h,
+                                         x->xso.type);
                        h = xfrm_src_hash(net, daddr, saddr, encap_family);
-                       hlist_add_head_rcu(&x->bysrc, net->xfrm.state_bysrc + h);
+                       XFRM_STATE_INSERT(bysrc, &x->bysrc,
+                                         net->xfrm.state_bysrc + h,
+                                         x->xso.type);
                        if (x->id.spi) {
                                h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, encap_family);
-                               hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h);
+                               XFRM_STATE_INSERT(byspi, &x->byspi,
+                                                 net->xfrm.state_byspi + h,
+                                                 x->xso.type);
                        }
                        if (x->km.seq) {
                                h = xfrm_seq_hash(net, x->km.seq);
-                               hlist_add_head_rcu(&x->byseq, net->xfrm.state_byseq + h);
+                               XFRM_STATE_INSERT(byseq, &x->byseq,
+                                                 net->xfrm.state_byseq + h,
+                                                 x->xso.type);
                        }
                        x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;
                        hrtimer_start(&x->mtimer,
@@ -1395,22 +1424,26 @@ static void __xfrm_state_insert(struct xfrm_state *x)
 
        h = xfrm_dst_hash(net, &x->id.daddr, &x->props.saddr,
                          x->props.reqid, x->props.family);
-       hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h);
+       XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h,
+                         x->xso.type);
 
        h = xfrm_src_hash(net, &x->id.daddr, &x->props.saddr, x->props.family);
-       hlist_add_head_rcu(&x->bysrc, net->xfrm.state_bysrc + h);
+       XFRM_STATE_INSERT(bysrc, &x->bysrc, net->xfrm.state_bysrc + h,
+                         x->xso.type);
 
        if (x->id.spi) {
                h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto,
                                  x->props.family);
 
-               hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h);
+               XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h,
+                                 x->xso.type);
        }
 
        if (x->km.seq) {
                h = xfrm_seq_hash(net, x->km.seq);
 
-               hlist_add_head_rcu(&x->byseq, net->xfrm.state_byseq + h);
+               XFRM_STATE_INSERT(byseq, &x->byseq, net->xfrm.state_byseq + h,
+                                 x->xso.type);
        }
 
        hrtimer_start(&x->mtimer, ktime_set(1, 0), HRTIMER_MODE_REL_SOFT);
@@ -1524,9 +1557,11 @@ static struct xfrm_state *__find_acq_core(struct net *net,
                              ktime_set(net->xfrm.sysctl_acq_expires, 0),
                              HRTIMER_MODE_REL_SOFT);
                list_add(&x->km.all, &net->xfrm.state_all);
-               hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h);
+               XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h,
+                                 x->xso.type);
                h = xfrm_src_hash(net, daddr, saddr, family);
-               hlist_add_head_rcu(&x->bysrc, net->xfrm.state_bysrc + h);
+               XFRM_STATE_INSERT(bysrc, &x->bysrc, net->xfrm.state_bysrc + h,
+                                 x->xso.type);
 
                net->xfrm.state_num++;
 
@@ -2209,7 +2244,8 @@ int xfrm_alloc_spi(struct xfrm_state *x, u32 low, u32 high,
                spin_lock_bh(&net->xfrm.xfrm_state_lock);
                x->id.spi = newspi;
                h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, x->props.family);
-               hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h);
+               XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h,
+                                 x->xso.type);
                spin_unlock_bh(&net->xfrm.xfrm_state_lock);
 
                err = 0;