drivers/net/ethernet/sfc/tc_conntrack.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /****************************************************************************
   3  * Driver for Solarflare network controllers and boards
   4  * Copyright 2023, Advanced Micro Devices, Inc.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 as published
   8  * by the Free Software Foundation, incorporated herein by reference.
   9  */
  10
  11 #include "tc_conntrack.h"
  12 #include "tc.h"
  13 #include "mae.h"
  14
  15 static int efx_tc_flow_block(enum tc_setup_type type, void *type_data,
  16                              void *cb_priv);
  17
  18 static const struct rhashtable_params efx_tc_ct_zone_ht_params = {
  19         .key_len        = sizeof_field(struct efx_tc_ct_zone, zone),
  20         .key_offset     = 0,
  21         .head_offset    = offsetof(struct efx_tc_ct_zone, linkage),
  22 };
  23
  24 static const struct rhashtable_params efx_tc_ct_ht_params = {
  25         .key_len        = offsetof(struct efx_tc_ct_entry, linkage),
  26         .key_offset     = 0,
  27         .head_offset    = offsetof(struct efx_tc_ct_entry, linkage),
  28 };
  29
  30 static void efx_tc_ct_zone_free(void *ptr, void *arg)
  31 {
  32         struct efx_tc_ct_zone *zone = ptr;
  33         struct efx_nic *efx = zone->efx;
  34
  35         netif_err(efx, drv, efx->net_dev,
  36                   "tc ct_zone %u still present at teardown, removing\n",
  37                   zone->zone);
  38
  39         nf_flow_table_offload_del_cb(zone->nf_ft, efx_tc_flow_block, zone);
  40         kfree(zone);
  41 }
  42
  43 static void efx_tc_ct_free(void *ptr, void *arg)
  44 {
  45         struct efx_tc_ct_entry *conn = ptr;
  46         struct efx_nic *efx = arg;
  47
  48         netif_err(efx, drv, efx->net_dev,
  49                   "tc ct_entry %lx still present at teardown\n",
  50                   conn->cookie);
  51
  52         /* We can release the counter, but we can't remove the CT itself
  53          * from hardware because the table meta is already gone.
  54          */
  55         efx_tc_flower_release_counter(efx, conn->cnt);
  56         kfree(conn);
  57 }
  58
  59 int efx_tc_init_conntrack(struct efx_nic *efx)
  60 {
  61         int rc;
  62
  63         rc = rhashtable_init(&efx->tc->ct_zone_ht, &efx_tc_ct_zone_ht_params);
  64         if (rc < 0)
  65                 goto fail_ct_zone_ht;
  66         rc = rhashtable_init(&efx->tc->ct_ht, &efx_tc_ct_ht_params);
  67         if (rc < 0)
  68                 goto fail_ct_ht;
  69         return 0;
  70 fail_ct_ht:
  71         rhashtable_destroy(&efx->tc->ct_zone_ht);
  72 fail_ct_zone_ht:
  73         return rc;
  74 }
  75
  76 /* Only call this in init failure teardown.
  77  * Normal exit should fini instead as there may be entries in the table.
  78  */
  79 void efx_tc_destroy_conntrack(struct efx_nic *efx)
  80 {
  81         rhashtable_destroy(&efx->tc->ct_ht);
  82         rhashtable_destroy(&efx->tc->ct_zone_ht);
  83 }
  84
  85 void efx_tc_fini_conntrack(struct efx_nic *efx)
  86 {
  87         rhashtable_free_and_destroy(&efx->tc->ct_zone_ht, efx_tc_ct_zone_free, NULL);
  88         rhashtable_free_and_destroy(&efx->tc->ct_ht, efx_tc_ct_free, efx);
  89 }
  90
  91 #define EFX_NF_TCP_FLAG(flg)    cpu_to_be16(be32_to_cpu(TCP_FLAG_##flg) >> 16)
  92
  93 static int efx_tc_ct_parse_match(struct efx_nic *efx, struct flow_rule *fr,
  94                                  struct efx_tc_ct_entry *conn)
  95 {
  96         struct flow_dissector *dissector = fr->match.dissector;
  97         unsigned char ipv = 0;
  98         bool tcp = false;
  99
 100         if (flow_rule_match_key(fr, FLOW_DISSECTOR_KEY_CONTROL)) {
 101                 struct flow_match_control fm;
 102
 103                 flow_rule_match_control(fr, &fm);
 104                 if (IS_ALL_ONES(fm.mask->addr_type))
 105                         switch (fm.key->addr_type) {
 106                         case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
 107                                 ipv = 4;
 108                                 break;
 109                         case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
 110                                 ipv = 6;
 111                                 break;
 112                         default:
 113                                 break;
 114                         }
 115         }
 116
 117         if (!ipv) {
 118                 netif_dbg(efx, drv, efx->net_dev,
 119                           "Conntrack missing ipv specification\n");
 120                 return -EOPNOTSUPP;
 121         }
 122
 123         if (dissector->used_keys &
 124             ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |
 125               BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) |
 126               BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
 127               BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
 128               BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
 129               BIT_ULL(FLOW_DISSECTOR_KEY_TCP) |
 130               BIT_ULL(FLOW_DISSECTOR_KEY_META))) {
 131                 netif_dbg(efx, drv, efx->net_dev,
 132                           "Unsupported conntrack keys %#llx\n",
 133                           dissector->used_keys);
 134                 return -EOPNOTSUPP;
 135         }
 136
 137         if (flow_rule_match_key(fr, FLOW_DISSECTOR_KEY_BASIC)) {
 138                 struct flow_match_basic fm;
 139
 140                 flow_rule_match_basic(fr, &fm);
 141                 if (!IS_ALL_ONES(fm.mask->n_proto)) {
 142                         netif_dbg(efx, drv, efx->net_dev,
 143                                   "Conntrack eth_proto is not exact-match; mask %04x\n",
 144                                    ntohs(fm.mask->n_proto));
 145                         return -EOPNOTSUPP;
 146                 }
 147                 conn->eth_proto = fm.key->n_proto;
 148                 if (conn->eth_proto != (ipv == 4 ? htons(ETH_P_IP)
 149                                                  : htons(ETH_P_IPV6))) {
 150                         netif_dbg(efx, drv, efx->net_dev,
 151                                   "Conntrack eth_proto is not IPv%u, is %04x\n",
 152                                    ipv, ntohs(conn->eth_proto));
 153                         return -EOPNOTSUPP;
 154                 }
 155                 if (!IS_ALL_ONES(fm.mask->ip_proto)) {
 156                         netif_dbg(efx, drv, efx->net_dev,
 157                                   "Conntrack ip_proto is not exact-match; mask %02x\n",
 158                                    fm.mask->ip_proto);
 159                         return -EOPNOTSUPP;
 160                 }
 161                 conn->ip_proto = fm.key->ip_proto;
 162                 switch (conn->ip_proto) {
 163                 case IPPROTO_TCP:
 164                         tcp = true;
 165                         break;
 166                 case IPPROTO_UDP:
 167                         break;
 168                 default:
 169                         netif_dbg(efx, drv, efx->net_dev,
 170                                   "Conntrack ip_proto not TCP or UDP, is %02x\n",
 171                                    conn->ip_proto);
 172                         return -EOPNOTSUPP;
 173                 }
 174         } else {
 175                 netif_dbg(efx, drv, efx->net_dev,
 176                           "Conntrack missing eth_proto, ip_proto\n");
 177                 return -EOPNOTSUPP;
 178         }
 179
 180         if (ipv == 4 && flow_rule_match_key(fr, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
 181                 struct flow_match_ipv4_addrs fm;
 182
 183                 flow_rule_match_ipv4_addrs(fr, &fm);
 184                 if (!IS_ALL_ONES(fm.mask->src)) {
 185                         netif_dbg(efx, drv, efx->net_dev,
 186                                   "Conntrack ipv4.src is not exact-match; mask %08x\n",
 187                                    ntohl(fm.mask->src));
 188                         return -EOPNOTSUPP;
 189                 }
 190                 conn->src_ip = fm.key->src;
 191                 if (!IS_ALL_ONES(fm.mask->dst)) {
 192                         netif_dbg(efx, drv, efx->net_dev,
 193                                   "Conntrack ipv4.dst is not exact-match; mask %08x\n",
 194                                    ntohl(fm.mask->dst));
 195                         return -EOPNOTSUPP;
 196                 }
 197                 conn->dst_ip = fm.key->dst;
 198         } else if (ipv == 6 && flow_rule_match_key(fr, FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
 199                 struct flow_match_ipv6_addrs fm;
 200
 201                 flow_rule_match_ipv6_addrs(fr, &fm);
 202                 if (!efx_ipv6_addr_all_ones(&fm.mask->src)) {
 203                         netif_dbg(efx, drv, efx->net_dev,
 204                                   "Conntrack ipv6.src is not exact-match; mask %pI6\n",
 205                                    &fm.mask->src);
 206                         return -EOPNOTSUPP;
 207                 }
 208                 conn->src_ip6 = fm.key->src;
 209                 if (!efx_ipv6_addr_all_ones(&fm.mask->dst)) {
 210                         netif_dbg(efx, drv, efx->net_dev,
 211                                   "Conntrack ipv6.dst is not exact-match; mask %pI6\n",
 212                                    &fm.mask->dst);
 213                         return -EOPNOTSUPP;
 214                 }
 215                 conn->dst_ip6 = fm.key->dst;
 216         } else {
 217                 netif_dbg(efx, drv, efx->net_dev,
 218                           "Conntrack missing IPv%u addrs\n", ipv);
 219                 return -EOPNOTSUPP;
 220         }
 221
 222         if (flow_rule_match_key(fr, FLOW_DISSECTOR_KEY_PORTS)) {
 223                 struct flow_match_ports fm;
 224
 225                 flow_rule_match_ports(fr, &fm);
 226                 if (!IS_ALL_ONES(fm.mask->src)) {
 227                         netif_dbg(efx, drv, efx->net_dev,
 228                                   "Conntrack ports.src is not exact-match; mask %04x\n",
 229                                    ntohs(fm.mask->src));
 230                         return -EOPNOTSUPP;
 231                 }
 232                 conn->l4_sport = fm.key->src;
 233                 if (!IS_ALL_ONES(fm.mask->dst)) {
 234                         netif_dbg(efx, drv, efx->net_dev,
 235                                   "Conntrack ports.dst is not exact-match; mask %04x\n",
 236                                    ntohs(fm.mask->dst));
 237                         return -EOPNOTSUPP;
 238                 }
 239                 conn->l4_dport = fm.key->dst;
 240         } else {
 241                 netif_dbg(efx, drv, efx->net_dev, "Conntrack missing L4 ports\n");
 242                 return -EOPNOTSUPP;
 243         }
 244
 245         if (flow_rule_match_key(fr, FLOW_DISSECTOR_KEY_TCP)) {
 246                 __be16 tcp_interesting_flags;
 247                 struct flow_match_tcp fm;
 248
 249                 if (!tcp) {
 250                         netif_dbg(efx, drv, efx->net_dev,
 251                                   "Conntrack matching on TCP keys but ipproto is not tcp\n");
 252                         return -EOPNOTSUPP;
 253                 }
 254                 flow_rule_match_tcp(fr, &fm);
 255                 tcp_interesting_flags = EFX_NF_TCP_FLAG(SYN) |
 256                                         EFX_NF_TCP_FLAG(RST) |
 257                                         EFX_NF_TCP_FLAG(FIN);
 258                 /* If any of the tcp_interesting_flags is set, we always
 259                  * inhibit CT lookup in LHS (so SW can update CT table).
 260                  */
 261                 if (fm.key->flags & tcp_interesting_flags) {
 262                         netif_dbg(efx, drv, efx->net_dev,
 263                                   "Unsupported conntrack tcp.flags %04x/%04x\n",
 264                                    ntohs(fm.key->flags), ntohs(fm.mask->flags));
 265                         return -EOPNOTSUPP;
 266                 }
 267                 /* Other TCP flags cannot be filtered at CT */
 268                 if (fm.mask->flags & ~tcp_interesting_flags) {
 269                         netif_dbg(efx, drv, efx->net_dev,
 270                                   "Unsupported conntrack tcp.flags %04x/%04x\n",
 271                                    ntohs(fm.key->flags), ntohs(fm.mask->flags));
 272                         return -EOPNOTSUPP;
 273                 }
 274         }
 275
 276         return 0;
 277 }
 278
 279 /**
 280  * struct efx_tc_ct_mangler_state - tracks which fields have been pedited
 281  *
 282  * @ipv4: IP source or destination addr has been set
 283  * @tcpudp: TCP/UDP source or destination port has been set
 284  */
 285 struct efx_tc_ct_mangler_state {
 286         u8 ipv4:1;
 287         u8 tcpudp:1;
 288 };
 289
 290 static int efx_tc_ct_mangle(struct efx_nic *efx, struct efx_tc_ct_entry *conn,
 291                             const struct flow_action_entry *fa,
 292                             struct efx_tc_ct_mangler_state *mung)
 293 {
 294         /* Is this the first mangle we've processed for this rule? */
 295         bool first = !(mung->ipv4 || mung->tcpudp);
 296         bool dnat = false;
 297
 298         switch (fa->mangle.htype) {
 299         case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
 300                 switch (fa->mangle.offset) {
 301                 case offsetof(struct iphdr, daddr):
 302                         dnat = true;
 303                         fallthrough;
 304                 case offsetof(struct iphdr, saddr):
 305                         if (fa->mangle.mask)
 306                                 return -EOPNOTSUPP;
 307                         conn->nat_ip = htonl(fa->mangle.val);
 308                         mung->ipv4 = 1;
 309                         break;
 310                 default:
 311                         return -EOPNOTSUPP;
 312                 }
 313                 break;
 314         case FLOW_ACT_MANGLE_HDR_TYPE_TCP:
 315         case FLOW_ACT_MANGLE_HDR_TYPE_UDP:
 316                 /* Both struct tcphdr and struct udphdr start with
 317                  *      __be16 source;
 318                  *      __be16 dest;
 319                  * so we can use the same code for both.
 320                  */
 321                 switch (fa->mangle.offset) {
 322                 case offsetof(struct tcphdr, dest):
 323                         BUILD_BUG_ON(offsetof(struct tcphdr, dest) !=
 324                                      offsetof(struct udphdr, dest));
 325                         dnat = true;
 326                         fallthrough;
 327                 case offsetof(struct tcphdr, source):
 328                         BUILD_BUG_ON(offsetof(struct tcphdr, source) !=
 329                                      offsetof(struct udphdr, source));
 330                         if (~fa->mangle.mask != 0xffff)
 331                                 return -EOPNOTSUPP;
 332                         conn->l4_natport = htons(fa->mangle.val);
 333                         mung->tcpudp = 1;
 334                         break;
 335                 default:
 336                         return -EOPNOTSUPP;
 337                 }
 338                 break;
 339         default:
 340                 return -EOPNOTSUPP;
 341         }
 342         /* first mangle tells us whether this is SNAT or DNAT;
 343          * subsequent mangles must match that
 344          */
 345         if (first)
 346                 conn->dnat = dnat;
 347         else if (conn->dnat != dnat)
 348                 return -EOPNOTSUPP;
 349         return 0;
 350 }
 351
 352 static int efx_tc_ct_replace(struct efx_tc_ct_zone *ct_zone,
 353                              struct flow_cls_offload *tc)
 354 {
 355         struct flow_rule *fr = flow_cls_offload_flow_rule(tc);
 356         struct efx_tc_ct_mangler_state mung = {};
 357         struct efx_tc_ct_entry *conn, *old;
 358         struct efx_nic *efx = ct_zone->efx;
 359         const struct flow_action_entry *fa;
 360         struct efx_tc_counter *cnt;
 361         int rc, i;
 362
 363         if (WARN_ON(!efx->tc))
 364                 return -ENETDOWN;
 365         if (WARN_ON(!efx->tc->up))
 366                 return -ENETDOWN;
 367
 368         conn = kzalloc(sizeof(*conn), GFP_USER);
 369         if (!conn)
 370                 return -ENOMEM;
 371         conn->cookie = tc->cookie;
 372         old = rhashtable_lookup_get_insert_fast(&efx->tc->ct_ht,
 373                                                 &conn->linkage,
 374                                                 efx_tc_ct_ht_params);
 375         if (IS_ERR(old)) {
 376                 rc = PTR_ERR(old);
 377                 goto release;
 378         } else if (old) {
 379                 netif_dbg(efx, drv, efx->net_dev,
 380                           "Already offloaded conntrack (cookie %lx)\n", tc->cookie);
 381                 rc = -EEXIST;
 382                 goto release;
 383         }
 384
 385         /* Parse match */
 386         conn->zone = ct_zone;
 387         rc = efx_tc_ct_parse_match(efx, fr, conn);
 388         if (rc)
 389                 goto release;
 390
 391         /* Parse actions */
 392         flow_action_for_each(i, fa, &fr->action) {
 393                 switch (fa->id) {
 394                 case FLOW_ACTION_CT_METADATA:
 395                         conn->mark = fa->ct_metadata.mark;
 396                         if (memchr_inv(fa->ct_metadata.labels, 0, sizeof(fa->ct_metadata.labels))) {
 397                                 netif_dbg(efx, drv, efx->net_dev,
 398                                           "Setting CT label not supported\n");
 399                                 rc = -EOPNOTSUPP;
 400                                 goto release;
 401                         }
 402                         break;
 403                 case FLOW_ACTION_MANGLE:
 404                         if (conn->eth_proto != htons(ETH_P_IP)) {
 405                                 netif_dbg(efx, drv, efx->net_dev,
 406                                           "NAT only supported for IPv4\n");
 407                                 rc = -EOPNOTSUPP;
 408                                 goto release;
 409                         }
 410                         rc = efx_tc_ct_mangle(efx, conn, fa, &mung);
 411                         if (rc)
 412                                 goto release;
 413                         break;
 414                 default:
 415                         netif_dbg(efx, drv, efx->net_dev,
 416                                   "Unhandled action %u for conntrack\n", fa->id);
 417                         rc = -EOPNOTSUPP;
 418                         goto release;
 419                 }
 420         }
 421
 422         /* fill in defaults for unmangled values */
 423         if (!mung.ipv4)
 424                 conn->nat_ip = conn->dnat ? conn->dst_ip : conn->src_ip;
 425         if (!mung.tcpudp)
 426                 conn->l4_natport = conn->dnat ? conn->l4_dport : conn->l4_sport;
 427
 428         cnt = efx_tc_flower_allocate_counter(efx, EFX_TC_COUNTER_TYPE_CT);
 429         if (IS_ERR(cnt)) {
 430                 rc = PTR_ERR(cnt);
 431                 goto release;
 432         }
 433         conn->cnt = cnt;
 434
 435         rc = efx_mae_insert_ct(efx, conn);
 436         if (rc) {
 437                 netif_dbg(efx, drv, efx->net_dev,
 438                           "Failed to insert conntrack, %d\n", rc);
 439                 goto release;
 440         }
 441         mutex_lock(&ct_zone->mutex);
 442         list_add_tail(&conn->list, &ct_zone->cts);
 443         mutex_unlock(&ct_zone->mutex);
 444         return 0;
 445 release:
 446         if (conn->cnt)
 447                 efx_tc_flower_release_counter(efx, conn->cnt);
 448         if (!old)
 449                 rhashtable_remove_fast(&efx->tc->ct_ht, &conn->linkage,
 450                                        efx_tc_ct_ht_params);
 451         kfree(conn);
 452         return rc;
 453 }
 454
 455 /* Caller must follow with efx_tc_ct_remove_finish() after RCU grace period! */
 456 static void efx_tc_ct_remove(struct efx_nic *efx, struct efx_tc_ct_entry *conn)
 457 {
 458         int rc;
 459
 460         /* Remove it from HW */
 461         rc = efx_mae_remove_ct(efx, conn);
 462         /* Delete it from SW */
 463         rhashtable_remove_fast(&efx->tc->ct_ht, &conn->linkage,
 464                                efx_tc_ct_ht_params);
 465         if (rc) {
 466                 netif_err(efx, drv, efx->net_dev,
 467                           "Failed to remove conntrack %lx from hw, rc %d\n",
 468                           conn->cookie, rc);
 469         } else {
 470                 netif_dbg(efx, drv, efx->net_dev, "Removed conntrack %lx\n",
 471                           conn->cookie);
 472         }
 473 }
 474
 475 static void efx_tc_ct_remove_finish(struct efx_nic *efx, struct efx_tc_ct_entry *conn)
 476 {
 477         /* Remove related CT counter.  This is delayed after the conn object we
 478          * are working with has been successfully removed.  This protects the
 479          * counter from being used-after-free inside efx_tc_ct_stats.
 480          */
 481         efx_tc_flower_release_counter(efx, conn->cnt);
 482         kfree(conn);
 483 }
 484
 485 static int efx_tc_ct_destroy(struct efx_tc_ct_zone *ct_zone,
 486                              struct flow_cls_offload *tc)
 487 {
 488         struct efx_nic *efx = ct_zone->efx;
 489         struct efx_tc_ct_entry *conn;
 490
 491         conn = rhashtable_lookup_fast(&efx->tc->ct_ht, &tc->cookie,
 492                                       efx_tc_ct_ht_params);
 493         if (!conn) {
 494                 netif_warn(efx, drv, efx->net_dev,
 495                            "Conntrack %lx not found to remove\n", tc->cookie);
 496                 return -ENOENT;
 497         }
 498
 499         mutex_lock(&ct_zone->mutex);
 500         list_del(&conn->list);
 501         efx_tc_ct_remove(efx, conn);
 502         mutex_unlock(&ct_zone->mutex);
 503         synchronize_rcu();
 504         efx_tc_ct_remove_finish(efx, conn);
 505         return 0;
 506 }
 507
 508 static int efx_tc_ct_stats(struct efx_tc_ct_zone *ct_zone,
 509                            struct flow_cls_offload *tc)
 510 {
 511         struct efx_nic *efx = ct_zone->efx;
 512         struct efx_tc_ct_entry *conn;
 513         struct efx_tc_counter *cnt;
 514
 515         rcu_read_lock();
 516         conn = rhashtable_lookup_fast(&efx->tc->ct_ht, &tc->cookie,
 517                                       efx_tc_ct_ht_params);
 518         if (!conn) {
 519                 netif_warn(efx, drv, efx->net_dev,
 520                            "Conntrack %lx not found for stats\n", tc->cookie);
 521                 rcu_read_unlock();
 522                 return -ENOENT;
 523         }
 524
 525         cnt = conn->cnt;
 526         spin_lock_bh(&cnt->lock);
 527         /* Report only last use */
 528         flow_stats_update(&tc->stats, 0, 0, 0, cnt->touched,
 529                           FLOW_ACTION_HW_STATS_DELAYED);
 530         spin_unlock_bh(&cnt->lock);
 531         rcu_read_unlock();
 532
 533         return 0;
 534 }
 535
 536 static int efx_tc_flow_block(enum tc_setup_type type, void *type_data,
 537                              void *cb_priv)
 538 {
 539         struct flow_cls_offload *tcb = type_data;
 540         struct efx_tc_ct_zone *ct_zone = cb_priv;
 541
 542         if (type != TC_SETUP_CLSFLOWER)
 543                 return -EOPNOTSUPP;
 544
 545         switch (tcb->command) {
 546         case FLOW_CLS_REPLACE:
 547                 return efx_tc_ct_replace(ct_zone, tcb);
 548         case FLOW_CLS_DESTROY:
 549                 return efx_tc_ct_destroy(ct_zone, tcb);
 550         case FLOW_CLS_STATS:
 551                 return efx_tc_ct_stats(ct_zone, tcb);
 552         default:
 553                 break;
 554         }
 555
 556         return -EOPNOTSUPP;
 557 }
 558
 559 struct efx_tc_ct_zone *efx_tc_ct_register_zone(struct efx_nic *efx, u16 zone,
 560                                                struct nf_flowtable *ct_ft)
 561 {
 562         struct efx_tc_ct_zone *ct_zone, *old;
 563         int rc;
 564
 565         ct_zone = kzalloc(sizeof(*ct_zone), GFP_USER);
 566         if (!ct_zone)
 567                 return ERR_PTR(-ENOMEM);
 568         ct_zone->zone = zone;
 569         old = rhashtable_lookup_get_insert_fast(&efx->tc->ct_zone_ht,
 570                                                 &ct_zone->linkage,
 571                                                 efx_tc_ct_zone_ht_params);
 572         if (old) {
 573                 /* don't need our new entry */
 574                 kfree(ct_zone);
 575                 if (IS_ERR(old)) /* oh dear, it's actually an error */
 576                         return ERR_CAST(old);
 577                 if (!refcount_inc_not_zero(&old->ref))
 578                         return ERR_PTR(-EAGAIN);
 579                 /* existing entry found */
 580                 WARN_ON_ONCE(old->nf_ft != ct_ft);
 581                 netif_dbg(efx, drv, efx->net_dev,
 582                           "Found existing ct_zone for %u\n", zone);
 583                 return old;
 584         }
 585         ct_zone->nf_ft = ct_ft;
 586         ct_zone->efx = efx;
 587         INIT_LIST_HEAD(&ct_zone->cts);
 588         mutex_init(&ct_zone->mutex);
 589         rc = nf_flow_table_offload_add_cb(ct_ft, efx_tc_flow_block, ct_zone);
 590         netif_dbg(efx, drv, efx->net_dev, "Adding new ct_zone for %u, rc %d\n",
 591                   zone, rc);
 592         if (rc < 0)
 593                 goto fail;
 594         refcount_set(&ct_zone->ref, 1);
 595         return ct_zone;
 596 fail:
 597         rhashtable_remove_fast(&efx->tc->ct_zone_ht, &ct_zone->linkage,
 598                                efx_tc_ct_zone_ht_params);
 599         kfree(ct_zone);
 600         return ERR_PTR(rc);
 601 }
 602
 603 void efx_tc_ct_unregister_zone(struct efx_nic *efx,
 604                                struct efx_tc_ct_zone *ct_zone)
 605 {
 606         struct efx_tc_ct_entry *conn, *next;
 607
 608         if (!refcount_dec_and_test(&ct_zone->ref))
 609                 return; /* still in use */
 610         nf_flow_table_offload_del_cb(ct_zone->nf_ft, efx_tc_flow_block, ct_zone);
 611         rhashtable_remove_fast(&efx->tc->ct_zone_ht, &ct_zone->linkage,
 612                                efx_tc_ct_zone_ht_params);
 613         mutex_lock(&ct_zone->mutex);
 614         list_for_each_entry(conn, &ct_zone->cts, list)
 615                 efx_tc_ct_remove(efx, conn);
 616         synchronize_rcu();
 617         /* need to use _safe because efx_tc_ct_remove_finish() frees conn */
 618         list_for_each_entry_safe(conn, next, &ct_zone->cts, list)
 619                 efx_tc_ct_remove_finish(efx, conn);
 620         mutex_unlock(&ct_zone->mutex);
 621         mutex_destroy(&ct_zone->mutex);
 622         netif_dbg(efx, drv, efx->net_dev, "Removed ct_zone for %u\n",
 623                   ct_zone->zone);
 624         kfree(ct_zone);
 625 }