net/netfilter/nf_tables_api.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2007-2009 Patrick McHardy <[email protected]>
   4  *
   5  * Development of this code funded by Astaro AG (http://www.astaro.com/)
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/init.h>
  10 #include <linux/list.h>
  11 #include <linux/skbuff.h>
  12 #include <linux/netlink.h>
  13 #include <linux/vmalloc.h>
  14 #include <linux/rhashtable.h>
  15 #include <linux/audit.h>
  16 #include <linux/netfilter.h>
  17 #include <linux/netfilter/nfnetlink.h>
  18 #include <linux/netfilter/nf_tables.h>
  19 #include <net/netfilter/nf_flow_table.h>
  20 #include <net/netfilter/nf_tables_core.h>
  21 #include <net/netfilter/nf_tables.h>
  22 #include <net/netfilter/nf_tables_offload.h>
  23 #include <net/net_namespace.h>
  24 #include <net/sock.h>
  25
  26 #define NFT_MODULE_AUTOLOAD_LIMIT (MODULE_NAME_LEN - sizeof("nft-expr-255-"))
  27 #define NFT_SET_MAX_ANONLEN 16
  28
  29 /* limit compaction to avoid huge kmalloc/krealloc sizes. */
  30 #define NFT_MAX_SET_NELEMS ((2048 - sizeof(struct nft_trans_elem)) / sizeof(struct nft_trans_one_elem))
  31
  32 unsigned int nf_tables_net_id __read_mostly;
  33
  34 static LIST_HEAD(nf_tables_expressions);
  35 static LIST_HEAD(nf_tables_objects);
  36 static LIST_HEAD(nf_tables_flowtables);
  37 static LIST_HEAD(nf_tables_destroy_list);
  38 static LIST_HEAD(nf_tables_gc_list);
  39 static DEFINE_SPINLOCK(nf_tables_destroy_list_lock);
  40 static DEFINE_SPINLOCK(nf_tables_gc_list_lock);
  41
  42 enum {
  43         NFT_VALIDATE_SKIP       = 0,
  44         NFT_VALIDATE_NEED,
  45         NFT_VALIDATE_DO,
  46 };
  47
  48 static struct rhltable nft_objname_ht;
  49
  50 static u32 nft_chain_hash(const void *data, u32 len, u32 seed);
  51 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed);
  52 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *, const void *);
  53
  54 static u32 nft_objname_hash(const void *data, u32 len, u32 seed);
  55 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed);
  56 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *, const void *);
  57
  58 static const struct rhashtable_params nft_chain_ht_params = {
  59         .head_offset            = offsetof(struct nft_chain, rhlhead),
  60         .key_offset             = offsetof(struct nft_chain, name),
  61         .hashfn                 = nft_chain_hash,
  62         .obj_hashfn             = nft_chain_hash_obj,
  63         .obj_cmpfn              = nft_chain_hash_cmp,
  64         .automatic_shrinking    = true,
  65 };
  66
  67 static const struct rhashtable_params nft_objname_ht_params = {
  68         .head_offset            = offsetof(struct nft_object, rhlhead),
  69         .key_offset             = offsetof(struct nft_object, key),
  70         .hashfn                 = nft_objname_hash,
  71         .obj_hashfn             = nft_objname_hash_obj,
  72         .obj_cmpfn              = nft_objname_hash_cmp,
  73         .automatic_shrinking    = true,
  74 };
  75
  76 struct nft_audit_data {
  77         struct nft_table *table;
  78         int entries;
  79         int op;
  80         struct list_head list;
  81 };
  82
  83 static const u8 nft2audit_op[NFT_MSG_MAX] = { // enum nf_tables_msg_types
  84         [NFT_MSG_NEWTABLE]      = AUDIT_NFT_OP_TABLE_REGISTER,
  85         [NFT_MSG_GETTABLE]      = AUDIT_NFT_OP_INVALID,
  86         [NFT_MSG_DELTABLE]      = AUDIT_NFT_OP_TABLE_UNREGISTER,
  87         [NFT_MSG_NEWCHAIN]      = AUDIT_NFT_OP_CHAIN_REGISTER,
  88         [NFT_MSG_GETCHAIN]      = AUDIT_NFT_OP_INVALID,
  89         [NFT_MSG_DELCHAIN]      = AUDIT_NFT_OP_CHAIN_UNREGISTER,
  90         [NFT_MSG_NEWRULE]       = AUDIT_NFT_OP_RULE_REGISTER,
  91         [NFT_MSG_GETRULE]       = AUDIT_NFT_OP_INVALID,
  92         [NFT_MSG_DELRULE]       = AUDIT_NFT_OP_RULE_UNREGISTER,
  93         [NFT_MSG_NEWSET]        = AUDIT_NFT_OP_SET_REGISTER,
  94         [NFT_MSG_GETSET]        = AUDIT_NFT_OP_INVALID,
  95         [NFT_MSG_DELSET]        = AUDIT_NFT_OP_SET_UNREGISTER,
  96         [NFT_MSG_NEWSETELEM]    = AUDIT_NFT_OP_SETELEM_REGISTER,
  97         [NFT_MSG_GETSETELEM]    = AUDIT_NFT_OP_INVALID,
  98         [NFT_MSG_DELSETELEM]    = AUDIT_NFT_OP_SETELEM_UNREGISTER,
  99         [NFT_MSG_NEWGEN]        = AUDIT_NFT_OP_GEN_REGISTER,
 100         [NFT_MSG_GETGEN]        = AUDIT_NFT_OP_INVALID,
 101         [NFT_MSG_TRACE]         = AUDIT_NFT_OP_INVALID,
 102         [NFT_MSG_NEWOBJ]        = AUDIT_NFT_OP_OBJ_REGISTER,
 103         [NFT_MSG_GETOBJ]        = AUDIT_NFT_OP_INVALID,
 104         [NFT_MSG_DELOBJ]        = AUDIT_NFT_OP_OBJ_UNREGISTER,
 105         [NFT_MSG_GETOBJ_RESET]  = AUDIT_NFT_OP_OBJ_RESET,
 106         [NFT_MSG_NEWFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_REGISTER,
 107         [NFT_MSG_GETFLOWTABLE]  = AUDIT_NFT_OP_INVALID,
 108         [NFT_MSG_DELFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_UNREGISTER,
 109         [NFT_MSG_GETSETELEM_RESET] = AUDIT_NFT_OP_SETELEM_RESET,
 110 };
 111
 112 static void nft_validate_state_update(struct nft_table *table, u8 new_validate_state)
 113 {
 114         switch (table->validate_state) {
 115         case NFT_VALIDATE_SKIP:
 116                 WARN_ON_ONCE(new_validate_state == NFT_VALIDATE_DO);
 117                 break;
 118         case NFT_VALIDATE_NEED:
 119                 break;
 120         case NFT_VALIDATE_DO:
 121                 if (new_validate_state == NFT_VALIDATE_NEED)
 122                         return;
 123         }
 124
 125         table->validate_state = new_validate_state;
 126 }
 127 static void nf_tables_trans_destroy_work(struct work_struct *w);
 128 static DECLARE_WORK(trans_destroy_work, nf_tables_trans_destroy_work);
 129
 130 static void nft_trans_gc_work(struct work_struct *work);
 131 static DECLARE_WORK(trans_gc_work, nft_trans_gc_work);
 132
 133 static void nft_ctx_init(struct nft_ctx *ctx,
 134                          struct net *net,
 135                          const struct sk_buff *skb,
 136                          const struct nlmsghdr *nlh,
 137                          u8 family,
 138                          struct nft_table *table,
 139                          struct nft_chain *chain,
 140                          const struct nlattr * const *nla)
 141 {
 142         ctx->net        = net;
 143         ctx->family     = family;
 144         ctx->level      = 0;
 145         ctx->table      = table;
 146         ctx->chain      = chain;
 147         ctx->nla        = nla;
 148         ctx->portid     = NETLINK_CB(skb).portid;
 149         ctx->report     = nlmsg_report(nlh);
 150         ctx->flags      = nlh->nlmsg_flags;
 151         ctx->seq        = nlh->nlmsg_seq;
 152
 153         bitmap_zero(ctx->reg_inited, NFT_REG32_NUM);
 154 }
 155
 156 static struct nft_trans *nft_trans_alloc_gfp(const struct nft_ctx *ctx,
 157                                              int msg_type, u32 size, gfp_t gfp)
 158 {
 159         struct nft_trans *trans;
 160
 161         trans = kzalloc(size, gfp);
 162         if (trans == NULL)
 163                 return NULL;
 164
 165         INIT_LIST_HEAD(&trans->list);
 166         trans->msg_type = msg_type;
 167
 168         trans->net = ctx->net;
 169         trans->table = ctx->table;
 170         trans->seq = ctx->seq;
 171         trans->flags = ctx->flags;
 172         trans->report = ctx->report;
 173
 174         return trans;
 175 }
 176
 177 static struct nft_trans *nft_trans_alloc(const struct nft_ctx *ctx,
 178                                          int msg_type, u32 size)
 179 {
 180         return nft_trans_alloc_gfp(ctx, msg_type, size, GFP_KERNEL);
 181 }
 182
 183 static struct nft_trans_binding *nft_trans_get_binding(struct nft_trans *trans)
 184 {
 185         switch (trans->msg_type) {
 186         case NFT_MSG_NEWCHAIN:
 187         case NFT_MSG_NEWSET:
 188                 return container_of(trans, struct nft_trans_binding, nft_trans);
 189         }
 190
 191         return NULL;
 192 }
 193
 194 static void nft_trans_list_del(struct nft_trans *trans)
 195 {
 196         struct nft_trans_binding *trans_binding;
 197
 198         list_del(&trans->list);
 199
 200         trans_binding = nft_trans_get_binding(trans);
 201         if (trans_binding)
 202                 list_del(&trans_binding->binding_list);
 203 }
 204
 205 static void nft_trans_destroy(struct nft_trans *trans)
 206 {
 207         nft_trans_list_del(trans);
 208         kfree(trans);
 209 }
 210
 211 static void __nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set,
 212                                  bool bind)
 213 {
 214         struct nftables_pernet *nft_net;
 215         struct net *net = ctx->net;
 216         struct nft_trans *trans;
 217
 218         if (!nft_set_is_anonymous(set))
 219                 return;
 220
 221         nft_net = nft_pernet(net);
 222         list_for_each_entry_reverse(trans, &nft_net->commit_list, list) {
 223                 switch (trans->msg_type) {
 224                 case NFT_MSG_NEWSET:
 225                         if (nft_trans_set(trans) == set)
 226                                 nft_trans_set_bound(trans) = bind;
 227                         break;
 228                 case NFT_MSG_NEWSETELEM:
 229                         if (nft_trans_elem_set(trans) == set)
 230                                 nft_trans_elem_set_bound(trans) = bind;
 231                         break;
 232                 }
 233         }
 234 }
 235
 236 static void nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set)
 237 {
 238         return __nft_set_trans_bind(ctx, set, true);
 239 }
 240
 241 static void nft_set_trans_unbind(const struct nft_ctx *ctx, struct nft_set *set)
 242 {
 243         return __nft_set_trans_bind(ctx, set, false);
 244 }
 245
 246 static void __nft_chain_trans_bind(const struct nft_ctx *ctx,
 247                                    struct nft_chain *chain, bool bind)
 248 {
 249         struct nftables_pernet *nft_net;
 250         struct net *net = ctx->net;
 251         struct nft_trans *trans;
 252
 253         if (!nft_chain_binding(chain))
 254                 return;
 255
 256         nft_net = nft_pernet(net);
 257         list_for_each_entry_reverse(trans, &nft_net->commit_list, list) {
 258                 switch (trans->msg_type) {
 259                 case NFT_MSG_NEWCHAIN:
 260                         if (nft_trans_chain(trans) == chain)
 261                                 nft_trans_chain_bound(trans) = bind;
 262                         break;
 263                 case NFT_MSG_NEWRULE:
 264                         if (nft_trans_rule_chain(trans) == chain)
 265                                 nft_trans_rule_bound(trans) = bind;
 266                         break;
 267                 }
 268         }
 269 }
 270
 271 static void nft_chain_trans_bind(const struct nft_ctx *ctx,
 272                                  struct nft_chain *chain)
 273 {
 274         __nft_chain_trans_bind(ctx, chain, true);
 275 }
 276
 277 int nf_tables_bind_chain(const struct nft_ctx *ctx, struct nft_chain *chain)
 278 {
 279         if (!nft_chain_binding(chain))
 280                 return 0;
 281
 282         if (nft_chain_binding(ctx->chain))
 283                 return -EOPNOTSUPP;
 284
 285         if (chain->bound)
 286                 return -EBUSY;
 287
 288         if (!nft_use_inc(&chain->use))
 289                 return -EMFILE;
 290
 291         chain->bound = true;
 292         nft_chain_trans_bind(ctx, chain);
 293
 294         return 0;
 295 }
 296
 297 void nf_tables_unbind_chain(const struct nft_ctx *ctx, struct nft_chain *chain)
 298 {
 299         __nft_chain_trans_bind(ctx, chain, false);
 300 }
 301
 302 static int nft_netdev_register_hooks(struct net *net,
 303                                      struct list_head *hook_list)
 304 {
 305         struct nft_hook *hook;
 306         int err, j;
 307
 308         j = 0;
 309         list_for_each_entry(hook, hook_list, list) {
 310                 err = nf_register_net_hook(net, &hook->ops);
 311                 if (err < 0)
 312                         goto err_register;
 313
 314                 j++;
 315         }
 316         return 0;
 317
 318 err_register:
 319         list_for_each_entry(hook, hook_list, list) {
 320                 if (j-- <= 0)
 321                         break;
 322
 323                 nf_unregister_net_hook(net, &hook->ops);
 324         }
 325         return err;
 326 }
 327
 328 static void nft_netdev_unregister_hooks(struct net *net,
 329                                         struct list_head *hook_list,
 330                                         bool release_netdev)
 331 {
 332         struct nft_hook *hook, *next;
 333
 334         list_for_each_entry_safe(hook, next, hook_list, list) {
 335                 nf_unregister_net_hook(net, &hook->ops);
 336                 if (release_netdev) {
 337                         list_del(&hook->list);
 338                         kfree_rcu(hook, rcu);
 339                 }
 340         }
 341 }
 342
 343 static int nf_tables_register_hook(struct net *net,
 344                                    const struct nft_table *table,
 345                                    struct nft_chain *chain)
 346 {
 347         struct nft_base_chain *basechain;
 348         const struct nf_hook_ops *ops;
 349
 350         if (table->flags & NFT_TABLE_F_DORMANT ||
 351             !nft_is_base_chain(chain))
 352                 return 0;
 353
 354         basechain = nft_base_chain(chain);
 355         ops = &basechain->ops;
 356
 357         if (basechain->type->ops_register)
 358                 return basechain->type->ops_register(net, ops);
 359
 360         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 361                 return nft_netdev_register_hooks(net, &basechain->hook_list);
 362
 363         return nf_register_net_hook(net, &basechain->ops);
 364 }
 365
 366 static void __nf_tables_unregister_hook(struct net *net,
 367                                         const struct nft_table *table,
 368                                         struct nft_chain *chain,
 369                                         bool release_netdev)
 370 {
 371         struct nft_base_chain *basechain;
 372         const struct nf_hook_ops *ops;
 373
 374         if (table->flags & NFT_TABLE_F_DORMANT ||
 375             !nft_is_base_chain(chain))
 376                 return;
 377         basechain = nft_base_chain(chain);
 378         ops = &basechain->ops;
 379
 380         if (basechain->type->ops_unregister)
 381                 return basechain->type->ops_unregister(net, ops);
 382
 383         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 384                 nft_netdev_unregister_hooks(net, &basechain->hook_list,
 385                                             release_netdev);
 386         else
 387                 nf_unregister_net_hook(net, &basechain->ops);
 388 }
 389
 390 static void nf_tables_unregister_hook(struct net *net,
 391                                       const struct nft_table *table,
 392                                       struct nft_chain *chain)
 393 {
 394         return __nf_tables_unregister_hook(net, table, chain, false);
 395 }
 396
 397 static bool nft_trans_collapse_set_elem_allowed(const struct nft_trans_elem *a, const struct nft_trans_elem *b)
 398 {
 399         /* NB: the ->bound equality check is defensive, at this time we only merge
 400          * a new nft_trans_elem transaction request with the transaction tail
 401          * element, but a->bound != b->bound would imply a NEWRULE transaction
 402          * is queued in-between.
 403          *
 404          * The set check is mandatory, the NFT_MAX_SET_NELEMS check prevents
 405          * huge krealloc() requests.
 406          */
 407         return a->set == b->set && a->bound == b->bound && a->nelems < NFT_MAX_SET_NELEMS;
 408 }
 409
 410 static bool nft_trans_collapse_set_elem(struct nftables_pernet *nft_net,
 411                                         struct nft_trans_elem *tail,
 412                                         struct nft_trans_elem *trans,
 413                                         gfp_t gfp)
 414 {
 415         unsigned int nelems, old_nelems = tail->nelems;
 416         struct nft_trans_elem *new_trans;
 417
 418         if (!nft_trans_collapse_set_elem_allowed(tail, trans))
 419                 return false;
 420
 421         /* "cannot happen", at this time userspace element add
 422          * requests always allocate a new transaction element.
 423          *
 424          * This serves as a reminder to adjust the list_add_tail
 425          * logic below in case this ever changes.
 426          */
 427         if (WARN_ON_ONCE(trans->nelems != 1))
 428                 return false;
 429
 430         if (check_add_overflow(old_nelems, trans->nelems, &nelems))
 431                 return false;
 432
 433         /* krealloc might free tail which invalidates list pointers */
 434         list_del_init(&tail->nft_trans.list);
 435
 436         new_trans = krealloc(tail, struct_size(tail, elems, nelems), gfp);
 437         if (!new_trans) {
 438                 list_add_tail(&tail->nft_trans.list, &nft_net->commit_list);
 439                 return false;
 440         }
 441
 442         /*
 443          * new_trans->nft_trans.list contains garbage, but
 444          * list_add_tail() doesn't care.
 445          */
 446         new_trans->nelems = nelems;
 447         new_trans->elems[old_nelems] = trans->elems[0];
 448         list_add_tail(&new_trans->nft_trans.list, &nft_net->commit_list);
 449
 450         return true;
 451 }
 452
 453 static bool nft_trans_try_collapse(struct nftables_pernet *nft_net,
 454                                    struct nft_trans *trans, gfp_t gfp)
 455 {
 456         struct nft_trans *tail;
 457
 458         if (list_empty(&nft_net->commit_list))
 459                 return false;
 460
 461         tail = list_last_entry(&nft_net->commit_list, struct nft_trans, list);
 462
 463         if (tail->msg_type != trans->msg_type)
 464                 return false;
 465
 466         switch (trans->msg_type) {
 467         case NFT_MSG_NEWSETELEM:
 468         case NFT_MSG_DELSETELEM:
 469                 return nft_trans_collapse_set_elem(nft_net,
 470                                                    nft_trans_container_elem(tail),
 471                                                    nft_trans_container_elem(trans), gfp);
 472         }
 473
 474         return false;
 475 }
 476
 477 static void nft_trans_commit_list_add_tail(struct net *net, struct nft_trans *trans)
 478 {
 479         struct nftables_pernet *nft_net = nft_pernet(net);
 480         struct nft_trans_binding *binding;
 481         struct nft_trans_set *trans_set;
 482
 483         list_add_tail(&trans->list, &nft_net->commit_list);
 484
 485         binding = nft_trans_get_binding(trans);
 486         if (!binding)
 487                 return;
 488
 489         switch (trans->msg_type) {
 490         case NFT_MSG_NEWSET:
 491                 trans_set = nft_trans_container_set(trans);
 492
 493                 if (!nft_trans_set_update(trans) &&
 494                     nft_set_is_anonymous(nft_trans_set(trans)))
 495                         list_add_tail(&binding->binding_list, &nft_net->binding_list);
 496
 497                 list_add_tail(&trans_set->list_trans_newset, &nft_net->commit_set_list);
 498                 break;
 499         case NFT_MSG_NEWCHAIN:
 500                 if (!nft_trans_chain_update(trans) &&
 501                     nft_chain_binding(nft_trans_chain(trans)))
 502                         list_add_tail(&binding->binding_list, &nft_net->binding_list);
 503                 break;
 504         }
 505 }
 506
 507 static void nft_trans_commit_list_add_elem(struct net *net, struct nft_trans *trans,
 508                                            gfp_t gfp)
 509 {
 510         struct nftables_pernet *nft_net = nft_pernet(net);
 511
 512         WARN_ON_ONCE(trans->msg_type != NFT_MSG_NEWSETELEM &&
 513                      trans->msg_type != NFT_MSG_DELSETELEM);
 514
 515         might_alloc(gfp);
 516
 517         if (nft_trans_try_collapse(nft_net, trans, gfp)) {
 518                 kfree(trans);
 519                 return;
 520         }
 521
 522         nft_trans_commit_list_add_tail(net, trans);
 523 }
 524
 525 static int nft_trans_table_add(struct nft_ctx *ctx, int msg_type)
 526 {
 527         struct nft_trans *trans;
 528
 529         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_table));
 530         if (trans == NULL)
 531                 return -ENOMEM;
 532
 533         if (msg_type == NFT_MSG_NEWTABLE)
 534                 nft_activate_next(ctx->net, ctx->table);
 535
 536         nft_trans_commit_list_add_tail(ctx->net, trans);
 537         return 0;
 538 }
 539
 540 static int nft_deltable(struct nft_ctx *ctx)
 541 {
 542         int err;
 543
 544         err = nft_trans_table_add(ctx, NFT_MSG_DELTABLE);
 545         if (err < 0)
 546                 return err;
 547
 548         nft_deactivate_next(ctx->net, ctx->table);
 549         return err;
 550 }
 551
 552 static struct nft_trans *
 553 nft_trans_alloc_chain(const struct nft_ctx *ctx, int msg_type)
 554 {
 555         struct nft_trans_chain *trans_chain;
 556         struct nft_trans *trans;
 557
 558         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_chain));
 559         if (!trans)
 560                 return NULL;
 561
 562         trans_chain = nft_trans_container_chain(trans);
 563         INIT_LIST_HEAD(&trans_chain->nft_trans_binding.binding_list);
 564         trans_chain->chain = ctx->chain;
 565
 566         return trans;
 567 }
 568
 569 static struct nft_trans *nft_trans_chain_add(struct nft_ctx *ctx, int msg_type)
 570 {
 571         struct nft_trans *trans;
 572
 573         trans = nft_trans_alloc_chain(ctx, msg_type);
 574         if (trans == NULL)
 575                 return ERR_PTR(-ENOMEM);
 576
 577         if (msg_type == NFT_MSG_NEWCHAIN) {
 578                 nft_activate_next(ctx->net, ctx->chain);
 579
 580                 if (ctx->nla[NFTA_CHAIN_ID]) {
 581                         nft_trans_chain_id(trans) =
 582                                 ntohl(nla_get_be32(ctx->nla[NFTA_CHAIN_ID]));
 583                 }
 584         }
 585         nft_trans_commit_list_add_tail(ctx->net, trans);
 586
 587         return trans;
 588 }
 589
 590 static int nft_delchain(struct nft_ctx *ctx)
 591 {
 592         struct nft_trans *trans;
 593
 594         trans = nft_trans_chain_add(ctx, NFT_MSG_DELCHAIN);
 595         if (IS_ERR(trans))
 596                 return PTR_ERR(trans);
 597
 598         nft_use_dec(&ctx->table->use);
 599         nft_deactivate_next(ctx->net, ctx->chain);
 600
 601         return 0;
 602 }
 603
 604 void nft_rule_expr_activate(const struct nft_ctx *ctx, struct nft_rule *rule)
 605 {
 606         struct nft_expr *expr;
 607
 608         expr = nft_expr_first(rule);
 609         while (nft_expr_more(rule, expr)) {
 610                 if (expr->ops->activate)
 611                         expr->ops->activate(ctx, expr);
 612
 613                 expr = nft_expr_next(expr);
 614         }
 615 }
 616
 617 void nft_rule_expr_deactivate(const struct nft_ctx *ctx, struct nft_rule *rule,
 618                               enum nft_trans_phase phase)
 619 {
 620         struct nft_expr *expr;
 621
 622         expr = nft_expr_first(rule);
 623         while (nft_expr_more(rule, expr)) {
 624                 if (expr->ops->deactivate)
 625                         expr->ops->deactivate(ctx, expr, phase);
 626
 627                 expr = nft_expr_next(expr);
 628         }
 629 }
 630
 631 static int
 632 nf_tables_delrule_deactivate(struct nft_ctx *ctx, struct nft_rule *rule)
 633 {
 634         /* You cannot delete the same rule twice */
 635         if (nft_is_active_next(ctx->net, rule)) {
 636                 nft_deactivate_next(ctx->net, rule);
 637                 nft_use_dec(&ctx->chain->use);
 638                 return 0;
 639         }
 640         return -ENOENT;
 641 }
 642
 643 static struct nft_trans *nft_trans_rule_add(struct nft_ctx *ctx, int msg_type,
 644                                             struct nft_rule *rule)
 645 {
 646         struct nft_trans *trans;
 647
 648         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_rule));
 649         if (trans == NULL)
 650                 return NULL;
 651
 652         if (msg_type == NFT_MSG_NEWRULE && ctx->nla[NFTA_RULE_ID] != NULL) {
 653                 nft_trans_rule_id(trans) =
 654                         ntohl(nla_get_be32(ctx->nla[NFTA_RULE_ID]));
 655         }
 656         nft_trans_rule(trans) = rule;
 657         nft_trans_rule_chain(trans) = ctx->chain;
 658         nft_trans_commit_list_add_tail(ctx->net, trans);
 659
 660         return trans;
 661 }
 662
 663 static int nft_delrule(struct nft_ctx *ctx, struct nft_rule *rule)
 664 {
 665         struct nft_flow_rule *flow;
 666         struct nft_trans *trans;
 667         int err;
 668
 669         trans = nft_trans_rule_add(ctx, NFT_MSG_DELRULE, rule);
 670         if (trans == NULL)
 671                 return -ENOMEM;
 672
 673         if (ctx->chain->flags & NFT_CHAIN_HW_OFFLOAD) {
 674                 flow = nft_flow_rule_create(ctx->net, rule);
 675                 if (IS_ERR(flow)) {
 676                         nft_trans_destroy(trans);
 677                         return PTR_ERR(flow);
 678                 }
 679
 680                 nft_trans_flow_rule(trans) = flow;
 681         }
 682
 683         err = nf_tables_delrule_deactivate(ctx, rule);
 684         if (err < 0) {
 685                 nft_trans_destroy(trans);
 686                 return err;
 687         }
 688         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_PREPARE);
 689
 690         return 0;
 691 }
 692
 693 static int nft_delrule_by_chain(struct nft_ctx *ctx)
 694 {
 695         struct nft_rule *rule;
 696         int err;
 697
 698         list_for_each_entry(rule, &ctx->chain->rules, list) {
 699                 if (!nft_is_active_next(ctx->net, rule))
 700                         continue;
 701
 702                 err = nft_delrule(ctx, rule);
 703                 if (err < 0)
 704                         return err;
 705         }
 706         return 0;
 707 }
 708
 709 static int __nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
 710                                struct nft_set *set,
 711                                const struct nft_set_desc *desc)
 712 {
 713         struct nft_trans_set *trans_set;
 714         struct nft_trans *trans;
 715
 716         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_set));
 717         if (trans == NULL)
 718                 return -ENOMEM;
 719
 720         trans_set = nft_trans_container_set(trans);
 721         INIT_LIST_HEAD(&trans_set->nft_trans_binding.binding_list);
 722         INIT_LIST_HEAD(&trans_set->list_trans_newset);
 723
 724         if (msg_type == NFT_MSG_NEWSET && ctx->nla[NFTA_SET_ID] && !desc) {
 725                 nft_trans_set_id(trans) =
 726                         ntohl(nla_get_be32(ctx->nla[NFTA_SET_ID]));
 727                 nft_activate_next(ctx->net, set);
 728         }
 729         nft_trans_set(trans) = set;
 730         if (desc) {
 731                 nft_trans_set_update(trans) = true;
 732                 nft_trans_set_gc_int(trans) = desc->gc_int;
 733                 nft_trans_set_timeout(trans) = desc->timeout;
 734                 nft_trans_set_size(trans) = desc->size;
 735         }
 736         nft_trans_commit_list_add_tail(ctx->net, trans);
 737
 738         return 0;
 739 }
 740
 741 static int nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
 742                              struct nft_set *set)
 743 {
 744         return __nft_trans_set_add(ctx, msg_type, set, NULL);
 745 }
 746
 747 static int nft_mapelem_deactivate(const struct nft_ctx *ctx,
 748                                   struct nft_set *set,
 749                                   const struct nft_set_iter *iter,
 750                                   struct nft_elem_priv *elem_priv)
 751 {
 752         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
 753
 754         if (!nft_set_elem_active(ext, iter->genmask))
 755                 return 0;
 756
 757         nft_set_elem_change_active(ctx->net, set, ext);
 758         nft_setelem_data_deactivate(ctx->net, set, elem_priv);
 759
 760         return 0;
 761 }
 762
 763 struct nft_set_elem_catchall {
 764         struct list_head        list;
 765         struct rcu_head         rcu;
 766         struct nft_elem_priv    *elem;
 767 };
 768
 769 static void nft_map_catchall_deactivate(const struct nft_ctx *ctx,
 770                                         struct nft_set *set)
 771 {
 772         u8 genmask = nft_genmask_next(ctx->net);
 773         struct nft_set_elem_catchall *catchall;
 774         struct nft_set_ext *ext;
 775
 776         list_for_each_entry(catchall, &set->catchall_list, list) {
 777                 ext = nft_set_elem_ext(set, catchall->elem);
 778                 if (!nft_set_elem_active(ext, genmask))
 779                         continue;
 780
 781                 nft_set_elem_change_active(ctx->net, set, ext);
 782                 nft_setelem_data_deactivate(ctx->net, set, catchall->elem);
 783                 break;
 784         }
 785 }
 786
 787 static void nft_map_deactivate(const struct nft_ctx *ctx, struct nft_set *set)
 788 {
 789         struct nft_set_iter iter = {
 790                 .genmask        = nft_genmask_next(ctx->net),
 791                 .type           = NFT_ITER_UPDATE,
 792                 .fn             = nft_mapelem_deactivate,
 793         };
 794
 795         set->ops->walk(ctx, set, &iter);
 796         WARN_ON_ONCE(iter.err);
 797
 798         nft_map_catchall_deactivate(ctx, set);
 799 }
 800
 801 static int nft_delset(const struct nft_ctx *ctx, struct nft_set *set)
 802 {
 803         int err;
 804
 805         err = nft_trans_set_add(ctx, NFT_MSG_DELSET, set);
 806         if (err < 0)
 807                 return err;
 808
 809         if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
 810                 nft_map_deactivate(ctx, set);
 811
 812         nft_deactivate_next(ctx->net, set);
 813         nft_use_dec(&ctx->table->use);
 814
 815         return err;
 816 }
 817
 818 static int nft_trans_obj_add(struct nft_ctx *ctx, int msg_type,
 819                              struct nft_object *obj)
 820 {
 821         struct nft_trans *trans;
 822
 823         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_obj));
 824         if (trans == NULL)
 825                 return -ENOMEM;
 826
 827         if (msg_type == NFT_MSG_NEWOBJ)
 828                 nft_activate_next(ctx->net, obj);
 829
 830         nft_trans_obj(trans) = obj;
 831         nft_trans_commit_list_add_tail(ctx->net, trans);
 832
 833         return 0;
 834 }
 835
 836 static int nft_delobj(struct nft_ctx *ctx, struct nft_object *obj)
 837 {
 838         int err;
 839
 840         err = nft_trans_obj_add(ctx, NFT_MSG_DELOBJ, obj);
 841         if (err < 0)
 842                 return err;
 843
 844         nft_deactivate_next(ctx->net, obj);
 845         nft_use_dec(&ctx->table->use);
 846
 847         return err;
 848 }
 849
 850 static struct nft_trans *
 851 nft_trans_flowtable_add(struct nft_ctx *ctx, int msg_type,
 852                         struct nft_flowtable *flowtable)
 853 {
 854         struct nft_trans *trans;
 855
 856         trans = nft_trans_alloc(ctx, msg_type,
 857                                 sizeof(struct nft_trans_flowtable));
 858         if (trans == NULL)
 859                 return ERR_PTR(-ENOMEM);
 860
 861         if (msg_type == NFT_MSG_NEWFLOWTABLE)
 862                 nft_activate_next(ctx->net, flowtable);
 863
 864         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
 865         nft_trans_flowtable(trans) = flowtable;
 866         nft_trans_commit_list_add_tail(ctx->net, trans);
 867
 868         return trans;
 869 }
 870
 871 static int nft_delflowtable(struct nft_ctx *ctx,
 872                             struct nft_flowtable *flowtable)
 873 {
 874         struct nft_trans *trans;
 875
 876         trans = nft_trans_flowtable_add(ctx, NFT_MSG_DELFLOWTABLE, flowtable);
 877         if (IS_ERR(trans))
 878                 return PTR_ERR(trans);
 879
 880         nft_deactivate_next(ctx->net, flowtable);
 881         nft_use_dec(&ctx->table->use);
 882
 883         return 0;
 884 }
 885
 886 static void __nft_reg_track_clobber(struct nft_regs_track *track, u8 dreg)
 887 {
 888         int i;
 889
 890         for (i = track->regs[dreg].num_reg; i > 0; i--)
 891                 __nft_reg_track_cancel(track, dreg - i);
 892 }
 893
 894 static void __nft_reg_track_update(struct nft_regs_track *track,
 895                                    const struct nft_expr *expr,
 896                                    u8 dreg, u8 num_reg)
 897 {
 898         track->regs[dreg].selector = expr;
 899         track->regs[dreg].bitwise = NULL;
 900         track->regs[dreg].num_reg = num_reg;
 901 }
 902
 903 void nft_reg_track_update(struct nft_regs_track *track,
 904                           const struct nft_expr *expr, u8 dreg, u8 len)
 905 {
 906         unsigned int regcount;
 907         int i;
 908
 909         __nft_reg_track_clobber(track, dreg);
 910
 911         regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
 912         for (i = 0; i < regcount; i++, dreg++)
 913                 __nft_reg_track_update(track, expr, dreg, i);
 914 }
 915 EXPORT_SYMBOL_GPL(nft_reg_track_update);
 916
 917 void nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg, u8 len)
 918 {
 919         unsigned int regcount;
 920         int i;
 921
 922         __nft_reg_track_clobber(track, dreg);
 923
 924         regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
 925         for (i = 0; i < regcount; i++, dreg++)
 926                 __nft_reg_track_cancel(track, dreg);
 927 }
 928 EXPORT_SYMBOL_GPL(nft_reg_track_cancel);
 929
 930 void __nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg)
 931 {
 932         track->regs[dreg].selector = NULL;
 933         track->regs[dreg].bitwise = NULL;
 934         track->regs[dreg].num_reg = 0;
 935 }
 936 EXPORT_SYMBOL_GPL(__nft_reg_track_cancel);
 937
 938 /*
 939  * Tables
 940  */
 941
 942 static struct nft_table *nft_table_lookup(const struct net *net,
 943                                           const struct nlattr *nla,
 944                                           u8 family, u8 genmask, u32 nlpid)
 945 {
 946         struct nftables_pernet *nft_net;
 947         struct nft_table *table;
 948
 949         if (nla == NULL)
 950                 return ERR_PTR(-EINVAL);
 951
 952         nft_net = nft_pernet(net);
 953         list_for_each_entry_rcu(table, &nft_net->tables, list,
 954                                 lockdep_is_held(&nft_net->commit_mutex)) {
 955                 if (!nla_strcmp(nla, table->name) &&
 956                     table->family == family &&
 957                     nft_active_genmask(table, genmask)) {
 958                         if (nft_table_has_owner(table) &&
 959                             nlpid && table->nlpid != nlpid)
 960                                 return ERR_PTR(-EPERM);
 961
 962                         return table;
 963                 }
 964         }
 965
 966         return ERR_PTR(-ENOENT);
 967 }
 968
 969 static struct nft_table *nft_table_lookup_byhandle(const struct net *net,
 970                                                    const struct nlattr *nla,
 971                                                    int family, u8 genmask, u32 nlpid)
 972 {
 973         struct nftables_pernet *nft_net;
 974         struct nft_table *table;
 975
 976         nft_net = nft_pernet(net);
 977         list_for_each_entry(table, &nft_net->tables, list) {
 978                 if (be64_to_cpu(nla_get_be64(nla)) == table->handle &&
 979                     table->family == family &&
 980                     nft_active_genmask(table, genmask)) {
 981                         if (nft_table_has_owner(table) &&
 982                             nlpid && table->nlpid != nlpid)
 983                                 return ERR_PTR(-EPERM);
 984
 985                         return table;
 986                 }
 987         }
 988
 989         return ERR_PTR(-ENOENT);
 990 }
 991
 992 static inline u64 nf_tables_alloc_handle(struct nft_table *table)
 993 {
 994         return ++table->hgenerator;
 995 }
 996
 997 static const struct nft_chain_type *chain_type[NFPROTO_NUMPROTO][NFT_CHAIN_T_MAX];
 998
 999 static const struct nft_chain_type *
1000 __nft_chain_type_get(u8 family, enum nft_chain_types type)
1001 {
1002         if (family >= NFPROTO_NUMPROTO ||
1003             type >= NFT_CHAIN_T_MAX)
1004                 return NULL;
1005
1006         return chain_type[family][type];
1007 }
1008
1009 static const struct nft_chain_type *
1010 __nf_tables_chain_type_lookup(const struct nlattr *nla, u8 family)
1011 {
1012         const struct nft_chain_type *type;
1013         int i;
1014
1015         for (i = 0; i < NFT_CHAIN_T_MAX; i++) {
1016                 type = __nft_chain_type_get(family, i);
1017                 if (!type)
1018                         continue;
1019                 if (!nla_strcmp(nla, type->name))
1020                         return type;
1021         }
1022         return NULL;
1023 }
1024
1025 struct nft_module_request {
1026         struct list_head        list;
1027         char                    module[MODULE_NAME_LEN];
1028         bool                    done;
1029 };
1030
1031 #ifdef CONFIG_MODULES
1032 __printf(2, 3) int nft_request_module(struct net *net, const char *fmt,
1033                                       ...)
1034 {
1035         char module_name[MODULE_NAME_LEN];
1036         struct nftables_pernet *nft_net;
1037         struct nft_module_request *req;
1038         va_list args;
1039         int ret;
1040
1041         va_start(args, fmt);
1042         ret = vsnprintf(module_name, MODULE_NAME_LEN, fmt, args);
1043         va_end(args);
1044         if (ret >= MODULE_NAME_LEN)
1045                 return 0;
1046
1047         nft_net = nft_pernet(net);
1048         list_for_each_entry(req, &nft_net->module_list, list) {
1049                 if (!strcmp(req->module, module_name)) {
1050                         if (req->done)
1051                                 return 0;
1052
1053                         /* A request to load this module already exists. */
1054                         return -EAGAIN;
1055                 }
1056         }
1057
1058         req = kmalloc(sizeof(*req), GFP_KERNEL);
1059         if (!req)
1060                 return -ENOMEM;
1061
1062         req->done = false;
1063         strscpy(req->module, module_name, MODULE_NAME_LEN);
1064         list_add_tail(&req->list, &nft_net->module_list);
1065
1066         return -EAGAIN;
1067 }
1068 EXPORT_SYMBOL_GPL(nft_request_module);
1069 #endif
1070
1071 static void lockdep_nfnl_nft_mutex_not_held(void)
1072 {
1073 #ifdef CONFIG_PROVE_LOCKING
1074         if (debug_locks)
1075                 WARN_ON_ONCE(lockdep_nfnl_is_held(NFNL_SUBSYS_NFTABLES));
1076 #endif
1077 }
1078
1079 static const struct nft_chain_type *
1080 nf_tables_chain_type_lookup(struct net *net, const struct nlattr *nla,
1081                             u8 family, bool autoload)
1082 {
1083         const struct nft_chain_type *type;
1084
1085         type = __nf_tables_chain_type_lookup(nla, family);
1086         if (type != NULL)
1087                 return type;
1088
1089         lockdep_nfnl_nft_mutex_not_held();
1090 #ifdef CONFIG_MODULES
1091         if (autoload) {
1092                 if (nft_request_module(net, "nft-chain-%u-%.*s", family,
1093                                        nla_len(nla),
1094                                        (const char *)nla_data(nla)) == -EAGAIN)
1095                         return ERR_PTR(-EAGAIN);
1096         }
1097 #endif
1098         return ERR_PTR(-ENOENT);
1099 }
1100
1101 static __be16 nft_base_seq(const struct net *net)
1102 {
1103         struct nftables_pernet *nft_net = nft_pernet(net);
1104
1105         return htons(nft_net->base_seq & 0xffff);
1106 }
1107
1108 static const struct nla_policy nft_table_policy[NFTA_TABLE_MAX + 1] = {
1109         [NFTA_TABLE_NAME]       = { .type = NLA_STRING,
1110                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
1111         [NFTA_TABLE_FLAGS]      = { .type = NLA_U32 },
1112         [NFTA_TABLE_HANDLE]     = { .type = NLA_U64 },
1113         [NFTA_TABLE_USERDATA]   = { .type = NLA_BINARY,
1114                                     .len = NFT_USERDATA_MAXLEN }
1115 };
1116
1117 static int nf_tables_fill_table_info(struct sk_buff *skb, struct net *net,
1118                                      u32 portid, u32 seq, int event, u32 flags,
1119                                      int family, const struct nft_table *table)
1120 {
1121         struct nlmsghdr *nlh;
1122
1123         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
1124         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
1125                            NFNETLINK_V0, nft_base_seq(net));
1126         if (!nlh)
1127                 goto nla_put_failure;
1128
1129         if (nla_put_string(skb, NFTA_TABLE_NAME, table->name) ||
1130             nla_put_be32(skb, NFTA_TABLE_USE, htonl(table->use)) ||
1131             nla_put_be64(skb, NFTA_TABLE_HANDLE, cpu_to_be64(table->handle),
1132                          NFTA_TABLE_PAD))
1133                 goto nla_put_failure;
1134
1135         if (event == NFT_MSG_DELTABLE) {
1136                 nlmsg_end(skb, nlh);
1137                 return 0;
1138         }
1139
1140         if (nla_put_be32(skb, NFTA_TABLE_FLAGS,
1141                          htonl(table->flags & NFT_TABLE_F_MASK)))
1142                 goto nla_put_failure;
1143
1144         if (nft_table_has_owner(table) &&
1145             nla_put_be32(skb, NFTA_TABLE_OWNER, htonl(table->nlpid)))
1146                 goto nla_put_failure;
1147
1148         if (table->udata) {
1149                 if (nla_put(skb, NFTA_TABLE_USERDATA, table->udlen, table->udata))
1150                         goto nla_put_failure;
1151         }
1152
1153         nlmsg_end(skb, nlh);
1154         return 0;
1155
1156 nla_put_failure:
1157         nlmsg_trim(skb, nlh);
1158         return -1;
1159 }
1160
1161 struct nftnl_skb_parms {
1162         bool report;
1163 };
1164 #define NFT_CB(skb)     (*(struct nftnl_skb_parms*)&((skb)->cb))
1165
1166 static void nft_notify_enqueue(struct sk_buff *skb, bool report,
1167                                struct list_head *notify_list)
1168 {
1169         NFT_CB(skb).report = report;
1170         list_add_tail(&skb->list, notify_list);
1171 }
1172
1173 static void nf_tables_table_notify(const struct nft_ctx *ctx, int event)
1174 {
1175         struct nftables_pernet *nft_net;
1176         struct sk_buff *skb;
1177         u16 flags = 0;
1178         int err;
1179
1180         if (!ctx->report &&
1181             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
1182                 return;
1183
1184         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
1185         if (skb == NULL)
1186                 goto err;
1187
1188         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
1189                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
1190
1191         err = nf_tables_fill_table_info(skb, ctx->net, ctx->portid, ctx->seq,
1192                                         event, flags, ctx->family, ctx->table);
1193         if (err < 0) {
1194                 kfree_skb(skb);
1195                 goto err;
1196         }
1197
1198         nft_net = nft_pernet(ctx->net);
1199         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
1200         return;
1201 err:
1202         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
1203 }
1204
1205 static int nf_tables_dump_tables(struct sk_buff *skb,
1206                                  struct netlink_callback *cb)
1207 {
1208         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
1209         struct nftables_pernet *nft_net;
1210         const struct nft_table *table;
1211         unsigned int idx = 0, s_idx = cb->args[0];
1212         struct net *net = sock_net(skb->sk);
1213         int family = nfmsg->nfgen_family;
1214
1215         rcu_read_lock();
1216         nft_net = nft_pernet(net);
1217         cb->seq = READ_ONCE(nft_net->base_seq);
1218
1219         list_for_each_entry_rcu(table, &nft_net->tables, list) {
1220                 if (family != NFPROTO_UNSPEC && family != table->family)
1221                         continue;
1222
1223                 if (idx < s_idx)
1224                         goto cont;
1225                 if (idx > s_idx)
1226                         memset(&cb->args[1], 0,
1227                                sizeof(cb->args) - sizeof(cb->args[0]));
1228                 if (!nft_is_active(net, table))
1229                         continue;
1230                 if (nf_tables_fill_table_info(skb, net,
1231                                               NETLINK_CB(cb->skb).portid,
1232                                               cb->nlh->nlmsg_seq,
1233                                               NFT_MSG_NEWTABLE, NLM_F_MULTI,
1234                                               table->family, table) < 0)
1235                         goto done;
1236
1237                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
1238 cont:
1239                 idx++;
1240         }
1241 done:
1242         rcu_read_unlock();
1243         cb->args[0] = idx;
1244         return skb->len;
1245 }
1246
1247 static int nft_netlink_dump_start_rcu(struct sock *nlsk, struct sk_buff *skb,
1248                                       const struct nlmsghdr *nlh,
1249                                       struct netlink_dump_control *c)
1250 {
1251         int err;
1252
1253         if (!try_module_get(THIS_MODULE))
1254                 return -EINVAL;
1255
1256         rcu_read_unlock();
1257         err = netlink_dump_start(nlsk, skb, nlh, c);
1258         rcu_read_lock();
1259         module_put(THIS_MODULE);
1260
1261         return err;
1262 }
1263
1264 /* called with rcu_read_lock held */
1265 static int nf_tables_gettable(struct sk_buff *skb, const struct nfnl_info *info,
1266                               const struct nlattr * const nla[])
1267 {
1268         struct netlink_ext_ack *extack = info->extack;
1269         u8 genmask = nft_genmask_cur(info->net);
1270         u8 family = info->nfmsg->nfgen_family;
1271         const struct nft_table *table;
1272         struct net *net = info->net;
1273         struct sk_buff *skb2;
1274         int err;
1275
1276         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
1277                 struct netlink_dump_control c = {
1278                         .dump = nf_tables_dump_tables,
1279                         .module = THIS_MODULE,
1280                 };
1281
1282                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
1283         }
1284
1285         table = nft_table_lookup(net, nla[NFTA_TABLE_NAME], family, genmask, 0);
1286         if (IS_ERR(table)) {
1287                 NL_SET_BAD_ATTR(extack, nla[NFTA_TABLE_NAME]);
1288                 return PTR_ERR(table);
1289         }
1290
1291         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
1292         if (!skb2)
1293                 return -ENOMEM;
1294
1295         err = nf_tables_fill_table_info(skb2, net, NETLINK_CB(skb).portid,
1296                                         info->nlh->nlmsg_seq, NFT_MSG_NEWTABLE,
1297                                         0, family, table);
1298         if (err < 0)
1299                 goto err_fill_table_info;
1300
1301         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
1302
1303 err_fill_table_info:
1304         kfree_skb(skb2);
1305         return err;
1306 }
1307
1308 static void nft_table_disable(struct net *net, struct nft_table *table, u32 cnt)
1309 {
1310         struct nft_chain *chain;
1311         u32 i = 0;
1312
1313         list_for_each_entry(chain, &table->chains, list) {
1314                 if (!nft_is_active_next(net, chain))
1315                         continue;
1316                 if (!nft_is_base_chain(chain))
1317                         continue;
1318
1319                 if (cnt && i++ == cnt)
1320                         break;
1321
1322                 nf_tables_unregister_hook(net, table, chain);
1323         }
1324 }
1325
1326 static int nf_tables_table_enable(struct net *net, struct nft_table *table)
1327 {
1328         struct nft_chain *chain;
1329         int err, i = 0;
1330
1331         list_for_each_entry(chain, &table->chains, list) {
1332                 if (!nft_is_active_next(net, chain))
1333                         continue;
1334                 if (!nft_is_base_chain(chain))
1335                         continue;
1336
1337                 err = nf_tables_register_hook(net, table, chain);
1338                 if (err < 0)
1339                         goto err_register_hooks;
1340
1341                 i++;
1342         }
1343         return 0;
1344
1345 err_register_hooks:
1346         if (i)
1347                 nft_table_disable(net, table, i);
1348         return err;
1349 }
1350
1351 static void nf_tables_table_disable(struct net *net, struct nft_table *table)
1352 {
1353         table->flags &= ~NFT_TABLE_F_DORMANT;
1354         nft_table_disable(net, table, 0);
1355         table->flags |= NFT_TABLE_F_DORMANT;
1356 }
1357
1358 #define __NFT_TABLE_F_INTERNAL          (NFT_TABLE_F_MASK + 1)
1359 #define __NFT_TABLE_F_WAS_DORMANT       (__NFT_TABLE_F_INTERNAL << 0)
1360 #define __NFT_TABLE_F_WAS_AWAKEN        (__NFT_TABLE_F_INTERNAL << 1)
1361 #define __NFT_TABLE_F_WAS_ORPHAN        (__NFT_TABLE_F_INTERNAL << 2)
1362 #define __NFT_TABLE_F_UPDATE            (__NFT_TABLE_F_WAS_DORMANT | \
1363                                          __NFT_TABLE_F_WAS_AWAKEN | \
1364                                          __NFT_TABLE_F_WAS_ORPHAN)
1365
1366 static bool nft_table_pending_update(const struct nft_ctx *ctx)
1367 {
1368         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
1369         struct nft_trans *trans;
1370
1371         if (ctx->table->flags & __NFT_TABLE_F_UPDATE)
1372                 return true;
1373
1374         list_for_each_entry(trans, &nft_net->commit_list, list) {
1375                 if (trans->table == ctx->table &&
1376                     ((trans->msg_type == NFT_MSG_NEWCHAIN &&
1377                       nft_trans_chain_update(trans)) ||
1378                      (trans->msg_type == NFT_MSG_DELCHAIN &&
1379                       nft_is_base_chain(nft_trans_chain(trans)))))
1380                         return true;
1381         }
1382
1383         return false;
1384 }
1385
1386 static int nf_tables_updtable(struct nft_ctx *ctx)
1387 {
1388         struct nft_trans *trans;
1389         u32 flags;
1390         int ret;
1391
1392         if (!ctx->nla[NFTA_TABLE_FLAGS])
1393                 return 0;
1394
1395         flags = ntohl(nla_get_be32(ctx->nla[NFTA_TABLE_FLAGS]));
1396         if (flags & ~NFT_TABLE_F_MASK)
1397                 return -EOPNOTSUPP;
1398
1399         if (flags == (ctx->table->flags & NFT_TABLE_F_MASK))
1400                 return 0;
1401
1402         if ((nft_table_has_owner(ctx->table) &&
1403              !(flags & NFT_TABLE_F_OWNER)) ||
1404             (flags & NFT_TABLE_F_OWNER &&
1405              !nft_table_is_orphan(ctx->table)))
1406                 return -EOPNOTSUPP;
1407
1408         if ((flags ^ ctx->table->flags) & NFT_TABLE_F_PERSIST)
1409                 return -EOPNOTSUPP;
1410
1411         /* No dormant off/on/off/on games in single transaction */
1412         if (nft_table_pending_update(ctx))
1413                 return -EINVAL;
1414
1415         trans = nft_trans_alloc(ctx, NFT_MSG_NEWTABLE,
1416                                 sizeof(struct nft_trans_table));
1417         if (trans == NULL)
1418                 return -ENOMEM;
1419
1420         if ((flags & NFT_TABLE_F_DORMANT) &&
1421             !(ctx->table->flags & NFT_TABLE_F_DORMANT)) {
1422                 ctx->table->flags |= NFT_TABLE_F_DORMANT;
1423                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE))
1424                         ctx->table->flags |= __NFT_TABLE_F_WAS_AWAKEN;
1425         } else if (!(flags & NFT_TABLE_F_DORMANT) &&
1426                    ctx->table->flags & NFT_TABLE_F_DORMANT) {
1427                 ctx->table->flags &= ~NFT_TABLE_F_DORMANT;
1428                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE)) {
1429                         ret = nf_tables_table_enable(ctx->net, ctx->table);
1430                         if (ret < 0)
1431                                 goto err_register_hooks;
1432
1433                         ctx->table->flags |= __NFT_TABLE_F_WAS_DORMANT;
1434                 }
1435         }
1436
1437         if ((flags & NFT_TABLE_F_OWNER) &&
1438             !nft_table_has_owner(ctx->table)) {
1439                 ctx->table->nlpid = ctx->portid;
1440                 ctx->table->flags |= NFT_TABLE_F_OWNER |
1441                                      __NFT_TABLE_F_WAS_ORPHAN;
1442         }
1443
1444         nft_trans_table_update(trans) = true;
1445         nft_trans_commit_list_add_tail(ctx->net, trans);
1446
1447         return 0;
1448
1449 err_register_hooks:
1450         ctx->table->flags |= NFT_TABLE_F_DORMANT;
1451         nft_trans_destroy(trans);
1452         return ret;
1453 }
1454
1455 static u32 nft_chain_hash(const void *data, u32 len, u32 seed)
1456 {
1457         const char *name = data;
1458
1459         return jhash(name, strlen(name), seed);
1460 }
1461
1462 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed)
1463 {
1464         const struct nft_chain *chain = data;
1465
1466         return nft_chain_hash(chain->name, 0, seed);
1467 }
1468
1469 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *arg,
1470                               const void *ptr)
1471 {
1472         const struct nft_chain *chain = ptr;
1473         const char *name = arg->key;
1474
1475         return strcmp(chain->name, name);
1476 }
1477
1478 static u32 nft_objname_hash(const void *data, u32 len, u32 seed)
1479 {
1480         const struct nft_object_hash_key *k = data;
1481
1482         seed ^= hash_ptr(k->table, 32);
1483
1484         return jhash(k->name, strlen(k->name), seed);
1485 }
1486
1487 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed)
1488 {
1489         const struct nft_object *obj = data;
1490
1491         return nft_objname_hash(&obj->key, 0, seed);
1492 }
1493
1494 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *arg,
1495                                 const void *ptr)
1496 {
1497         const struct nft_object_hash_key *k = arg->key;
1498         const struct nft_object *obj = ptr;
1499
1500         if (obj->key.table != k->table)
1501                 return -1;
1502
1503         return strcmp(obj->key.name, k->name);
1504 }
1505
1506 static bool nft_supported_family(u8 family)
1507 {
1508         return false
1509 #ifdef CONFIG_NF_TABLES_INET
1510                 || family == NFPROTO_INET
1511 #endif
1512 #ifdef CONFIG_NF_TABLES_IPV4
1513                 || family == NFPROTO_IPV4
1514 #endif
1515 #ifdef CONFIG_NF_TABLES_ARP
1516                 || family == NFPROTO_ARP
1517 #endif
1518 #ifdef CONFIG_NF_TABLES_NETDEV
1519                 || family == NFPROTO_NETDEV
1520 #endif
1521 #if IS_ENABLED(CONFIG_NF_TABLES_BRIDGE)
1522                 || family == NFPROTO_BRIDGE
1523 #endif
1524 #ifdef CONFIG_NF_TABLES_IPV6
1525                 || family == NFPROTO_IPV6
1526 #endif
1527                 ;
1528 }
1529
1530 static int nf_tables_newtable(struct sk_buff *skb, const struct nfnl_info *info,
1531                               const struct nlattr * const nla[])
1532 {
1533         struct nftables_pernet *nft_net = nft_pernet(info->net);
1534         struct netlink_ext_ack *extack = info->extack;
1535         u8 genmask = nft_genmask_next(info->net);
1536         u8 family = info->nfmsg->nfgen_family;
1537         struct net *net = info->net;
1538         const struct nlattr *attr;
1539         struct nft_table *table;
1540         struct nft_ctx ctx;
1541         u32 flags = 0;
1542         int err;
1543
1544         if (!nft_supported_family(family))
1545                 return -EOPNOTSUPP;
1546
1547         lockdep_assert_held(&nft_net->commit_mutex);
1548         attr = nla[NFTA_TABLE_NAME];
1549         table = nft_table_lookup(net, attr, family, genmask,
1550                                  NETLINK_CB(skb).portid);
1551         if (IS_ERR(table)) {
1552                 if (PTR_ERR(table) != -ENOENT)
1553                         return PTR_ERR(table);
1554         } else {
1555                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
1556                         NL_SET_BAD_ATTR(extack, attr);
1557                         return -EEXIST;
1558                 }
1559                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
1560                         return -EOPNOTSUPP;
1561
1562                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1563
1564                 return nf_tables_updtable(&ctx);
1565         }
1566
1567         if (nla[NFTA_TABLE_FLAGS]) {
1568                 flags = ntohl(nla_get_be32(nla[NFTA_TABLE_FLAGS]));
1569                 if (flags & ~NFT_TABLE_F_MASK)
1570                         return -EOPNOTSUPP;
1571         }
1572
1573         err = -ENOMEM;
1574         table = kzalloc(sizeof(*table), GFP_KERNEL_ACCOUNT);
1575         if (table == NULL)
1576                 goto err_kzalloc;
1577
1578         table->validate_state = nft_net->validate_state;
1579         table->name = nla_strdup(attr, GFP_KERNEL_ACCOUNT);
1580         if (table->name == NULL)
1581                 goto err_strdup;
1582
1583         if (nla[NFTA_TABLE_USERDATA]) {
1584                 table->udata = nla_memdup(nla[NFTA_TABLE_USERDATA], GFP_KERNEL_ACCOUNT);
1585                 if (table->udata == NULL)
1586                         goto err_table_udata;
1587
1588                 table->udlen = nla_len(nla[NFTA_TABLE_USERDATA]);
1589         }
1590
1591         err = rhltable_init(&table->chains_ht, &nft_chain_ht_params);
1592         if (err)
1593                 goto err_chain_ht;
1594
1595         INIT_LIST_HEAD(&table->chains);
1596         INIT_LIST_HEAD(&table->sets);
1597         INIT_LIST_HEAD(&table->objects);
1598         INIT_LIST_HEAD(&table->flowtables);
1599         table->family = family;
1600         table->flags = flags;
1601         table->handle = ++nft_net->table_handle;
1602         if (table->flags & NFT_TABLE_F_OWNER)
1603                 table->nlpid = NETLINK_CB(skb).portid;
1604
1605         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1606         err = nft_trans_table_add(&ctx, NFT_MSG_NEWTABLE);
1607         if (err < 0)
1608                 goto err_trans;
1609
1610         list_add_tail_rcu(&table->list, &nft_net->tables);
1611         return 0;
1612 err_trans:
1613         rhltable_destroy(&table->chains_ht);
1614 err_chain_ht:
1615         kfree(table->udata);
1616 err_table_udata:
1617         kfree(table->name);
1618 err_strdup:
1619         kfree(table);
1620 err_kzalloc:
1621         return err;
1622 }
1623
1624 static int nft_flush_table(struct nft_ctx *ctx)
1625 {
1626         struct nft_flowtable *flowtable, *nft;
1627         struct nft_chain *chain, *nc;
1628         struct nft_object *obj, *ne;
1629         struct nft_set *set, *ns;
1630         int err;
1631
1632         list_for_each_entry(chain, &ctx->table->chains, list) {
1633                 if (!nft_is_active_next(ctx->net, chain))
1634                         continue;
1635
1636                 if (nft_chain_binding(chain))
1637                         continue;
1638
1639                 ctx->chain = chain;
1640
1641                 err = nft_delrule_by_chain(ctx);
1642                 if (err < 0)
1643                         goto out;
1644         }
1645
1646         list_for_each_entry_safe(set, ns, &ctx->table->sets, list) {
1647                 if (!nft_is_active_next(ctx->net, set))
1648                         continue;
1649
1650                 if (nft_set_is_anonymous(set))
1651                         continue;
1652
1653                 err = nft_delset(ctx, set);
1654                 if (err < 0)
1655                         goto out;
1656         }
1657
1658         list_for_each_entry_safe(flowtable, nft, &ctx->table->flowtables, list) {
1659                 if (!nft_is_active_next(ctx->net, flowtable))
1660                         continue;
1661
1662                 err = nft_delflowtable(ctx, flowtable);
1663                 if (err < 0)
1664                         goto out;
1665         }
1666
1667         list_for_each_entry_safe(obj, ne, &ctx->table->objects, list) {
1668                 if (!nft_is_active_next(ctx->net, obj))
1669                         continue;
1670
1671                 err = nft_delobj(ctx, obj);
1672                 if (err < 0)
1673                         goto out;
1674         }
1675
1676         list_for_each_entry_safe(chain, nc, &ctx->table->chains, list) {
1677                 if (!nft_is_active_next(ctx->net, chain))
1678                         continue;
1679
1680                 if (nft_chain_binding(chain))
1681                         continue;
1682
1683                 ctx->chain = chain;
1684
1685                 err = nft_delchain(ctx);
1686                 if (err < 0)
1687                         goto out;
1688         }
1689
1690         err = nft_deltable(ctx);
1691 out:
1692         return err;
1693 }
1694
1695 static int nft_flush(struct nft_ctx *ctx, int family)
1696 {
1697         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
1698         const struct nlattr * const *nla = ctx->nla;
1699         struct nft_table *table, *nt;
1700         int err = 0;
1701
1702         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
1703                 if (family != AF_UNSPEC && table->family != family)
1704                         continue;
1705
1706                 ctx->family = table->family;
1707
1708                 if (!nft_is_active_next(ctx->net, table))
1709                         continue;
1710
1711                 if (nft_table_has_owner(table) && table->nlpid != ctx->portid)
1712                         continue;
1713
1714                 if (nla[NFTA_TABLE_NAME] &&
1715                     nla_strcmp(nla[NFTA_TABLE_NAME], table->name) != 0)
1716                         continue;
1717
1718                 ctx->table = table;
1719
1720                 err = nft_flush_table(ctx);
1721                 if (err < 0)
1722                         goto out;
1723         }
1724 out:
1725         return err;
1726 }
1727
1728 static int nf_tables_deltable(struct sk_buff *skb, const struct nfnl_info *info,
1729                               const struct nlattr * const nla[])
1730 {
1731         struct netlink_ext_ack *extack = info->extack;
1732         u8 genmask = nft_genmask_next(info->net);
1733         u8 family = info->nfmsg->nfgen_family;
1734         struct net *net = info->net;
1735         const struct nlattr *attr;
1736         struct nft_table *table;
1737         struct nft_ctx ctx;
1738
1739         nft_ctx_init(&ctx, net, skb, info->nlh, 0, NULL, NULL, nla);
1740         if (family == AF_UNSPEC ||
1741             (!nla[NFTA_TABLE_NAME] && !nla[NFTA_TABLE_HANDLE]))
1742                 return nft_flush(&ctx, family);
1743
1744         if (nla[NFTA_TABLE_HANDLE]) {
1745                 attr = nla[NFTA_TABLE_HANDLE];
1746                 table = nft_table_lookup_byhandle(net, attr, family, genmask,
1747                                                   NETLINK_CB(skb).portid);
1748         } else {
1749                 attr = nla[NFTA_TABLE_NAME];
1750                 table = nft_table_lookup(net, attr, family, genmask,
1751                                          NETLINK_CB(skb).portid);
1752         }
1753
1754         if (IS_ERR(table)) {
1755                 if (PTR_ERR(table) == -ENOENT &&
1756                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYTABLE)
1757                         return 0;
1758
1759                 NL_SET_BAD_ATTR(extack, attr);
1760                 return PTR_ERR(table);
1761         }
1762
1763         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
1764             table->use > 0)
1765                 return -EBUSY;
1766
1767         ctx.family = family;
1768         ctx.table = table;
1769
1770         return nft_flush_table(&ctx);
1771 }
1772
1773 static void nf_tables_table_destroy(struct nft_table *table)
1774 {
1775         if (WARN_ON(table->use > 0))
1776                 return;
1777
1778         rhltable_destroy(&table->chains_ht);
1779         kfree(table->name);
1780         kfree(table->udata);
1781         kfree(table);
1782 }
1783
1784 void nft_register_chain_type(const struct nft_chain_type *ctype)
1785 {
1786         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1787         if (WARN_ON(__nft_chain_type_get(ctype->family, ctype->type))) {
1788                 nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1789                 return;
1790         }
1791         chain_type[ctype->family][ctype->type] = ctype;
1792         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1793 }
1794 EXPORT_SYMBOL_GPL(nft_register_chain_type);
1795
1796 void nft_unregister_chain_type(const struct nft_chain_type *ctype)
1797 {
1798         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1799         chain_type[ctype->family][ctype->type] = NULL;
1800         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1801 }
1802 EXPORT_SYMBOL_GPL(nft_unregister_chain_type);
1803
1804 /*
1805  * Chains
1806  */
1807
1808 static struct nft_chain *
1809 nft_chain_lookup_byhandle(const struct nft_table *table, u64 handle, u8 genmask)
1810 {
1811         struct nft_chain *chain;
1812
1813         list_for_each_entry(chain, &table->chains, list) {
1814                 if (chain->handle == handle &&
1815                     nft_active_genmask(chain, genmask))
1816                         return chain;
1817         }
1818
1819         return ERR_PTR(-ENOENT);
1820 }
1821
1822 static bool lockdep_commit_lock_is_held(const struct net *net)
1823 {
1824 #ifdef CONFIG_PROVE_LOCKING
1825         struct nftables_pernet *nft_net = nft_pernet(net);
1826
1827         return lockdep_is_held(&nft_net->commit_mutex);
1828 #else
1829         return true;
1830 #endif
1831 }
1832
1833 static struct nft_chain *nft_chain_lookup(struct net *net,
1834                                           struct nft_table *table,
1835                                           const struct nlattr *nla, u8 genmask)
1836 {
1837         char search[NFT_CHAIN_MAXNAMELEN + 1];
1838         struct rhlist_head *tmp, *list;
1839         struct nft_chain *chain;
1840
1841         if (nla == NULL)
1842                 return ERR_PTR(-EINVAL);
1843
1844         nla_strscpy(search, nla, sizeof(search));
1845
1846         WARN_ON(!rcu_read_lock_held() &&
1847                 !lockdep_commit_lock_is_held(net));
1848
1849         chain = ERR_PTR(-ENOENT);
1850         rcu_read_lock();
1851         list = rhltable_lookup(&table->chains_ht, search, nft_chain_ht_params);
1852         if (!list)
1853                 goto out_unlock;
1854
1855         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
1856                 if (nft_active_genmask(chain, genmask))
1857                         goto out_unlock;
1858         }
1859         chain = ERR_PTR(-ENOENT);
1860 out_unlock:
1861         rcu_read_unlock();
1862         return chain;
1863 }
1864
1865 static const struct nla_policy nft_chain_policy[NFTA_CHAIN_MAX + 1] = {
1866         [NFTA_CHAIN_TABLE]      = { .type = NLA_STRING,
1867                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
1868         [NFTA_CHAIN_HANDLE]     = { .type = NLA_U64 },
1869         [NFTA_CHAIN_NAME]       = { .type = NLA_STRING,
1870                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
1871         [NFTA_CHAIN_HOOK]       = { .type = NLA_NESTED },
1872         [NFTA_CHAIN_POLICY]     = { .type = NLA_U32 },
1873         [NFTA_CHAIN_TYPE]       = { .type = NLA_STRING,
1874                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
1875         [NFTA_CHAIN_COUNTERS]   = { .type = NLA_NESTED },
1876         [NFTA_CHAIN_FLAGS]      = { .type = NLA_U32 },
1877         [NFTA_CHAIN_ID]         = { .type = NLA_U32 },
1878         [NFTA_CHAIN_USERDATA]   = { .type = NLA_BINARY,
1879                                     .len = NFT_USERDATA_MAXLEN },
1880 };
1881
1882 static const struct nla_policy nft_hook_policy[NFTA_HOOK_MAX + 1] = {
1883         [NFTA_HOOK_HOOKNUM]     = { .type = NLA_U32 },
1884         [NFTA_HOOK_PRIORITY]    = { .type = NLA_U32 },
1885         [NFTA_HOOK_DEV]         = { .type = NLA_STRING,
1886                                     .len = IFNAMSIZ - 1 },
1887 };
1888
1889 static int nft_dump_stats(struct sk_buff *skb, struct nft_stats __percpu *stats)
1890 {
1891         struct nft_stats *cpu_stats, total;
1892         struct nlattr *nest;
1893         unsigned int seq;
1894         u64 pkts, bytes;
1895         int cpu;
1896
1897         if (!stats)
1898                 return 0;
1899
1900         memset(&total, 0, sizeof(total));
1901         for_each_possible_cpu(cpu) {
1902                 cpu_stats = per_cpu_ptr(stats, cpu);
1903                 do {
1904                         seq = u64_stats_fetch_begin(&cpu_stats->syncp);
1905                         pkts = cpu_stats->pkts;
1906                         bytes = cpu_stats->bytes;
1907                 } while (u64_stats_fetch_retry(&cpu_stats->syncp, seq));
1908                 total.pkts += pkts;
1909                 total.bytes += bytes;
1910         }
1911         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_COUNTERS);
1912         if (nest == NULL)
1913                 goto nla_put_failure;
1914
1915         if (nla_put_be64(skb, NFTA_COUNTER_PACKETS, cpu_to_be64(total.pkts),
1916                          NFTA_COUNTER_PAD) ||
1917             nla_put_be64(skb, NFTA_COUNTER_BYTES, cpu_to_be64(total.bytes),
1918                          NFTA_COUNTER_PAD))
1919                 goto nla_put_failure;
1920
1921         nla_nest_end(skb, nest);
1922         return 0;
1923
1924 nla_put_failure:
1925         return -ENOSPC;
1926 }
1927
1928 static int nft_dump_basechain_hook(struct sk_buff *skb,
1929                                    const struct net *net, int family,
1930                                    const struct nft_base_chain *basechain,
1931                                    const struct list_head *hook_list)
1932 {
1933         const struct nf_hook_ops *ops = &basechain->ops;
1934         struct nft_hook *hook, *first = NULL;
1935         struct nlattr *nest, *nest_devs;
1936         int n = 0;
1937
1938         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_HOOK);
1939         if (nest == NULL)
1940                 goto nla_put_failure;
1941         if (nla_put_be32(skb, NFTA_HOOK_HOOKNUM, htonl(ops->hooknum)))
1942                 goto nla_put_failure;
1943         if (nla_put_be32(skb, NFTA_HOOK_PRIORITY, htonl(ops->priority)))
1944                 goto nla_put_failure;
1945
1946         if (nft_base_chain_netdev(family, ops->hooknum)) {
1947                 nest_devs = nla_nest_start_noflag(skb, NFTA_HOOK_DEVS);
1948                 if (!nest_devs)
1949                         goto nla_put_failure;
1950
1951                 if (!hook_list)
1952                         hook_list = &basechain->hook_list;
1953
1954                 list_for_each_entry_rcu(hook, hook_list, list,
1955                                         lockdep_commit_lock_is_held(net)) {
1956                         if (!first)
1957                                 first = hook;
1958
1959                         if (nla_put_string(skb, NFTA_DEVICE_NAME,
1960                                            hook->ops.dev->name))
1961                                 goto nla_put_failure;
1962                         n++;
1963                 }
1964                 nla_nest_end(skb, nest_devs);
1965
1966                 if (n == 1 &&
1967                     nla_put_string(skb, NFTA_HOOK_DEV, first->ops.dev->name))
1968                         goto nla_put_failure;
1969         }
1970         nla_nest_end(skb, nest);
1971
1972         return 0;
1973 nla_put_failure:
1974         return -1;
1975 }
1976
1977 static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net,
1978                                      u32 portid, u32 seq, int event, u32 flags,
1979                                      int family, const struct nft_table *table,
1980                                      const struct nft_chain *chain,
1981                                      const struct list_head *hook_list)
1982 {
1983         struct nlmsghdr *nlh;
1984
1985         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
1986         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
1987                            NFNETLINK_V0, nft_base_seq(net));
1988         if (!nlh)
1989                 goto nla_put_failure;
1990
1991         if (nla_put_string(skb, NFTA_CHAIN_TABLE, table->name) ||
1992             nla_put_string(skb, NFTA_CHAIN_NAME, chain->name) ||
1993             nla_put_be64(skb, NFTA_CHAIN_HANDLE, cpu_to_be64(chain->handle),
1994                          NFTA_CHAIN_PAD))
1995                 goto nla_put_failure;
1996
1997         if (event == NFT_MSG_DELCHAIN && !hook_list) {
1998                 nlmsg_end(skb, nlh);
1999                 return 0;
2000         }
2001
2002         if (nft_is_base_chain(chain)) {
2003                 const struct nft_base_chain *basechain = nft_base_chain(chain);
2004                 struct nft_stats __percpu *stats;
2005
2006                 if (nft_dump_basechain_hook(skb, net, family, basechain, hook_list))
2007                         goto nla_put_failure;
2008
2009                 if (nla_put_be32(skb, NFTA_CHAIN_POLICY,
2010                                  htonl(basechain->policy)))
2011                         goto nla_put_failure;
2012
2013                 if (nla_put_string(skb, NFTA_CHAIN_TYPE, basechain->type->name))
2014                         goto nla_put_failure;
2015
2016                 stats = rcu_dereference_check(basechain->stats,
2017                                               lockdep_commit_lock_is_held(net));
2018                 if (nft_dump_stats(skb, stats))
2019                         goto nla_put_failure;
2020         }
2021
2022         if (chain->flags &&
2023             nla_put_be32(skb, NFTA_CHAIN_FLAGS, htonl(chain->flags)))
2024                 goto nla_put_failure;
2025
2026         if (nla_put_be32(skb, NFTA_CHAIN_USE, htonl(chain->use)))
2027                 goto nla_put_failure;
2028
2029         if (chain->udata &&
2030             nla_put(skb, NFTA_CHAIN_USERDATA, chain->udlen, chain->udata))
2031                 goto nla_put_failure;
2032
2033         nlmsg_end(skb, nlh);
2034         return 0;
2035
2036 nla_put_failure:
2037         nlmsg_trim(skb, nlh);
2038         return -1;
2039 }
2040
2041 static void nf_tables_chain_notify(const struct nft_ctx *ctx, int event,
2042                                    const struct list_head *hook_list)
2043 {
2044         struct nftables_pernet *nft_net;
2045         struct sk_buff *skb;
2046         u16 flags = 0;
2047         int err;
2048
2049         if (!ctx->report &&
2050             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
2051                 return;
2052
2053         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
2054         if (skb == NULL)
2055                 goto err;
2056
2057         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
2058                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
2059
2060         err = nf_tables_fill_chain_info(skb, ctx->net, ctx->portid, ctx->seq,
2061                                         event, flags, ctx->family, ctx->table,
2062                                         ctx->chain, hook_list);
2063         if (err < 0) {
2064                 kfree_skb(skb);
2065                 goto err;
2066         }
2067
2068         nft_net = nft_pernet(ctx->net);
2069         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
2070         return;
2071 err:
2072         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
2073 }
2074
2075 static int nf_tables_dump_chains(struct sk_buff *skb,
2076                                  struct netlink_callback *cb)
2077 {
2078         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
2079         unsigned int idx = 0, s_idx = cb->args[0];
2080         struct net *net = sock_net(skb->sk);
2081         int family = nfmsg->nfgen_family;
2082         struct nftables_pernet *nft_net;
2083         const struct nft_table *table;
2084         const struct nft_chain *chain;
2085
2086         rcu_read_lock();
2087         nft_net = nft_pernet(net);
2088         cb->seq = READ_ONCE(nft_net->base_seq);
2089
2090         list_for_each_entry_rcu(table, &nft_net->tables, list) {
2091                 if (family != NFPROTO_UNSPEC && family != table->family)
2092                         continue;
2093
2094                 list_for_each_entry_rcu(chain, &table->chains, list) {
2095                         if (idx < s_idx)
2096                                 goto cont;
2097                         if (idx > s_idx)
2098                                 memset(&cb->args[1], 0,
2099                                        sizeof(cb->args) - sizeof(cb->args[0]));
2100                         if (!nft_is_active(net, chain))
2101                                 continue;
2102                         if (nf_tables_fill_chain_info(skb, net,
2103                                                       NETLINK_CB(cb->skb).portid,
2104                                                       cb->nlh->nlmsg_seq,
2105                                                       NFT_MSG_NEWCHAIN,
2106                                                       NLM_F_MULTI,
2107                                                       table->family, table,
2108                                                       chain, NULL) < 0)
2109                                 goto done;
2110
2111                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
2112 cont:
2113                         idx++;
2114                 }
2115         }
2116 done:
2117         rcu_read_unlock();
2118         cb->args[0] = idx;
2119         return skb->len;
2120 }
2121
2122 /* called with rcu_read_lock held */
2123 static int nf_tables_getchain(struct sk_buff *skb, const struct nfnl_info *info,
2124                               const struct nlattr * const nla[])
2125 {
2126         struct netlink_ext_ack *extack = info->extack;
2127         u8 genmask = nft_genmask_cur(info->net);
2128         u8 family = info->nfmsg->nfgen_family;
2129         const struct nft_chain *chain;
2130         struct net *net = info->net;
2131         struct nft_table *table;
2132         struct sk_buff *skb2;
2133         int err;
2134
2135         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
2136                 struct netlink_dump_control c = {
2137                         .dump = nf_tables_dump_chains,
2138                         .module = THIS_MODULE,
2139                 };
2140
2141                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
2142         }
2143
2144         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask, 0);
2145         if (IS_ERR(table)) {
2146                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2147                 return PTR_ERR(table);
2148         }
2149
2150         chain = nft_chain_lookup(net, table, nla[NFTA_CHAIN_NAME], genmask);
2151         if (IS_ERR(chain)) {
2152                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2153                 return PTR_ERR(chain);
2154         }
2155
2156         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
2157         if (!skb2)
2158                 return -ENOMEM;
2159
2160         err = nf_tables_fill_chain_info(skb2, net, NETLINK_CB(skb).portid,
2161                                         info->nlh->nlmsg_seq, NFT_MSG_NEWCHAIN,
2162                                         0, family, table, chain, NULL);
2163         if (err < 0)
2164                 goto err_fill_chain_info;
2165
2166         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
2167
2168 err_fill_chain_info:
2169         kfree_skb(skb2);
2170         return err;
2171 }
2172
2173 static const struct nla_policy nft_counter_policy[NFTA_COUNTER_MAX + 1] = {
2174         [NFTA_COUNTER_PACKETS]  = { .type = NLA_U64 },
2175         [NFTA_COUNTER_BYTES]    = { .type = NLA_U64 },
2176 };
2177
2178 static struct nft_stats __percpu *nft_stats_alloc(const struct nlattr *attr)
2179 {
2180         struct nlattr *tb[NFTA_COUNTER_MAX+1];
2181         struct nft_stats __percpu *newstats;
2182         struct nft_stats *stats;
2183         int err;
2184
2185         err = nla_parse_nested_deprecated(tb, NFTA_COUNTER_MAX, attr,
2186                                           nft_counter_policy, NULL);
2187         if (err < 0)
2188                 return ERR_PTR_PCPU(err);
2189
2190         if (!tb[NFTA_COUNTER_BYTES] || !tb[NFTA_COUNTER_PACKETS])
2191                 return ERR_PTR_PCPU(-EINVAL);
2192
2193         newstats = netdev_alloc_pcpu_stats(struct nft_stats);
2194         if (newstats == NULL)
2195                 return ERR_PTR_PCPU(-ENOMEM);
2196
2197         /* Restore old counters on this cpu, no problem. Per-cpu statistics
2198          * are not exposed to userspace.
2199          */
2200         preempt_disable();
2201         stats = this_cpu_ptr(newstats);
2202         stats->bytes = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_BYTES]));
2203         stats->pkts = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_PACKETS]));
2204         preempt_enable();
2205
2206         return newstats;
2207 }
2208
2209 static void nft_chain_stats_replace(struct nft_trans_chain *trans)
2210 {
2211         const struct nft_trans *t = &trans->nft_trans_binding.nft_trans;
2212         struct nft_base_chain *chain = nft_base_chain(trans->chain);
2213
2214         if (!trans->stats)
2215                 return;
2216
2217         trans->stats =
2218                 rcu_replace_pointer(chain->stats, trans->stats,
2219                                     lockdep_commit_lock_is_held(t->net));
2220
2221         if (!trans->stats)
2222                 static_branch_inc(&nft_counters_enabled);
2223 }
2224
2225 static void nf_tables_chain_free_chain_rules(struct nft_chain *chain)
2226 {
2227         struct nft_rule_blob *g0 = rcu_dereference_raw(chain->blob_gen_0);
2228         struct nft_rule_blob *g1 = rcu_dereference_raw(chain->blob_gen_1);
2229
2230         if (g0 != g1)
2231                 kvfree(g1);
2232         kvfree(g0);
2233
2234         /* should be NULL either via abort or via successful commit */
2235         WARN_ON_ONCE(chain->blob_next);
2236         kvfree(chain->blob_next);
2237 }
2238
2239 void nf_tables_chain_destroy(struct nft_chain *chain)
2240 {
2241         const struct nft_table *table = chain->table;
2242         struct nft_hook *hook, *next;
2243
2244         if (WARN_ON(chain->use > 0))
2245                 return;
2246
2247         /* no concurrent access possible anymore */
2248         nf_tables_chain_free_chain_rules(chain);
2249
2250         if (nft_is_base_chain(chain)) {
2251                 struct nft_base_chain *basechain = nft_base_chain(chain);
2252
2253                 if (nft_base_chain_netdev(table->family, basechain->ops.hooknum)) {
2254                         list_for_each_entry_safe(hook, next,
2255                                                  &basechain->hook_list, list) {
2256                                 list_del_rcu(&hook->list);
2257                                 kfree_rcu(hook, rcu);
2258                         }
2259                 }
2260                 module_put(basechain->type->owner);
2261                 if (rcu_access_pointer(basechain->stats)) {
2262                         static_branch_dec(&nft_counters_enabled);
2263                         free_percpu(rcu_dereference_raw(basechain->stats));
2264                 }
2265                 kfree(chain->name);
2266                 kfree(chain->udata);
2267                 kfree(basechain);
2268         } else {
2269                 kfree(chain->name);
2270                 kfree(chain->udata);
2271                 kfree(chain);
2272         }
2273 }
2274
2275 static struct nft_hook *nft_netdev_hook_alloc(struct net *net,
2276                                               const struct nlattr *attr)
2277 {
2278         struct net_device *dev;
2279         char ifname[IFNAMSIZ];
2280         struct nft_hook *hook;
2281         int err;
2282
2283         hook = kzalloc(sizeof(struct nft_hook), GFP_KERNEL_ACCOUNT);
2284         if (!hook) {
2285                 err = -ENOMEM;
2286                 goto err_hook_alloc;
2287         }
2288
2289         nla_strscpy(ifname, attr, IFNAMSIZ);
2290         /* nf_tables_netdev_event() is called under rtnl_mutex, this is
2291          * indirectly serializing all the other holders of the commit_mutex with
2292          * the rtnl_mutex.
2293          */
2294         dev = __dev_get_by_name(net, ifname);
2295         if (!dev) {
2296                 err = -ENOENT;
2297                 goto err_hook_dev;
2298         }
2299         hook->ops.dev = dev;
2300
2301         return hook;
2302
2303 err_hook_dev:
2304         kfree(hook);
2305 err_hook_alloc:
2306         return ERR_PTR(err);
2307 }
2308
2309 static struct nft_hook *nft_hook_list_find(struct list_head *hook_list,
2310                                            const struct nft_hook *this)
2311 {
2312         struct nft_hook *hook;
2313
2314         list_for_each_entry(hook, hook_list, list) {
2315                 if (this->ops.dev == hook->ops.dev)
2316                         return hook;
2317         }
2318
2319         return NULL;
2320 }
2321
2322 static int nf_tables_parse_netdev_hooks(struct net *net,
2323                                         const struct nlattr *attr,
2324                                         struct list_head *hook_list,
2325                                         struct netlink_ext_ack *extack)
2326 {
2327         struct nft_hook *hook, *next;
2328         const struct nlattr *tmp;
2329         int rem, n = 0, err;
2330
2331         nla_for_each_nested(tmp, attr, rem) {
2332                 if (nla_type(tmp) != NFTA_DEVICE_NAME) {
2333                         err = -EINVAL;
2334                         goto err_hook;
2335                 }
2336
2337                 hook = nft_netdev_hook_alloc(net, tmp);
2338                 if (IS_ERR(hook)) {
2339                         NL_SET_BAD_ATTR(extack, tmp);
2340                         err = PTR_ERR(hook);
2341                         goto err_hook;
2342                 }
2343                 if (nft_hook_list_find(hook_list, hook)) {
2344                         NL_SET_BAD_ATTR(extack, tmp);
2345                         kfree(hook);
2346                         err = -EEXIST;
2347                         goto err_hook;
2348                 }
2349                 list_add_tail(&hook->list, hook_list);
2350                 n++;
2351
2352                 if (n == NFT_NETDEVICE_MAX) {
2353                         err = -EFBIG;
2354                         goto err_hook;
2355                 }
2356         }
2357
2358         return 0;
2359
2360 err_hook:
2361         list_for_each_entry_safe(hook, next, hook_list, list) {
2362                 list_del(&hook->list);
2363                 kfree(hook);
2364         }
2365         return err;
2366 }
2367
2368 struct nft_chain_hook {
2369         u32                             num;
2370         s32                             priority;
2371         const struct nft_chain_type     *type;
2372         struct list_head                list;
2373 };
2374
2375 static int nft_chain_parse_netdev(struct net *net, struct nlattr *tb[],
2376                                   struct list_head *hook_list,
2377                                   struct netlink_ext_ack *extack, u32 flags)
2378 {
2379         struct nft_hook *hook;
2380         int err;
2381
2382         if (tb[NFTA_HOOK_DEV]) {
2383                 hook = nft_netdev_hook_alloc(net, tb[NFTA_HOOK_DEV]);
2384                 if (IS_ERR(hook)) {
2385                         NL_SET_BAD_ATTR(extack, tb[NFTA_HOOK_DEV]);
2386                         return PTR_ERR(hook);
2387                 }
2388
2389                 list_add_tail(&hook->list, hook_list);
2390         } else if (tb[NFTA_HOOK_DEVS]) {
2391                 err = nf_tables_parse_netdev_hooks(net, tb[NFTA_HOOK_DEVS],
2392                                                    hook_list, extack);
2393                 if (err < 0)
2394                         return err;
2395
2396         }
2397
2398         if (flags & NFT_CHAIN_HW_OFFLOAD &&
2399             list_empty(hook_list))
2400                 return -EINVAL;
2401
2402         return 0;
2403 }
2404
2405 static int nft_chain_parse_hook(struct net *net,
2406                                 struct nft_base_chain *basechain,
2407                                 const struct nlattr * const nla[],
2408                                 struct nft_chain_hook *hook, u8 family,
2409                                 u32 flags, struct netlink_ext_ack *extack)
2410 {
2411         struct nftables_pernet *nft_net = nft_pernet(net);
2412         struct nlattr *ha[NFTA_HOOK_MAX + 1];
2413         const struct nft_chain_type *type;
2414         int err;
2415
2416         lockdep_assert_held(&nft_net->commit_mutex);
2417         lockdep_nfnl_nft_mutex_not_held();
2418
2419         err = nla_parse_nested_deprecated(ha, NFTA_HOOK_MAX,
2420                                           nla[NFTA_CHAIN_HOOK],
2421                                           nft_hook_policy, NULL);
2422         if (err < 0)
2423                 return err;
2424
2425         if (!basechain) {
2426                 if (!ha[NFTA_HOOK_HOOKNUM] ||
2427                     !ha[NFTA_HOOK_PRIORITY]) {
2428                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2429                         return -ENOENT;
2430                 }
2431
2432                 hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
2433                 hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
2434
2435                 type = __nft_chain_type_get(family, NFT_CHAIN_T_DEFAULT);
2436                 if (!type)
2437                         return -EOPNOTSUPP;
2438
2439                 if (nla[NFTA_CHAIN_TYPE]) {
2440                         type = nf_tables_chain_type_lookup(net, nla[NFTA_CHAIN_TYPE],
2441                                                            family, true);
2442                         if (IS_ERR(type)) {
2443                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2444                                 return PTR_ERR(type);
2445                         }
2446                 }
2447                 if (hook->num >= NFT_MAX_HOOKS || !(type->hook_mask & (1 << hook->num)))
2448                         return -EOPNOTSUPP;
2449
2450                 if (type->type == NFT_CHAIN_T_NAT &&
2451                     hook->priority <= NF_IP_PRI_CONNTRACK)
2452                         return -EOPNOTSUPP;
2453         } else {
2454                 if (ha[NFTA_HOOK_HOOKNUM]) {
2455                         hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
2456                         if (hook->num != basechain->ops.hooknum)
2457                                 return -EOPNOTSUPP;
2458                 }
2459                 if (ha[NFTA_HOOK_PRIORITY]) {
2460                         hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
2461                         if (hook->priority != basechain->ops.priority)
2462                                 return -EOPNOTSUPP;
2463                 }
2464
2465                 if (nla[NFTA_CHAIN_TYPE]) {
2466                         type = __nf_tables_chain_type_lookup(nla[NFTA_CHAIN_TYPE],
2467                                                              family);
2468                         if (!type) {
2469                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2470                                 return -ENOENT;
2471                         }
2472                 } else {
2473                         type = basechain->type;
2474                 }
2475         }
2476
2477         if (!try_module_get(type->owner)) {
2478                 if (nla[NFTA_CHAIN_TYPE])
2479                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2480                 return -ENOENT;
2481         }
2482
2483         hook->type = type;
2484
2485         INIT_LIST_HEAD(&hook->list);
2486         if (nft_base_chain_netdev(family, hook->num)) {
2487                 err = nft_chain_parse_netdev(net, ha, &hook->list, extack, flags);
2488                 if (err < 0) {
2489                         module_put(type->owner);
2490                         return err;
2491                 }
2492         } else if (ha[NFTA_HOOK_DEV] || ha[NFTA_HOOK_DEVS]) {
2493                 module_put(type->owner);
2494                 return -EOPNOTSUPP;
2495         }
2496
2497         return 0;
2498 }
2499
2500 static void nft_chain_release_hook(struct nft_chain_hook *hook)
2501 {
2502         struct nft_hook *h, *next;
2503
2504         list_for_each_entry_safe(h, next, &hook->list, list) {
2505                 list_del(&h->list);
2506                 kfree(h);
2507         }
2508         module_put(hook->type->owner);
2509 }
2510
2511 static void nft_last_rule(const struct nft_chain *chain, const void *ptr)
2512 {
2513         struct nft_rule_dp_last *lrule;
2514
2515         BUILD_BUG_ON(offsetof(struct nft_rule_dp_last, end) != 0);
2516
2517         lrule = (struct nft_rule_dp_last *)ptr;
2518         lrule->end.is_last = 1;
2519         lrule->chain = chain;
2520         /* blob size does not include the trailer rule */
2521 }
2522
2523 static struct nft_rule_blob *nf_tables_chain_alloc_rules(const struct nft_chain *chain,
2524                                                          unsigned int size)
2525 {
2526         struct nft_rule_blob *blob;
2527
2528         if (size > INT_MAX)
2529                 return NULL;
2530
2531         size += sizeof(struct nft_rule_blob) + sizeof(struct nft_rule_dp_last);
2532
2533         blob = kvmalloc(size, GFP_KERNEL_ACCOUNT);
2534         if (!blob)
2535                 return NULL;
2536
2537         blob->size = 0;
2538         nft_last_rule(chain, blob->data);
2539
2540         return blob;
2541 }
2542
2543 static void nft_basechain_hook_init(struct nf_hook_ops *ops, u8 family,
2544                                     const struct nft_chain_hook *hook,
2545                                     struct nft_chain *chain)
2546 {
2547         ops->pf                 = family;
2548         ops->hooknum            = hook->num;
2549         ops->priority           = hook->priority;
2550         ops->priv               = chain;
2551         ops->hook               = hook->type->hooks[ops->hooknum];
2552         ops->hook_ops_type      = NF_HOOK_OP_NF_TABLES;
2553 }
2554
2555 static int nft_basechain_init(struct nft_base_chain *basechain, u8 family,
2556                               struct nft_chain_hook *hook, u32 flags)
2557 {
2558         struct nft_chain *chain;
2559         struct nft_hook *h;
2560
2561         basechain->type = hook->type;
2562         INIT_LIST_HEAD(&basechain->hook_list);
2563         chain = &basechain->chain;
2564
2565         if (nft_base_chain_netdev(family, hook->num)) {
2566                 list_splice_init(&hook->list, &basechain->hook_list);
2567                 list_for_each_entry(h, &basechain->hook_list, list)
2568                         nft_basechain_hook_init(&h->ops, family, hook, chain);
2569         }
2570         nft_basechain_hook_init(&basechain->ops, family, hook, chain);
2571
2572         chain->flags |= NFT_CHAIN_BASE | flags;
2573         basechain->policy = NF_ACCEPT;
2574         if (chain->flags & NFT_CHAIN_HW_OFFLOAD &&
2575             !nft_chain_offload_support(basechain)) {
2576                 list_splice_init(&basechain->hook_list, &hook->list);
2577                 return -EOPNOTSUPP;
2578         }
2579
2580         flow_block_init(&basechain->flow_block);
2581
2582         return 0;
2583 }
2584
2585 int nft_chain_add(struct nft_table *table, struct nft_chain *chain)
2586 {
2587         int err;
2588
2589         err = rhltable_insert_key(&table->chains_ht, chain->name,
2590                                   &chain->rhlhead, nft_chain_ht_params);
2591         if (err)
2592                 return err;
2593
2594         list_add_tail_rcu(&chain->list, &table->chains);
2595
2596         return 0;
2597 }
2598
2599 static u64 chain_id;
2600
2601 static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 genmask,
2602                               u8 policy, u32 flags,
2603                               struct netlink_ext_ack *extack)
2604 {
2605         const struct nlattr * const *nla = ctx->nla;
2606         struct nft_table *table = ctx->table;
2607         struct nft_base_chain *basechain;
2608         struct net *net = ctx->net;
2609         char name[NFT_NAME_MAXLEN];
2610         struct nft_rule_blob *blob;
2611         struct nft_trans *trans;
2612         struct nft_chain *chain;
2613         int err;
2614
2615         if (nla[NFTA_CHAIN_HOOK]) {
2616                 struct nft_stats __percpu *stats = NULL;
2617                 struct nft_chain_hook hook = {};
2618
2619                 if (table->flags & __NFT_TABLE_F_UPDATE)
2620                         return -EINVAL;
2621
2622                 if (flags & NFT_CHAIN_BINDING)
2623                         return -EOPNOTSUPP;
2624
2625                 err = nft_chain_parse_hook(net, NULL, nla, &hook, family, flags,
2626                                            extack);
2627                 if (err < 0)
2628                         return err;
2629
2630                 basechain = kzalloc(sizeof(*basechain), GFP_KERNEL_ACCOUNT);
2631                 if (basechain == NULL) {
2632                         nft_chain_release_hook(&hook);
2633                         return -ENOMEM;
2634                 }
2635                 chain = &basechain->chain;
2636
2637                 if (nla[NFTA_CHAIN_COUNTERS]) {
2638                         stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2639                         if (IS_ERR_PCPU(stats)) {
2640                                 nft_chain_release_hook(&hook);
2641                                 kfree(basechain);
2642                                 return PTR_ERR_PCPU(stats);
2643                         }
2644                         rcu_assign_pointer(basechain->stats, stats);
2645                 }
2646
2647                 err = nft_basechain_init(basechain, family, &hook, flags);
2648                 if (err < 0) {
2649                         nft_chain_release_hook(&hook);
2650                         kfree(basechain);
2651                         free_percpu(stats);
2652                         return err;
2653                 }
2654                 if (stats)
2655                         static_branch_inc(&nft_counters_enabled);
2656         } else {
2657                 if (flags & NFT_CHAIN_BASE)
2658                         return -EINVAL;
2659                 if (flags & NFT_CHAIN_HW_OFFLOAD)
2660                         return -EOPNOTSUPP;
2661
2662                 chain = kzalloc(sizeof(*chain), GFP_KERNEL_ACCOUNT);
2663                 if (chain == NULL)
2664                         return -ENOMEM;
2665
2666                 chain->flags = flags;
2667         }
2668         ctx->chain = chain;
2669
2670         INIT_LIST_HEAD(&chain->rules);
2671         chain->handle = nf_tables_alloc_handle(table);
2672         chain->table = table;
2673
2674         if (nla[NFTA_CHAIN_NAME]) {
2675                 chain->name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
2676         } else {
2677                 if (!(flags & NFT_CHAIN_BINDING)) {
2678                         err = -EINVAL;
2679                         goto err_destroy_chain;
2680                 }
2681
2682                 snprintf(name, sizeof(name), "__chain%llu", ++chain_id);
2683                 chain->name = kstrdup(name, GFP_KERNEL_ACCOUNT);
2684         }
2685
2686         if (!chain->name) {
2687                 err = -ENOMEM;
2688                 goto err_destroy_chain;
2689         }
2690
2691         if (nla[NFTA_CHAIN_USERDATA]) {
2692                 chain->udata = nla_memdup(nla[NFTA_CHAIN_USERDATA], GFP_KERNEL_ACCOUNT);
2693                 if (chain->udata == NULL) {
2694                         err = -ENOMEM;
2695                         goto err_destroy_chain;
2696                 }
2697                 chain->udlen = nla_len(nla[NFTA_CHAIN_USERDATA]);
2698         }
2699
2700         blob = nf_tables_chain_alloc_rules(chain, 0);
2701         if (!blob) {
2702                 err = -ENOMEM;
2703                 goto err_destroy_chain;
2704         }
2705
2706         RCU_INIT_POINTER(chain->blob_gen_0, blob);
2707         RCU_INIT_POINTER(chain->blob_gen_1, blob);
2708
2709         if (!nft_use_inc(&table->use)) {
2710                 err = -EMFILE;
2711                 goto err_destroy_chain;
2712         }
2713
2714         trans = nft_trans_chain_add(ctx, NFT_MSG_NEWCHAIN);
2715         if (IS_ERR(trans)) {
2716                 err = PTR_ERR(trans);
2717                 goto err_trans;
2718         }
2719
2720         nft_trans_chain_policy(trans) = NFT_CHAIN_POLICY_UNSET;
2721         if (nft_is_base_chain(chain))
2722                 nft_trans_chain_policy(trans) = policy;
2723
2724         err = nft_chain_add(table, chain);
2725         if (err < 0)
2726                 goto err_chain_add;
2727
2728         /* This must be LAST to ensure no packets are walking over this chain. */
2729         err = nf_tables_register_hook(net, table, chain);
2730         if (err < 0)
2731                 goto err_register_hook;
2732
2733         return 0;
2734
2735 err_register_hook:
2736         nft_chain_del(chain);
2737 err_chain_add:
2738         nft_trans_destroy(trans);
2739 err_trans:
2740         nft_use_dec_restore(&table->use);
2741 err_destroy_chain:
2742         nf_tables_chain_destroy(chain);
2743
2744         return err;
2745 }
2746
2747 static int nf_tables_updchain(struct nft_ctx *ctx, u8 genmask, u8 policy,
2748                               u32 flags, const struct nlattr *attr,
2749                               struct netlink_ext_ack *extack)
2750 {
2751         const struct nlattr * const *nla = ctx->nla;
2752         struct nft_base_chain *basechain = NULL;
2753         struct nft_table *table = ctx->table;
2754         struct nft_chain *chain = ctx->chain;
2755         struct nft_chain_hook hook = {};
2756         struct nft_stats __percpu *stats = NULL;
2757         struct nft_hook *h, *next;
2758         struct nf_hook_ops *ops;
2759         struct nft_trans *trans;
2760         bool unregister = false;
2761         int err;
2762
2763         if (chain->flags ^ flags)
2764                 return -EOPNOTSUPP;
2765
2766         INIT_LIST_HEAD(&hook.list);
2767
2768         if (nla[NFTA_CHAIN_HOOK]) {
2769                 if (!nft_is_base_chain(chain)) {
2770                         NL_SET_BAD_ATTR(extack, attr);
2771                         return -EEXIST;
2772                 }
2773
2774                 basechain = nft_base_chain(chain);
2775                 err = nft_chain_parse_hook(ctx->net, basechain, nla, &hook,
2776                                            ctx->family, flags, extack);
2777                 if (err < 0)
2778                         return err;
2779
2780                 if (basechain->type != hook.type) {
2781                         nft_chain_release_hook(&hook);
2782                         NL_SET_BAD_ATTR(extack, attr);
2783                         return -EEXIST;
2784                 }
2785
2786                 if (nft_base_chain_netdev(ctx->family, basechain->ops.hooknum)) {
2787                         list_for_each_entry_safe(h, next, &hook.list, list) {
2788                                 h->ops.pf       = basechain->ops.pf;
2789                                 h->ops.hooknum  = basechain->ops.hooknum;
2790                                 h->ops.priority = basechain->ops.priority;
2791                                 h->ops.priv     = basechain->ops.priv;
2792                                 h->ops.hook     = basechain->ops.hook;
2793
2794                                 if (nft_hook_list_find(&basechain->hook_list, h)) {
2795                                         list_del(&h->list);
2796                                         kfree(h);
2797                                 }
2798                         }
2799                 } else {
2800                         ops = &basechain->ops;
2801                         if (ops->hooknum != hook.num ||
2802                             ops->priority != hook.priority) {
2803                                 nft_chain_release_hook(&hook);
2804                                 NL_SET_BAD_ATTR(extack, attr);
2805                                 return -EEXIST;
2806                         }
2807                 }
2808         }
2809
2810         if (nla[NFTA_CHAIN_HANDLE] &&
2811             nla[NFTA_CHAIN_NAME]) {
2812                 struct nft_chain *chain2;
2813
2814                 chain2 = nft_chain_lookup(ctx->net, table,
2815                                           nla[NFTA_CHAIN_NAME], genmask);
2816                 if (!IS_ERR(chain2)) {
2817                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2818                         err = -EEXIST;
2819                         goto err_hooks;
2820                 }
2821         }
2822
2823         if (table->flags & __NFT_TABLE_F_UPDATE &&
2824             !list_empty(&hook.list)) {
2825                 NL_SET_BAD_ATTR(extack, attr);
2826                 err = -EOPNOTSUPP;
2827                 goto err_hooks;
2828         }
2829
2830         if (!(table->flags & NFT_TABLE_F_DORMANT) &&
2831             nft_is_base_chain(chain) &&
2832             !list_empty(&hook.list)) {
2833                 basechain = nft_base_chain(chain);
2834                 ops = &basechain->ops;
2835
2836                 if (nft_base_chain_netdev(table->family, basechain->ops.hooknum)) {
2837                         err = nft_netdev_register_hooks(ctx->net, &hook.list);
2838                         if (err < 0)
2839                                 goto err_hooks;
2840                 }
2841         }
2842
2843         unregister = true;
2844
2845         if (nla[NFTA_CHAIN_COUNTERS]) {
2846                 if (!nft_is_base_chain(chain)) {
2847                         err = -EOPNOTSUPP;
2848                         goto err_hooks;
2849                 }
2850
2851                 stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2852                 if (IS_ERR_PCPU(stats)) {
2853                         err = PTR_ERR_PCPU(stats);
2854                         goto err_hooks;
2855                 }
2856         }
2857
2858         err = -ENOMEM;
2859         trans = nft_trans_alloc_chain(ctx, NFT_MSG_NEWCHAIN);
2860         if (trans == NULL)
2861                 goto err_trans;
2862
2863         nft_trans_chain_stats(trans) = stats;
2864         nft_trans_chain_update(trans) = true;
2865
2866         if (nla[NFTA_CHAIN_POLICY])
2867                 nft_trans_chain_policy(trans) = policy;
2868         else
2869                 nft_trans_chain_policy(trans) = -1;
2870
2871         if (nla[NFTA_CHAIN_HANDLE] &&
2872             nla[NFTA_CHAIN_NAME]) {
2873                 struct nftables_pernet *nft_net = nft_pernet(ctx->net);
2874                 struct nft_trans *tmp;
2875                 char *name;
2876
2877                 err = -ENOMEM;
2878                 name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
2879                 if (!name)
2880                         goto err_trans;
2881
2882                 err = -EEXIST;
2883                 list_for_each_entry(tmp, &nft_net->commit_list, list) {
2884                         if (tmp->msg_type == NFT_MSG_NEWCHAIN &&
2885                             tmp->table == table &&
2886                             nft_trans_chain_update(tmp) &&
2887                             nft_trans_chain_name(tmp) &&
2888                             strcmp(name, nft_trans_chain_name(tmp)) == 0) {
2889                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2890                                 kfree(name);
2891                                 goto err_trans;
2892                         }
2893                 }
2894
2895                 nft_trans_chain_name(trans) = name;
2896         }
2897
2898         nft_trans_basechain(trans) = basechain;
2899         INIT_LIST_HEAD(&nft_trans_chain_hooks(trans));
2900         list_splice(&hook.list, &nft_trans_chain_hooks(trans));
2901         if (nla[NFTA_CHAIN_HOOK])
2902                 module_put(hook.type->owner);
2903
2904         nft_trans_commit_list_add_tail(ctx->net, trans);
2905
2906         return 0;
2907
2908 err_trans:
2909         free_percpu(stats);
2910         kfree(trans);
2911 err_hooks:
2912         if (nla[NFTA_CHAIN_HOOK]) {
2913                 list_for_each_entry_safe(h, next, &hook.list, list) {
2914                         if (unregister)
2915                                 nf_unregister_net_hook(ctx->net, &h->ops);
2916                         list_del(&h->list);
2917                         kfree_rcu(h, rcu);
2918                 }
2919                 module_put(hook.type->owner);
2920         }
2921
2922         return err;
2923 }
2924
2925 static struct nft_chain *nft_chain_lookup_byid(const struct net *net,
2926                                                const struct nft_table *table,
2927                                                const struct nlattr *nla, u8 genmask)
2928 {
2929         struct nftables_pernet *nft_net = nft_pernet(net);
2930         u32 id = ntohl(nla_get_be32(nla));
2931         struct nft_trans *trans;
2932
2933         list_for_each_entry(trans, &nft_net->commit_list, list) {
2934                 if (trans->msg_type == NFT_MSG_NEWCHAIN &&
2935                     nft_trans_chain(trans)->table == table &&
2936                     id == nft_trans_chain_id(trans) &&
2937                     nft_active_genmask(nft_trans_chain(trans), genmask))
2938                         return nft_trans_chain(trans);
2939         }
2940         return ERR_PTR(-ENOENT);
2941 }
2942
2943 static int nf_tables_newchain(struct sk_buff *skb, const struct nfnl_info *info,
2944                               const struct nlattr * const nla[])
2945 {
2946         struct nftables_pernet *nft_net = nft_pernet(info->net);
2947         struct netlink_ext_ack *extack = info->extack;
2948         u8 genmask = nft_genmask_next(info->net);
2949         u8 family = info->nfmsg->nfgen_family;
2950         struct nft_chain *chain = NULL;
2951         struct net *net = info->net;
2952         const struct nlattr *attr;
2953         struct nft_table *table;
2954         u8 policy = NF_ACCEPT;
2955         struct nft_ctx ctx;
2956         u64 handle = 0;
2957         u32 flags = 0;
2958
2959         lockdep_assert_held(&nft_net->commit_mutex);
2960
2961         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
2962                                  NETLINK_CB(skb).portid);
2963         if (IS_ERR(table)) {
2964                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2965                 return PTR_ERR(table);
2966         }
2967
2968         chain = NULL;
2969         attr = nla[NFTA_CHAIN_NAME];
2970
2971         if (nla[NFTA_CHAIN_HANDLE]) {
2972                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_CHAIN_HANDLE]));
2973                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2974                 if (IS_ERR(chain)) {
2975                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_HANDLE]);
2976                         return PTR_ERR(chain);
2977                 }
2978                 attr = nla[NFTA_CHAIN_HANDLE];
2979         } else if (nla[NFTA_CHAIN_NAME]) {
2980                 chain = nft_chain_lookup(net, table, attr, genmask);
2981                 if (IS_ERR(chain)) {
2982                         if (PTR_ERR(chain) != -ENOENT) {
2983                                 NL_SET_BAD_ATTR(extack, attr);
2984                                 return PTR_ERR(chain);
2985                         }
2986                         chain = NULL;
2987                 }
2988         } else if (!nla[NFTA_CHAIN_ID]) {
2989                 return -EINVAL;
2990         }
2991
2992         if (nla[NFTA_CHAIN_POLICY]) {
2993                 if (chain != NULL &&
2994                     !nft_is_base_chain(chain)) {
2995                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2996                         return -EOPNOTSUPP;
2997                 }
2998
2999                 if (chain == NULL &&
3000                     nla[NFTA_CHAIN_HOOK] == NULL) {
3001                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
3002                         return -EOPNOTSUPP;
3003                 }
3004
3005                 policy = ntohl(nla_get_be32(nla[NFTA_CHAIN_POLICY]));
3006                 switch (policy) {
3007                 case NF_DROP:
3008                 case NF_ACCEPT:
3009                         break;
3010                 default:
3011                         return -EINVAL;
3012                 }
3013         }
3014
3015         if (nla[NFTA_CHAIN_FLAGS])
3016                 flags = ntohl(nla_get_be32(nla[NFTA_CHAIN_FLAGS]));
3017         else if (chain)
3018                 flags = chain->flags;
3019
3020         if (flags & ~NFT_CHAIN_FLAGS)
3021                 return -EOPNOTSUPP;
3022
3023         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
3024
3025         if (chain != NULL) {
3026                 if (chain->flags & NFT_CHAIN_BINDING)
3027                         return -EINVAL;
3028
3029                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
3030                         NL_SET_BAD_ATTR(extack, attr);
3031                         return -EEXIST;
3032                 }
3033                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
3034                         return -EOPNOTSUPP;
3035
3036                 flags |= chain->flags & NFT_CHAIN_BASE;
3037                 return nf_tables_updchain(&ctx, genmask, policy, flags, attr,
3038                                           extack);
3039         }
3040
3041         return nf_tables_addchain(&ctx, family, genmask, policy, flags, extack);
3042 }
3043
3044 static int nft_delchain_hook(struct nft_ctx *ctx,
3045                              struct nft_base_chain *basechain,
3046                              struct netlink_ext_ack *extack)
3047 {
3048         const struct nft_chain *chain = &basechain->chain;
3049         const struct nlattr * const *nla = ctx->nla;
3050         struct nft_chain_hook chain_hook = {};
3051         struct nft_hook *this, *hook;
3052         LIST_HEAD(chain_del_list);
3053         struct nft_trans *trans;
3054         int err;
3055
3056         if (ctx->table->flags & __NFT_TABLE_F_UPDATE)
3057                 return -EOPNOTSUPP;
3058
3059         err = nft_chain_parse_hook(ctx->net, basechain, nla, &chain_hook,
3060                                    ctx->family, chain->flags, extack);
3061         if (err < 0)
3062                 return err;
3063
3064         list_for_each_entry(this, &chain_hook.list, list) {
3065                 hook = nft_hook_list_find(&basechain->hook_list, this);
3066                 if (!hook) {
3067                         err = -ENOENT;
3068                         goto err_chain_del_hook;
3069                 }
3070                 list_move(&hook->list, &chain_del_list);
3071         }
3072
3073         trans = nft_trans_alloc_chain(ctx, NFT_MSG_DELCHAIN);
3074         if (!trans) {
3075                 err = -ENOMEM;
3076                 goto err_chain_del_hook;
3077         }
3078
3079         nft_trans_basechain(trans) = basechain;
3080         nft_trans_chain_update(trans) = true;
3081         INIT_LIST_HEAD(&nft_trans_chain_hooks(trans));
3082         list_splice(&chain_del_list, &nft_trans_chain_hooks(trans));
3083         nft_chain_release_hook(&chain_hook);
3084
3085         nft_trans_commit_list_add_tail(ctx->net, trans);
3086
3087         return 0;
3088
3089 err_chain_del_hook:
3090         list_splice(&chain_del_list, &basechain->hook_list);
3091         nft_chain_release_hook(&chain_hook);
3092
3093         return err;
3094 }
3095
3096 static int nf_tables_delchain(struct sk_buff *skb, const struct nfnl_info *info,
3097                               const struct nlattr * const nla[])
3098 {
3099         struct netlink_ext_ack *extack = info->extack;
3100         u8 genmask = nft_genmask_next(info->net);
3101         u8 family = info->nfmsg->nfgen_family;
3102         struct net *net = info->net;
3103         const struct nlattr *attr;
3104         struct nft_table *table;
3105         struct nft_chain *chain;
3106         struct nft_rule *rule;
3107         struct nft_ctx ctx;
3108         u64 handle;
3109         u32 use;
3110         int err;
3111
3112         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
3113                                  NETLINK_CB(skb).portid);
3114         if (IS_ERR(table)) {
3115                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
3116                 return PTR_ERR(table);
3117         }
3118
3119         if (nla[NFTA_CHAIN_HANDLE]) {
3120                 attr = nla[NFTA_CHAIN_HANDLE];
3121                 handle = be64_to_cpu(nla_get_be64(attr));
3122                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
3123         } else {
3124                 attr = nla[NFTA_CHAIN_NAME];
3125                 chain = nft_chain_lookup(net, table, attr, genmask);
3126         }
3127         if (IS_ERR(chain)) {
3128                 if (PTR_ERR(chain) == -ENOENT &&
3129                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYCHAIN)
3130                         return 0;
3131
3132                 NL_SET_BAD_ATTR(extack, attr);
3133                 return PTR_ERR(chain);
3134         }
3135
3136         if (nft_chain_binding(chain))
3137                 return -EOPNOTSUPP;
3138
3139         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
3140
3141         if (nla[NFTA_CHAIN_HOOK]) {
3142                 if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYCHAIN ||
3143                     chain->flags & NFT_CHAIN_HW_OFFLOAD)
3144                         return -EOPNOTSUPP;
3145
3146                 if (nft_is_base_chain(chain)) {
3147                         struct nft_base_chain *basechain = nft_base_chain(chain);
3148
3149                         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
3150                                 return nft_delchain_hook(&ctx, basechain, extack);
3151                 }
3152         }
3153
3154         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
3155             chain->use > 0)
3156                 return -EBUSY;
3157
3158         use = chain->use;
3159         list_for_each_entry(rule, &chain->rules, list) {
3160                 if (!nft_is_active_next(net, rule))
3161                         continue;
3162                 use--;
3163
3164                 err = nft_delrule(&ctx, rule);
3165                 if (err < 0)
3166                         return err;
3167         }
3168
3169         /* There are rules and elements that are still holding references to us,
3170          * we cannot do a recursive removal in this case.
3171          */
3172         if (use > 0) {
3173                 NL_SET_BAD_ATTR(extack, attr);
3174                 return -EBUSY;
3175         }
3176
3177         return nft_delchain(&ctx);
3178 }
3179
3180 /*
3181  * Expressions
3182  */
3183
3184 /**
3185  *      nft_register_expr - register nf_tables expr type
3186  *      @type: expr type
3187  *
3188  *      Registers the expr type for use with nf_tables. Returns zero on
3189  *      success or a negative errno code otherwise.
3190  */
3191 int nft_register_expr(struct nft_expr_type *type)
3192 {
3193         if (WARN_ON_ONCE(type->maxattr > NFT_EXPR_MAXATTR))
3194                 return -ENOMEM;
3195
3196         nfnl_lock(NFNL_SUBSYS_NFTABLES);
3197         if (type->family == NFPROTO_UNSPEC)
3198                 list_add_tail_rcu(&type->list, &nf_tables_expressions);
3199         else
3200                 list_add_rcu(&type->list, &nf_tables_expressions);
3201         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
3202         return 0;
3203 }
3204 EXPORT_SYMBOL_GPL(nft_register_expr);
3205
3206 /**
3207  *      nft_unregister_expr - unregister nf_tables expr type
3208  *      @type: expr type
3209  *
3210  *      Unregisters the expr typefor use with nf_tables.
3211  */
3212 void nft_unregister_expr(struct nft_expr_type *type)
3213 {
3214         nfnl_lock(NFNL_SUBSYS_NFTABLES);
3215         list_del_rcu(&type->list);
3216         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
3217 }
3218 EXPORT_SYMBOL_GPL(nft_unregister_expr);
3219
3220 static const struct nft_expr_type *__nft_expr_type_get(u8 family,
3221                                                        struct nlattr *nla)
3222 {
3223         const struct nft_expr_type *type, *candidate = NULL;
3224
3225         list_for_each_entry_rcu(type, &nf_tables_expressions, list) {
3226                 if (!nla_strcmp(nla, type->name)) {
3227                         if (!type->family && !candidate)
3228                                 candidate = type;
3229                         else if (type->family == family)
3230                                 candidate = type;
3231                 }
3232         }
3233         return candidate;
3234 }
3235
3236 #ifdef CONFIG_MODULES
3237 static int nft_expr_type_request_module(struct net *net, u8 family,
3238                                         struct nlattr *nla)
3239 {
3240         if (nft_request_module(net, "nft-expr-%u-%.*s", family,
3241                                nla_len(nla), (char *)nla_data(nla)) == -EAGAIN)
3242                 return -EAGAIN;
3243
3244         return 0;
3245 }
3246 #endif
3247
3248 static const struct nft_expr_type *nft_expr_type_get(struct net *net,
3249                                                      u8 family,
3250                                                      struct nlattr *nla)
3251 {
3252         const struct nft_expr_type *type;
3253
3254         if (nla == NULL)
3255                 return ERR_PTR(-EINVAL);
3256
3257         rcu_read_lock();
3258         type = __nft_expr_type_get(family, nla);
3259         if (type != NULL && try_module_get(type->owner)) {
3260                 rcu_read_unlock();
3261                 return type;
3262         }
3263         rcu_read_unlock();
3264
3265         lockdep_nfnl_nft_mutex_not_held();
3266 #ifdef CONFIG_MODULES
3267         if (type == NULL) {
3268                 if (nft_expr_type_request_module(net, family, nla) == -EAGAIN)
3269                         return ERR_PTR(-EAGAIN);
3270
3271                 if (nft_request_module(net, "nft-expr-%.*s",
3272                                        nla_len(nla),
3273                                        (char *)nla_data(nla)) == -EAGAIN)
3274                         return ERR_PTR(-EAGAIN);
3275         }
3276 #endif
3277         return ERR_PTR(-ENOENT);
3278 }
3279
3280 static const struct nla_policy nft_expr_policy[NFTA_EXPR_MAX + 1] = {
3281         [NFTA_EXPR_NAME]        = { .type = NLA_STRING,
3282                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
3283         [NFTA_EXPR_DATA]        = { .type = NLA_NESTED },
3284 };
3285
3286 static int nf_tables_fill_expr_info(struct sk_buff *skb,
3287                                     const struct nft_expr *expr, bool reset)
3288 {
3289         if (nla_put_string(skb, NFTA_EXPR_NAME, expr->ops->type->name))
3290                 goto nla_put_failure;
3291
3292         if (expr->ops->dump) {
3293                 struct nlattr *data = nla_nest_start_noflag(skb,
3294                                                             NFTA_EXPR_DATA);
3295                 if (data == NULL)
3296                         goto nla_put_failure;
3297                 if (expr->ops->dump(skb, expr, reset) < 0)
3298                         goto nla_put_failure;
3299                 nla_nest_end(skb, data);
3300         }
3301
3302         return skb->len;
3303
3304 nla_put_failure:
3305         return -1;
3306 };
3307
3308 int nft_expr_dump(struct sk_buff *skb, unsigned int attr,
3309                   const struct nft_expr *expr, bool reset)
3310 {
3311         struct nlattr *nest;
3312
3313         nest = nla_nest_start_noflag(skb, attr);
3314         if (!nest)
3315                 goto nla_put_failure;
3316         if (nf_tables_fill_expr_info(skb, expr, reset) < 0)
3317                 goto nla_put_failure;
3318         nla_nest_end(skb, nest);
3319         return 0;
3320
3321 nla_put_failure:
3322         return -1;
3323 }
3324
3325 struct nft_expr_info {
3326         const struct nft_expr_ops       *ops;
3327         const struct nlattr             *attr;
3328         struct nlattr                   *tb[NFT_EXPR_MAXATTR + 1];
3329 };
3330
3331 static int nf_tables_expr_parse(const struct nft_ctx *ctx,
3332                                 const struct nlattr *nla,
3333                                 struct nft_expr_info *info)
3334 {
3335         const struct nft_expr_type *type;
3336         const struct nft_expr_ops *ops;
3337         struct nlattr *tb[NFTA_EXPR_MAX + 1];
3338         int err;
3339
3340         err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
3341                                           nft_expr_policy, NULL);
3342         if (err < 0)
3343                 return err;
3344
3345         type = nft_expr_type_get(ctx->net, ctx->family, tb[NFTA_EXPR_NAME]);
3346         if (IS_ERR(type))
3347                 return PTR_ERR(type);
3348
3349         if (tb[NFTA_EXPR_DATA]) {
3350                 err = nla_parse_nested_deprecated(info->tb, type->maxattr,
3351                                                   tb[NFTA_EXPR_DATA],
3352                                                   type->policy, NULL);
3353                 if (err < 0)
3354                         goto err1;
3355         } else
3356                 memset(info->tb, 0, sizeof(info->tb[0]) * (type->maxattr + 1));
3357
3358         if (type->select_ops != NULL) {
3359                 ops = type->select_ops(ctx,
3360                                        (const struct nlattr * const *)info->tb);
3361                 if (IS_ERR(ops)) {
3362                         err = PTR_ERR(ops);
3363 #ifdef CONFIG_MODULES
3364                         if (err == -EAGAIN)
3365                                 if (nft_expr_type_request_module(ctx->net,
3366                                                                  ctx->family,
3367                                                                  tb[NFTA_EXPR_NAME]) != -EAGAIN)
3368                                         err = -ENOENT;
3369 #endif
3370                         goto err1;
3371                 }
3372         } else
3373                 ops = type->ops;
3374
3375         info->attr = nla;
3376         info->ops = ops;
3377
3378         return 0;
3379
3380 err1:
3381         module_put(type->owner);
3382         return err;
3383 }
3384
3385 int nft_expr_inner_parse(const struct nft_ctx *ctx, const struct nlattr *nla,
3386                          struct nft_expr_info *info)
3387 {
3388         struct nlattr *tb[NFTA_EXPR_MAX + 1];
3389         const struct nft_expr_type *type;
3390         int err;
3391
3392         err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
3393                                           nft_expr_policy, NULL);
3394         if (err < 0)
3395                 return err;
3396
3397         if (!tb[NFTA_EXPR_DATA] || !tb[NFTA_EXPR_NAME])
3398                 return -EINVAL;
3399
3400         rcu_read_lock();
3401
3402         type = __nft_expr_type_get(ctx->family, tb[NFTA_EXPR_NAME]);
3403         if (!type) {
3404                 err = -ENOENT;
3405                 goto out_unlock;
3406         }
3407
3408         if (!type->inner_ops) {
3409                 err = -EOPNOTSUPP;
3410                 goto out_unlock;
3411         }
3412
3413         err = nla_parse_nested_deprecated(info->tb, type->maxattr,
3414                                           tb[NFTA_EXPR_DATA],
3415                                           type->policy, NULL);
3416         if (err < 0)
3417                 goto out_unlock;
3418
3419         info->attr = nla;
3420         info->ops = type->inner_ops;
3421
3422         /* No module reference will be taken on type->owner.
3423          * Presence of type->inner_ops implies that the expression
3424          * is builtin, so it cannot go away.
3425          */
3426         rcu_read_unlock();
3427         return 0;
3428
3429 out_unlock:
3430         rcu_read_unlock();
3431         return err;
3432 }
3433
3434 static int nf_tables_newexpr(const struct nft_ctx *ctx,
3435                              const struct nft_expr_info *expr_info,
3436                              struct nft_expr *expr)
3437 {
3438         const struct nft_expr_ops *ops = expr_info->ops;
3439         int err;
3440
3441         expr->ops = ops;
3442         if (ops->init) {
3443                 err = ops->init(ctx, expr, (const struct nlattr **)expr_info->tb);
3444                 if (err < 0)
3445                         goto err1;
3446         }
3447
3448         return 0;
3449 err1:
3450         expr->ops = NULL;
3451         return err;
3452 }
3453
3454 static void nf_tables_expr_destroy(const struct nft_ctx *ctx,
3455                                    struct nft_expr *expr)
3456 {
3457         const struct nft_expr_type *type = expr->ops->type;
3458
3459         if (expr->ops->destroy)
3460                 expr->ops->destroy(ctx, expr);
3461         module_put(type->owner);
3462 }
3463
3464 static struct nft_expr *nft_expr_init(const struct nft_ctx *ctx,
3465                                       const struct nlattr *nla)
3466 {
3467         struct nft_expr_info expr_info;
3468         struct nft_expr *expr;
3469         struct module *owner;
3470         int err;
3471
3472         err = nf_tables_expr_parse(ctx, nla, &expr_info);
3473         if (err < 0)
3474                 goto err_expr_parse;
3475
3476         err = -EOPNOTSUPP;
3477         if (!(expr_info.ops->type->flags & NFT_EXPR_STATEFUL))
3478                 goto err_expr_stateful;
3479
3480         err = -ENOMEM;
3481         expr = kzalloc(expr_info.ops->size, GFP_KERNEL_ACCOUNT);
3482         if (expr == NULL)
3483                 goto err_expr_stateful;
3484
3485         err = nf_tables_newexpr(ctx, &expr_info, expr);
3486         if (err < 0)
3487                 goto err_expr_new;
3488
3489         return expr;
3490 err_expr_new:
3491         kfree(expr);
3492 err_expr_stateful:
3493         owner = expr_info.ops->type->owner;
3494         if (expr_info.ops->type->release_ops)
3495                 expr_info.ops->type->release_ops(expr_info.ops);
3496
3497         module_put(owner);
3498 err_expr_parse:
3499         return ERR_PTR(err);
3500 }
3501
3502 int nft_expr_clone(struct nft_expr *dst, struct nft_expr *src, gfp_t gfp)
3503 {
3504         int err;
3505
3506         if (WARN_ON_ONCE(!src->ops->clone))
3507                 return -EINVAL;
3508
3509         dst->ops = src->ops;
3510         err = src->ops->clone(dst, src, gfp);
3511         if (err < 0)
3512                 return err;
3513
3514         __module_get(src->ops->type->owner);
3515
3516         return 0;
3517 }
3518
3519 void nft_expr_destroy(const struct nft_ctx *ctx, struct nft_expr *expr)
3520 {
3521         nf_tables_expr_destroy(ctx, expr);
3522         kfree(expr);
3523 }
3524
3525 /*
3526  * Rules
3527  */
3528
3529 static struct nft_rule *__nft_rule_lookup(const struct net *net,
3530                                           const struct nft_chain *chain,
3531                                           u64 handle)
3532 {
3533         struct nft_rule *rule;
3534
3535         // FIXME: this sucks
3536         list_for_each_entry_rcu(rule, &chain->rules, list,
3537                                 lockdep_commit_lock_is_held(net)) {
3538                 if (handle == rule->handle)
3539                         return rule;
3540         }
3541
3542         return ERR_PTR(-ENOENT);
3543 }
3544
3545 static struct nft_rule *nft_rule_lookup(const struct net *net,
3546                                         const struct nft_chain *chain,
3547                                         const struct nlattr *nla)
3548 {
3549         if (nla == NULL)
3550                 return ERR_PTR(-EINVAL);
3551
3552         return __nft_rule_lookup(net, chain, be64_to_cpu(nla_get_be64(nla)));
3553 }
3554
3555 static const struct nla_policy nft_rule_policy[NFTA_RULE_MAX + 1] = {
3556         [NFTA_RULE_TABLE]       = { .type = NLA_STRING,
3557                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
3558         [NFTA_RULE_CHAIN]       = { .type = NLA_STRING,
3559                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
3560         [NFTA_RULE_HANDLE]      = { .type = NLA_U64 },
3561         [NFTA_RULE_EXPRESSIONS] = NLA_POLICY_NESTED_ARRAY(nft_expr_policy),
3562         [NFTA_RULE_COMPAT]      = { .type = NLA_NESTED },
3563         [NFTA_RULE_POSITION]    = { .type = NLA_U64 },
3564         [NFTA_RULE_USERDATA]    = { .type = NLA_BINARY,
3565                                     .len = NFT_USERDATA_MAXLEN },
3566         [NFTA_RULE_ID]          = { .type = NLA_U32 },
3567         [NFTA_RULE_POSITION_ID] = { .type = NLA_U32 },
3568         [NFTA_RULE_CHAIN_ID]    = { .type = NLA_U32 },
3569 };
3570
3571 static int nf_tables_fill_rule_info(struct sk_buff *skb, struct net *net,
3572                                     u32 portid, u32 seq, int event,
3573                                     u32 flags, int family,
3574                                     const struct nft_table *table,
3575                                     const struct nft_chain *chain,
3576                                     const struct nft_rule *rule, u64 handle,
3577                                     bool reset)
3578 {
3579         struct nlmsghdr *nlh;
3580         const struct nft_expr *expr, *next;
3581         struct nlattr *list;
3582         u16 type = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
3583
3584         nlh = nfnl_msg_put(skb, portid, seq, type, flags, family, NFNETLINK_V0,
3585                            nft_base_seq(net));
3586         if (!nlh)
3587                 goto nla_put_failure;
3588
3589         if (nla_put_string(skb, NFTA_RULE_TABLE, table->name))
3590                 goto nla_put_failure;
3591         if (nla_put_string(skb, NFTA_RULE_CHAIN, chain->name))
3592                 goto nla_put_failure;
3593         if (nla_put_be64(skb, NFTA_RULE_HANDLE, cpu_to_be64(rule->handle),
3594                          NFTA_RULE_PAD))
3595                 goto nla_put_failure;
3596
3597         if (event != NFT_MSG_DELRULE && handle) {
3598                 if (nla_put_be64(skb, NFTA_RULE_POSITION, cpu_to_be64(handle),
3599                                  NFTA_RULE_PAD))
3600                         goto nla_put_failure;
3601         }
3602
3603         if (chain->flags & NFT_CHAIN_HW_OFFLOAD)
3604                 nft_flow_rule_stats(chain, rule);
3605
3606         list = nla_nest_start_noflag(skb, NFTA_RULE_EXPRESSIONS);
3607         if (list == NULL)
3608                 goto nla_put_failure;
3609         nft_rule_for_each_expr(expr, next, rule) {
3610                 if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr, reset) < 0)
3611                         goto nla_put_failure;
3612         }
3613         nla_nest_end(skb, list);
3614
3615         if (rule->udata) {
3616                 struct nft_userdata *udata = nft_userdata(rule);
3617                 if (nla_put(skb, NFTA_RULE_USERDATA, udata->len + 1,
3618                             udata->data) < 0)
3619                         goto nla_put_failure;
3620         }
3621
3622         nlmsg_end(skb, nlh);
3623         return 0;
3624
3625 nla_put_failure:
3626         nlmsg_trim(skb, nlh);
3627         return -1;
3628 }
3629
3630 static void nf_tables_rule_notify(const struct nft_ctx *ctx,
3631                                   const struct nft_rule *rule, int event)
3632 {
3633         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
3634         const struct nft_rule *prule;
3635         struct sk_buff *skb;
3636         u64 handle = 0;
3637         u16 flags = 0;
3638         int err;
3639
3640         if (!ctx->report &&
3641             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
3642                 return;
3643
3644         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
3645         if (skb == NULL)
3646                 goto err;
3647
3648         if (event == NFT_MSG_NEWRULE &&
3649             !list_is_first(&rule->list, &ctx->chain->rules) &&
3650             !list_is_last(&rule->list, &ctx->chain->rules)) {
3651                 prule = list_prev_entry(rule, list);
3652                 handle = prule->handle;
3653         }
3654         if (ctx->flags & (NLM_F_APPEND | NLM_F_REPLACE))
3655                 flags |= NLM_F_APPEND;
3656         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
3657                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
3658
3659         err = nf_tables_fill_rule_info(skb, ctx->net, ctx->portid, ctx->seq,
3660                                        event, flags, ctx->family, ctx->table,
3661                                        ctx->chain, rule, handle, false);
3662         if (err < 0) {
3663                 kfree_skb(skb);
3664                 goto err;
3665         }
3666
3667         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
3668         return;
3669 err:
3670         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
3671 }
3672
3673 static void audit_log_rule_reset(const struct nft_table *table,
3674                                  unsigned int base_seq,
3675                                  unsigned int nentries)
3676 {
3677         char *buf = kasprintf(GFP_ATOMIC, "%s:%u",
3678                               table->name, base_seq);
3679
3680         audit_log_nfcfg(buf, table->family, nentries,
3681                         AUDIT_NFT_OP_RULE_RESET, GFP_ATOMIC);
3682         kfree(buf);
3683 }
3684
3685 struct nft_rule_dump_ctx {
3686         unsigned int s_idx;
3687         char *table;
3688         char *chain;
3689         bool reset;
3690 };
3691
3692 static int __nf_tables_dump_rules(struct sk_buff *skb,
3693                                   unsigned int *idx,
3694                                   struct netlink_callback *cb,
3695                                   const struct nft_table *table,
3696                                   const struct nft_chain *chain)
3697 {
3698         struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
3699         struct net *net = sock_net(skb->sk);
3700         const struct nft_rule *rule, *prule;
3701         unsigned int entries = 0;
3702         int ret = 0;
3703         u64 handle;
3704
3705         prule = NULL;
3706         list_for_each_entry_rcu(rule, &chain->rules, list) {
3707                 if (!nft_is_active(net, rule))
3708                         goto cont_skip;
3709                 if (*idx < ctx->s_idx)
3710                         goto cont;
3711                 if (prule)
3712                         handle = prule->handle;
3713                 else
3714                         handle = 0;
3715
3716                 if (nf_tables_fill_rule_info(skb, net, NETLINK_CB(cb->skb).portid,
3717                                         cb->nlh->nlmsg_seq,
3718                                         NFT_MSG_NEWRULE,
3719                                         NLM_F_MULTI | NLM_F_APPEND,
3720                                         table->family,
3721                                         table, chain, rule, handle, ctx->reset) < 0) {
3722                         ret = 1;
3723                         break;
3724                 }
3725                 entries++;
3726                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
3727 cont:
3728                 prule = rule;
3729 cont_skip:
3730                 (*idx)++;
3731         }
3732
3733         if (ctx->reset && entries)
3734                 audit_log_rule_reset(table, cb->seq, entries);
3735
3736         return ret;
3737 }
3738
3739 static int nf_tables_dump_rules(struct sk_buff *skb,
3740                                 struct netlink_callback *cb)
3741 {
3742         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
3743         struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
3744         struct nft_table *table;
3745         const struct nft_chain *chain;
3746         unsigned int idx = 0;
3747         struct net *net = sock_net(skb->sk);
3748         int family = nfmsg->nfgen_family;
3749         struct nftables_pernet *nft_net;
3750
3751         rcu_read_lock();
3752         nft_net = nft_pernet(net);
3753         cb->seq = READ_ONCE(nft_net->base_seq);
3754
3755         list_for_each_entry_rcu(table, &nft_net->tables, list) {
3756                 if (family != NFPROTO_UNSPEC && family != table->family)
3757                         continue;
3758
3759                 if (ctx->table && strcmp(ctx->table, table->name) != 0)
3760                         continue;
3761
3762                 if (ctx->table && ctx->chain) {
3763                         struct rhlist_head *list, *tmp;
3764
3765                         list = rhltable_lookup(&table->chains_ht, ctx->chain,
3766                                                nft_chain_ht_params);
3767                         if (!list)
3768                                 goto done;
3769
3770                         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
3771                                 if (!nft_is_active(net, chain))
3772                                         continue;
3773                                 __nf_tables_dump_rules(skb, &idx,
3774                                                        cb, table, chain);
3775                                 break;
3776                         }
3777                         goto done;
3778                 }
3779
3780                 list_for_each_entry_rcu(chain, &table->chains, list) {
3781                         if (__nf_tables_dump_rules(skb, &idx,
3782                                                    cb, table, chain))
3783                                 goto done;
3784                 }
3785
3786                 if (ctx->table)
3787                         break;
3788         }
3789 done:
3790         rcu_read_unlock();
3791
3792         ctx->s_idx = idx;
3793         return skb->len;
3794 }
3795
3796 static int nf_tables_dumpreset_rules(struct sk_buff *skb,
3797                                      struct netlink_callback *cb)
3798 {
3799         struct nftables_pernet *nft_net = nft_pernet(sock_net(skb->sk));
3800         int ret;
3801
3802         /* Mutex is held is to prevent that two concurrent dump-and-reset calls
3803          * do not underrun counters and quotas. The commit_mutex is used for
3804          * the lack a better lock, this is not transaction path.
3805          */
3806         mutex_lock(&nft_net->commit_mutex);
3807         ret = nf_tables_dump_rules(skb, cb);
3808         mutex_unlock(&nft_net->commit_mutex);
3809
3810         return ret;
3811 }
3812
3813 static int nf_tables_dump_rules_start(struct netlink_callback *cb)
3814 {
3815         struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
3816         const struct nlattr * const *nla = cb->data;
3817
3818         BUILD_BUG_ON(sizeof(*ctx) > sizeof(cb->ctx));
3819
3820         if (nla[NFTA_RULE_TABLE]) {
3821                 ctx->table = nla_strdup(nla[NFTA_RULE_TABLE], GFP_ATOMIC);
3822                 if (!ctx->table)
3823                         return -ENOMEM;
3824         }
3825         if (nla[NFTA_RULE_CHAIN]) {
3826                 ctx->chain = nla_strdup(nla[NFTA_RULE_CHAIN], GFP_ATOMIC);
3827                 if (!ctx->chain) {
3828                         kfree(ctx->table);
3829                         return -ENOMEM;
3830                 }
3831         }
3832         return 0;
3833 }
3834
3835 static int nf_tables_dumpreset_rules_start(struct netlink_callback *cb)
3836 {
3837         struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
3838
3839         ctx->reset = true;
3840
3841         return nf_tables_dump_rules_start(cb);
3842 }
3843
3844 static int nf_tables_dump_rules_done(struct netlink_callback *cb)
3845 {
3846         struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
3847
3848         kfree(ctx->table);
3849         kfree(ctx->chain);
3850         return 0;
3851 }
3852
3853 /* Caller must hold rcu read lock or transaction mutex */
3854 static struct sk_buff *
3855 nf_tables_getrule_single(u32 portid, const struct nfnl_info *info,
3856                          const struct nlattr * const nla[], bool reset)
3857 {
3858         struct netlink_ext_ack *extack = info->extack;
3859         u8 genmask = nft_genmask_cur(info->net);
3860         u8 family = info->nfmsg->nfgen_family;
3861         const struct nft_chain *chain;
3862         const struct nft_rule *rule;
3863         struct net *net = info->net;
3864         struct nft_table *table;
3865         struct sk_buff *skb2;
3866         int err;
3867
3868         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask, 0);
3869         if (IS_ERR(table)) {
3870                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3871                 return ERR_CAST(table);
3872         }
3873
3874         chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN], genmask);
3875         if (IS_ERR(chain)) {
3876                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3877                 return ERR_CAST(chain);
3878         }
3879
3880         rule = nft_rule_lookup(net, chain, nla[NFTA_RULE_HANDLE]);
3881         if (IS_ERR(rule)) {
3882                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3883                 return ERR_CAST(rule);
3884         }
3885
3886         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
3887         if (!skb2)
3888                 return ERR_PTR(-ENOMEM);
3889
3890         err = nf_tables_fill_rule_info(skb2, net, portid,
3891                                        info->nlh->nlmsg_seq, NFT_MSG_NEWRULE, 0,
3892                                        family, table, chain, rule, 0, reset);
3893         if (err < 0) {
3894                 kfree_skb(skb2);
3895                 return ERR_PTR(err);
3896         }
3897
3898         return skb2;
3899 }
3900
3901 static int nf_tables_getrule(struct sk_buff *skb, const struct nfnl_info *info,
3902                              const struct nlattr * const nla[])
3903 {
3904         u32 portid = NETLINK_CB(skb).portid;
3905         struct net *net = info->net;
3906         struct sk_buff *skb2;
3907
3908         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
3909                 struct netlink_dump_control c = {
3910                         .start= nf_tables_dump_rules_start,
3911                         .dump = nf_tables_dump_rules,
3912                         .done = nf_tables_dump_rules_done,
3913                         .module = THIS_MODULE,
3914                         .data = (void *)nla,
3915                 };
3916
3917                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
3918         }
3919
3920         skb2 = nf_tables_getrule_single(portid, info, nla, false);
3921         if (IS_ERR(skb2))
3922                 return PTR_ERR(skb2);
3923
3924         return nfnetlink_unicast(skb2, net, portid);
3925 }
3926
3927 static int nf_tables_getrule_reset(struct sk_buff *skb,
3928                                    const struct nfnl_info *info,
3929                                    const struct nlattr * const nla[])
3930 {
3931         struct nftables_pernet *nft_net = nft_pernet(info->net);
3932         u32 portid = NETLINK_CB(skb).portid;
3933         struct net *net = info->net;
3934         struct sk_buff *skb2;
3935         char *buf;
3936
3937         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
3938                 struct netlink_dump_control c = {
3939                         .start= nf_tables_dumpreset_rules_start,
3940                         .dump = nf_tables_dumpreset_rules,
3941                         .done = nf_tables_dump_rules_done,
3942                         .module = THIS_MODULE,
3943                         .data = (void *)nla,
3944                 };
3945
3946                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
3947         }
3948
3949         if (!try_module_get(THIS_MODULE))
3950                 return -EINVAL;
3951         rcu_read_unlock();
3952         mutex_lock(&nft_net->commit_mutex);
3953         skb2 = nf_tables_getrule_single(portid, info, nla, true);
3954         mutex_unlock(&nft_net->commit_mutex);
3955         rcu_read_lock();
3956         module_put(THIS_MODULE);
3957
3958         if (IS_ERR(skb2))
3959                 return PTR_ERR(skb2);
3960
3961         buf = kasprintf(GFP_ATOMIC, "%.*s:%u",
3962                         nla_len(nla[NFTA_RULE_TABLE]),
3963                         (char *)nla_data(nla[NFTA_RULE_TABLE]),
3964                         nft_net->base_seq);
3965         audit_log_nfcfg(buf, info->nfmsg->nfgen_family, 1,
3966                         AUDIT_NFT_OP_RULE_RESET, GFP_ATOMIC);
3967         kfree(buf);
3968
3969         return nfnetlink_unicast(skb2, net, portid);
3970 }
3971
3972 void nf_tables_rule_destroy(const struct nft_ctx *ctx, struct nft_rule *rule)
3973 {
3974         struct nft_expr *expr, *next;
3975
3976         /*
3977          * Careful: some expressions might not be initialized in case this
3978          * is called on error from nf_tables_newrule().
3979          */
3980         expr = nft_expr_first(rule);
3981         while (nft_expr_more(rule, expr)) {
3982                 next = nft_expr_next(expr);
3983                 nf_tables_expr_destroy(ctx, expr);
3984                 expr = next;
3985         }
3986         kfree(rule);
3987 }
3988
3989 /* can only be used if rule is no longer visible to dumps */
3990 static void nf_tables_rule_release(const struct nft_ctx *ctx, struct nft_rule *rule)
3991 {
3992         lockdep_commit_lock_is_held(ctx->net);
3993
3994         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_RELEASE);
3995         nf_tables_rule_destroy(ctx, rule);
3996 }
3997
3998 /** nft_chain_validate - loop detection and hook validation
3999  *
4000  * @ctx: context containing call depth and base chain
4001  * @chain: chain to validate
4002  *
4003  * Walk through the rules of the given chain and chase all jumps/gotos
4004  * and set lookups until either the jump limit is hit or all reachable
4005  * chains have been validated.
4006  */
4007 int nft_chain_validate(const struct nft_ctx *ctx, const struct nft_chain *chain)
4008 {
4009         struct nft_expr *expr, *last;
4010         struct nft_rule *rule;
4011         int err;
4012
4013         if (ctx->level == NFT_JUMP_STACK_SIZE)
4014                 return -EMLINK;
4015
4016         list_for_each_entry(rule, &chain->rules, list) {
4017                 if (fatal_signal_pending(current))
4018                         return -EINTR;
4019
4020                 if (!nft_is_active_next(ctx->net, rule))
4021                         continue;
4022
4023                 nft_rule_for_each_expr(expr, last, rule) {
4024                         if (!expr->ops->validate)
4025                                 continue;
4026
4027                         /* This may call nft_chain_validate() recursively,
4028                          * callers that do so must increment ctx->level.
4029                          */
4030                         err = expr->ops->validate(ctx, expr);
4031                         if (err < 0)
4032                                 return err;
4033                 }
4034         }
4035
4036         return 0;
4037 }
4038 EXPORT_SYMBOL_GPL(nft_chain_validate);
4039
4040 static int nft_table_validate(struct net *net, const struct nft_table *table)
4041 {
4042         struct nft_chain *chain;
4043         struct nft_ctx ctx = {
4044                 .net    = net,
4045                 .family = table->family,
4046         };
4047         int err;
4048
4049         list_for_each_entry(chain, &table->chains, list) {
4050                 if (!nft_is_base_chain(chain))
4051                         continue;
4052
4053                 ctx.chain = chain;
4054                 err = nft_chain_validate(&ctx, chain);
4055                 if (err < 0)
4056                         return err;
4057
4058                 cond_resched();
4059         }
4060
4061         return 0;
4062 }
4063
4064 int nft_setelem_validate(const struct nft_ctx *ctx, struct nft_set *set,
4065                          const struct nft_set_iter *iter,
4066                          struct nft_elem_priv *elem_priv)
4067 {
4068         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
4069         struct nft_ctx *pctx = (struct nft_ctx *)ctx;
4070         const struct nft_data *data;
4071         int err;
4072
4073         if (!nft_set_elem_active(ext, iter->genmask))
4074                 return 0;
4075
4076         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
4077             *nft_set_ext_flags(ext) & NFT_SET_ELEM_INTERVAL_END)
4078                 return 0;
4079
4080         data = nft_set_ext_data(ext);
4081         switch (data->verdict.code) {
4082         case NFT_JUMP:
4083         case NFT_GOTO:
4084                 pctx->level++;
4085                 err = nft_chain_validate(ctx, data->verdict.chain);
4086                 if (err < 0)
4087                         return err;
4088                 pctx->level--;
4089                 break;
4090         default:
4091                 break;
4092         }
4093
4094         return 0;
4095 }
4096
4097 int nft_set_catchall_validate(const struct nft_ctx *ctx, struct nft_set *set)
4098 {
4099         struct nft_set_iter dummy_iter = {
4100                 .genmask        = nft_genmask_next(ctx->net),
4101         };
4102         struct nft_set_elem_catchall *catchall;
4103
4104         struct nft_set_ext *ext;
4105         int ret = 0;
4106
4107         list_for_each_entry_rcu(catchall, &set->catchall_list, list,
4108                                 lockdep_commit_lock_is_held(ctx->net)) {
4109                 ext = nft_set_elem_ext(set, catchall->elem);
4110                 if (!nft_set_elem_active(ext, dummy_iter.genmask))
4111                         continue;
4112
4113                 ret = nft_setelem_validate(ctx, set, &dummy_iter, catchall->elem);
4114                 if (ret < 0)
4115                         return ret;
4116         }
4117
4118         return ret;
4119 }
4120
4121 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
4122                                              const struct nft_chain *chain,
4123                                              const struct nlattr *nla);
4124
4125 #define NFT_RULE_MAXEXPRS       128
4126
4127 static int nf_tables_newrule(struct sk_buff *skb, const struct nfnl_info *info,
4128                              const struct nlattr * const nla[])
4129 {
4130         struct nftables_pernet *nft_net = nft_pernet(info->net);
4131         struct netlink_ext_ack *extack = info->extack;
4132         unsigned int size, i, n, ulen = 0, usize = 0;
4133         u8 genmask = nft_genmask_next(info->net);
4134         struct nft_rule *rule, *old_rule = NULL;
4135         struct nft_expr_info *expr_info = NULL;
4136         u8 family = info->nfmsg->nfgen_family;
4137         struct nft_flow_rule *flow = NULL;
4138         struct net *net = info->net;
4139         struct nft_userdata *udata;
4140         struct nft_table *table;
4141         struct nft_chain *chain;
4142         struct nft_trans *trans;
4143         u64 handle, pos_handle;
4144         struct nft_expr *expr;
4145         struct nft_ctx ctx;
4146         struct nlattr *tmp;
4147         int err, rem;
4148
4149         lockdep_assert_held(&nft_net->commit_mutex);
4150
4151         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
4152                                  NETLINK_CB(skb).portid);
4153         if (IS_ERR(table)) {
4154                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
4155                 return PTR_ERR(table);
4156         }
4157
4158         if (nla[NFTA_RULE_CHAIN]) {
4159                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
4160                                          genmask);
4161                 if (IS_ERR(chain)) {
4162                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
4163                         return PTR_ERR(chain);
4164                 }
4165
4166         } else if (nla[NFTA_RULE_CHAIN_ID]) {
4167                 chain = nft_chain_lookup_byid(net, table, nla[NFTA_RULE_CHAIN_ID],
4168                                               genmask);
4169                 if (IS_ERR(chain)) {
4170                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN_ID]);
4171                         return PTR_ERR(chain);
4172                 }
4173         } else {
4174                 return -EINVAL;
4175         }
4176
4177         if (nft_chain_is_bound(chain))
4178                 return -EOPNOTSUPP;
4179
4180         if (nla[NFTA_RULE_HANDLE]) {
4181                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_HANDLE]));
4182                 rule = __nft_rule_lookup(net, chain, handle);
4183                 if (IS_ERR(rule)) {
4184                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
4185                         return PTR_ERR(rule);
4186                 }
4187
4188                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
4189                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
4190                         return -EEXIST;
4191                 }
4192                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
4193                         old_rule = rule;
4194                 else
4195                         return -EOPNOTSUPP;
4196         } else {
4197                 if (!(info->nlh->nlmsg_flags & NLM_F_CREATE) ||
4198                     info->nlh->nlmsg_flags & NLM_F_REPLACE)
4199                         return -EINVAL;
4200                 handle = nf_tables_alloc_handle(table);
4201
4202                 if (nla[NFTA_RULE_POSITION]) {
4203                         pos_handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_POSITION]));
4204                         old_rule = __nft_rule_lookup(net, chain, pos_handle);
4205                         if (IS_ERR(old_rule)) {
4206                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION]);
4207                                 return PTR_ERR(old_rule);
4208                         }
4209                 } else if (nla[NFTA_RULE_POSITION_ID]) {
4210                         old_rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_POSITION_ID]);
4211                         if (IS_ERR(old_rule)) {
4212                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION_ID]);
4213                                 return PTR_ERR(old_rule);
4214                         }
4215                 }
4216         }
4217
4218         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
4219
4220         n = 0;
4221         size = 0;
4222         if (nla[NFTA_RULE_EXPRESSIONS]) {
4223                 expr_info = kvmalloc_array(NFT_RULE_MAXEXPRS,
4224                                            sizeof(struct nft_expr_info),
4225                                            GFP_KERNEL);
4226                 if (!expr_info)
4227                         return -ENOMEM;
4228
4229                 nla_for_each_nested(tmp, nla[NFTA_RULE_EXPRESSIONS], rem) {
4230                         err = -EINVAL;
4231                         if (nla_type(tmp) != NFTA_LIST_ELEM)
4232                                 goto err_release_expr;
4233                         if (n == NFT_RULE_MAXEXPRS)
4234                                 goto err_release_expr;
4235                         err = nf_tables_expr_parse(&ctx, tmp, &expr_info[n]);
4236                         if (err < 0) {
4237                                 NL_SET_BAD_ATTR(extack, tmp);
4238                                 goto err_release_expr;
4239                         }
4240                         size += expr_info[n].ops->size;
4241                         n++;
4242                 }
4243         }
4244         /* Check for overflow of dlen field */
4245         err = -EFBIG;
4246         if (size >= 1 << 12)
4247                 goto err_release_expr;
4248
4249         if (nla[NFTA_RULE_USERDATA]) {
4250                 ulen = nla_len(nla[NFTA_RULE_USERDATA]);
4251                 if (ulen > 0)
4252                         usize = sizeof(struct nft_userdata) + ulen;
4253         }
4254
4255         err = -ENOMEM;
4256         rule = kzalloc(sizeof(*rule) + size + usize, GFP_KERNEL_ACCOUNT);
4257         if (rule == NULL)
4258                 goto err_release_expr;
4259
4260         nft_activate_next(net, rule);
4261
4262         rule->handle = handle;
4263         rule->dlen   = size;
4264         rule->udata  = ulen ? 1 : 0;
4265
4266         if (ulen) {
4267                 udata = nft_userdata(rule);
4268                 udata->len = ulen - 1;
4269                 nla_memcpy(udata->data, nla[NFTA_RULE_USERDATA], ulen);
4270         }
4271
4272         expr = nft_expr_first(rule);
4273         for (i = 0; i < n; i++) {
4274                 err = nf_tables_newexpr(&ctx, &expr_info[i], expr);
4275                 if (err < 0) {
4276                         NL_SET_BAD_ATTR(extack, expr_info[i].attr);
4277                         goto err_release_rule;
4278                 }
4279
4280                 if (expr_info[i].ops->validate)
4281                         nft_validate_state_update(table, NFT_VALIDATE_NEED);
4282
4283                 expr_info[i].ops = NULL;
4284                 expr = nft_expr_next(expr);
4285         }
4286
4287         if (chain->flags & NFT_CHAIN_HW_OFFLOAD) {
4288                 flow = nft_flow_rule_create(net, rule);
4289                 if (IS_ERR(flow)) {
4290                         err = PTR_ERR(flow);
4291                         goto err_release_rule;
4292                 }
4293         }
4294
4295         if (!nft_use_inc(&chain->use)) {
4296                 err = -EMFILE;
4297                 goto err_release_rule;
4298         }
4299
4300         if (info->nlh->nlmsg_flags & NLM_F_REPLACE) {
4301                 if (nft_chain_binding(chain)) {
4302                         err = -EOPNOTSUPP;
4303                         goto err_destroy_flow_rule;
4304                 }
4305
4306                 err = nft_delrule(&ctx, old_rule);
4307                 if (err < 0)
4308                         goto err_destroy_flow_rule;
4309
4310                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
4311                 if (trans == NULL) {
4312                         err = -ENOMEM;
4313                         goto err_destroy_flow_rule;
4314                 }
4315                 list_add_tail_rcu(&rule->list, &old_rule->list);
4316         } else {
4317                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
4318                 if (!trans) {
4319                         err = -ENOMEM;
4320                         goto err_destroy_flow_rule;
4321                 }
4322
4323                 if (info->nlh->nlmsg_flags & NLM_F_APPEND) {
4324                         if (old_rule)
4325                                 list_add_rcu(&rule->list, &old_rule->list);
4326                         else
4327                                 list_add_tail_rcu(&rule->list, &chain->rules);
4328                  } else {
4329                         if (old_rule)
4330                                 list_add_tail_rcu(&rule->list, &old_rule->list);
4331                         else
4332                                 list_add_rcu(&rule->list, &chain->rules);
4333                 }
4334         }
4335         kvfree(expr_info);
4336
4337         if (flow)
4338                 nft_trans_flow_rule(trans) = flow;
4339
4340         if (table->validate_state == NFT_VALIDATE_DO)
4341                 return nft_table_validate(net, table);
4342
4343         return 0;
4344
4345 err_destroy_flow_rule:
4346         nft_use_dec_restore(&chain->use);
4347         if (flow)
4348                 nft_flow_rule_destroy(flow);
4349 err_release_rule:
4350         nft_rule_expr_deactivate(&ctx, rule, NFT_TRANS_PREPARE_ERROR);
4351         nf_tables_rule_destroy(&ctx, rule);
4352 err_release_expr:
4353         for (i = 0; i < n; i++) {
4354                 if (expr_info[i].ops) {
4355                         module_put(expr_info[i].ops->type->owner);
4356                         if (expr_info[i].ops->type->release_ops)
4357                                 expr_info[i].ops->type->release_ops(expr_info[i].ops);
4358                 }
4359         }
4360         kvfree(expr_info);
4361
4362         return err;
4363 }
4364
4365 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
4366                                              const struct nft_chain *chain,
4367                                              const struct nlattr *nla)
4368 {
4369         struct nftables_pernet *nft_net = nft_pernet(net);
4370         u32 id = ntohl(nla_get_be32(nla));
4371         struct nft_trans *trans;
4372
4373         list_for_each_entry(trans, &nft_net->commit_list, list) {
4374                 if (trans->msg_type == NFT_MSG_NEWRULE &&
4375                     nft_trans_rule_chain(trans) == chain &&
4376                     id == nft_trans_rule_id(trans))
4377                         return nft_trans_rule(trans);
4378         }
4379         return ERR_PTR(-ENOENT);
4380 }
4381
4382 static int nf_tables_delrule(struct sk_buff *skb, const struct nfnl_info *info,
4383                              const struct nlattr * const nla[])
4384 {
4385         struct netlink_ext_ack *extack = info->extack;
4386         u8 genmask = nft_genmask_next(info->net);
4387         u8 family = info->nfmsg->nfgen_family;
4388         struct nft_chain *chain = NULL;
4389         struct net *net = info->net;
4390         struct nft_table *table;
4391         struct nft_rule *rule;
4392         struct nft_ctx ctx;
4393         int err = 0;
4394
4395         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
4396                                  NETLINK_CB(skb).portid);
4397         if (IS_ERR(table)) {
4398                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
4399                 return PTR_ERR(table);
4400         }
4401
4402         if (nla[NFTA_RULE_CHAIN]) {
4403                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
4404                                          genmask);
4405                 if (IS_ERR(chain)) {
4406                         if (PTR_ERR(chain) == -ENOENT &&
4407                             NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYRULE)
4408                                 return 0;
4409
4410                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
4411                         return PTR_ERR(chain);
4412                 }
4413                 if (nft_chain_binding(chain))
4414                         return -EOPNOTSUPP;
4415         }
4416
4417         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
4418
4419         if (chain) {
4420                 if (nla[NFTA_RULE_HANDLE]) {
4421                         rule = nft_rule_lookup(info->net, chain, nla[NFTA_RULE_HANDLE]);
4422                         if (IS_ERR(rule)) {
4423                                 if (PTR_ERR(rule) == -ENOENT &&
4424                                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYRULE)
4425                                         return 0;
4426
4427                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
4428                                 return PTR_ERR(rule);
4429                         }
4430
4431                         err = nft_delrule(&ctx, rule);
4432                 } else if (nla[NFTA_RULE_ID]) {
4433                         rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_ID]);
4434                         if (IS_ERR(rule)) {
4435                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_ID]);
4436                                 return PTR_ERR(rule);
4437                         }
4438
4439                         err = nft_delrule(&ctx, rule);
4440                 } else {
4441                         err = nft_delrule_by_chain(&ctx);
4442                 }
4443         } else {
4444                 list_for_each_entry(chain, &table->chains, list) {
4445                         if (!nft_is_active_next(net, chain))
4446                                 continue;
4447                         if (nft_chain_binding(chain))
4448                                 continue;
4449
4450                         ctx.chain = chain;
4451                         err = nft_delrule_by_chain(&ctx);
4452                         if (err < 0)
4453                                 break;
4454                 }
4455         }
4456
4457         return err;
4458 }
4459
4460 /*
4461  * Sets
4462  */
4463 static const struct nft_set_type *nft_set_types[] = {
4464         &nft_set_hash_fast_type,
4465         &nft_set_hash_type,
4466         &nft_set_rhash_type,
4467         &nft_set_bitmap_type,
4468         &nft_set_rbtree_type,
4469 #if defined(CONFIG_X86_64) && !defined(CONFIG_UML)
4470         &nft_set_pipapo_avx2_type,
4471 #endif
4472         &nft_set_pipapo_type,
4473 };
4474
4475 #define NFT_SET_FEATURES        (NFT_SET_INTERVAL | NFT_SET_MAP | \
4476                                  NFT_SET_TIMEOUT | NFT_SET_OBJECT | \
4477                                  NFT_SET_EVAL)
4478
4479 static bool nft_set_ops_candidate(const struct nft_set_type *type, u32 flags)
4480 {
4481         return (flags & type->features) == (flags & NFT_SET_FEATURES);
4482 }
4483
4484 /*
4485  * Select a set implementation based on the data characteristics and the
4486  * given policy. The total memory use might not be known if no size is
4487  * given, in that case the amount of memory per element is used.
4488  */
4489 static const struct nft_set_ops *
4490 nft_select_set_ops(const struct nft_ctx *ctx, u32 flags,
4491                    const struct nft_set_desc *desc)
4492 {
4493         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
4494         const struct nft_set_ops *ops, *bops;
4495         struct nft_set_estimate est, best;
4496         const struct nft_set_type *type;
4497         int i;
4498
4499         lockdep_assert_held(&nft_net->commit_mutex);
4500         lockdep_nfnl_nft_mutex_not_held();
4501
4502         bops        = NULL;
4503         best.size   = ~0;
4504         best.lookup = ~0;
4505         best.space  = ~0;
4506
4507         for (i = 0; i < ARRAY_SIZE(nft_set_types); i++) {
4508                 type = nft_set_types[i];
4509                 ops = &type->ops;
4510
4511                 if (!nft_set_ops_candidate(type, flags))
4512                         continue;
4513                 if (!ops->estimate(desc, flags, &est))
4514                         continue;
4515
4516                 switch (desc->policy) {
4517                 case NFT_SET_POL_PERFORMANCE:
4518                         if (est.lookup < best.lookup)
4519                                 break;
4520                         if (est.lookup == best.lookup &&
4521                             est.space < best.space)
4522                                 break;
4523                         continue;
4524                 case NFT_SET_POL_MEMORY:
4525                         if (!desc->size) {
4526                                 if (est.space < best.space)
4527                                         break;
4528                                 if (est.space == best.space &&
4529                                     est.lookup < best.lookup)
4530                                         break;
4531                         } else if (est.size < best.size || !bops) {
4532                                 break;
4533                         }
4534                         continue;
4535                 default:
4536                         break;
4537                 }
4538
4539                 bops = ops;
4540                 best = est;
4541         }
4542
4543         if (bops != NULL)
4544                 return bops;
4545
4546         return ERR_PTR(-EOPNOTSUPP);
4547 }
4548
4549 static const struct nla_policy nft_set_policy[NFTA_SET_MAX + 1] = {
4550         [NFTA_SET_TABLE]                = { .type = NLA_STRING,
4551                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
4552         [NFTA_SET_NAME]                 = { .type = NLA_STRING,
4553                                             .len = NFT_SET_MAXNAMELEN - 1 },
4554         [NFTA_SET_FLAGS]                = { .type = NLA_U32 },
4555         [NFTA_SET_KEY_TYPE]             = { .type = NLA_U32 },
4556         [NFTA_SET_KEY_LEN]              = { .type = NLA_U32 },
4557         [NFTA_SET_DATA_TYPE]            = { .type = NLA_U32 },
4558         [NFTA_SET_DATA_LEN]             = { .type = NLA_U32 },
4559         [NFTA_SET_POLICY]               = { .type = NLA_U32 },
4560         [NFTA_SET_DESC]                 = { .type = NLA_NESTED },
4561         [NFTA_SET_ID]                   = { .type = NLA_U32 },
4562         [NFTA_SET_TIMEOUT]              = { .type = NLA_U64 },
4563         [NFTA_SET_GC_INTERVAL]          = { .type = NLA_U32 },
4564         [NFTA_SET_USERDATA]             = { .type = NLA_BINARY,
4565                                             .len  = NFT_USERDATA_MAXLEN },
4566         [NFTA_SET_OBJ_TYPE]             = { .type = NLA_U32 },
4567         [NFTA_SET_HANDLE]               = { .type = NLA_U64 },
4568         [NFTA_SET_EXPR]                 = { .type = NLA_NESTED },
4569         [NFTA_SET_EXPRESSIONS]          = NLA_POLICY_NESTED_ARRAY(nft_expr_policy),
4570 };
4571
4572 static const struct nla_policy nft_concat_policy[NFTA_SET_FIELD_MAX + 1] = {
4573         [NFTA_SET_FIELD_LEN]    = { .type = NLA_U32 },
4574 };
4575
4576 static const struct nla_policy nft_set_desc_policy[NFTA_SET_DESC_MAX + 1] = {
4577         [NFTA_SET_DESC_SIZE]            = { .type = NLA_U32 },
4578         [NFTA_SET_DESC_CONCAT]          = NLA_POLICY_NESTED_ARRAY(nft_concat_policy),
4579 };
4580
4581 static struct nft_set *nft_set_lookup(const struct net *net,
4582                                       const struct nft_table *table,
4583                                       const struct nlattr *nla, u8 genmask)
4584 {
4585         struct nft_set *set;
4586
4587         if (nla == NULL)
4588                 return ERR_PTR(-EINVAL);
4589
4590         list_for_each_entry_rcu(set, &table->sets, list,
4591                                 lockdep_commit_lock_is_held(net)) {
4592                 if (!nla_strcmp(nla, set->name) &&
4593                     nft_active_genmask(set, genmask))
4594                         return set;
4595         }
4596         return ERR_PTR(-ENOENT);
4597 }
4598
4599 static struct nft_set *nft_set_lookup_byhandle(const struct nft_table *table,
4600                                                const struct nlattr *nla,
4601                                                u8 genmask)
4602 {
4603         struct nft_set *set;
4604
4605         list_for_each_entry(set, &table->sets, list) {
4606                 if (be64_to_cpu(nla_get_be64(nla)) == set->handle &&
4607                     nft_active_genmask(set, genmask))
4608                         return set;
4609         }
4610         return ERR_PTR(-ENOENT);
4611 }
4612
4613 static struct nft_set *nft_set_lookup_byid(const struct net *net,
4614                                            const struct nft_table *table,
4615                                            const struct nlattr *nla, u8 genmask)
4616 {
4617         struct nftables_pernet *nft_net = nft_pernet(net);
4618         u32 id = ntohl(nla_get_be32(nla));
4619         struct nft_trans_set *trans;
4620
4621         /* its likely the id we need is at the tail, not at start */
4622         list_for_each_entry_reverse(trans, &nft_net->commit_set_list, list_trans_newset) {
4623                 struct nft_set *set = trans->set;
4624
4625                 if (id == trans->set_id &&
4626                     set->table == table &&
4627                     nft_active_genmask(set, genmask))
4628                         return set;
4629         }
4630         return ERR_PTR(-ENOENT);
4631 }
4632
4633 struct nft_set *nft_set_lookup_global(const struct net *net,
4634                                       const struct nft_table *table,
4635                                       const struct nlattr *nla_set_name,
4636                                       const struct nlattr *nla_set_id,
4637                                       u8 genmask)
4638 {
4639         struct nft_set *set;
4640
4641         set = nft_set_lookup(net, table, nla_set_name, genmask);
4642         if (IS_ERR(set)) {
4643                 if (!nla_set_id)
4644                         return set;
4645
4646                 set = nft_set_lookup_byid(net, table, nla_set_id, genmask);
4647         }
4648         return set;
4649 }
4650 EXPORT_SYMBOL_GPL(nft_set_lookup_global);
4651
4652 static int nf_tables_set_alloc_name(struct nft_ctx *ctx, struct nft_set *set,
4653                                     const char *name)
4654 {
4655         const struct nft_set *i;
4656         const char *p;
4657         unsigned long *inuse;
4658         unsigned int n = 0, min = 0;
4659
4660         p = strchr(name, '%');
4661         if (p != NULL) {
4662                 if (p[1] != 'd' || strchr(p + 2, '%'))
4663                         return -EINVAL;
4664
4665                 if (strnlen(name, NFT_SET_MAX_ANONLEN) >= NFT_SET_MAX_ANONLEN)
4666                         return -EINVAL;
4667
4668                 inuse = (unsigned long *)get_zeroed_page(GFP_KERNEL);
4669                 if (inuse == NULL)
4670                         return -ENOMEM;
4671 cont:
4672                 list_for_each_entry(i, &ctx->table->sets, list) {
4673                         int tmp;
4674
4675                         if (!nft_is_active_next(ctx->net, i))
4676                                 continue;
4677                         if (!sscanf(i->name, name, &tmp))
4678                                 continue;
4679                         if (tmp < min || tmp >= min + BITS_PER_BYTE * PAGE_SIZE)
4680                                 continue;
4681
4682                         set_bit(tmp - min, inuse);
4683                 }
4684
4685                 n = find_first_zero_bit(inuse, BITS_PER_BYTE * PAGE_SIZE);
4686                 if (n >= BITS_PER_BYTE * PAGE_SIZE) {
4687                         min += BITS_PER_BYTE * PAGE_SIZE;
4688                         memset(inuse, 0, PAGE_SIZE);
4689                         goto cont;
4690                 }
4691                 free_page((unsigned long)inuse);
4692         }
4693
4694         set->name = kasprintf(GFP_KERNEL_ACCOUNT, name, min + n);
4695         if (!set->name)
4696                 return -ENOMEM;
4697
4698         list_for_each_entry(i, &ctx->table->sets, list) {
4699                 if (!nft_is_active_next(ctx->net, i))
4700                         continue;
4701                 if (!strcmp(set->name, i->name)) {
4702                         kfree(set->name);
4703                         set->name = NULL;
4704                         return -ENFILE;
4705                 }
4706         }
4707         return 0;
4708 }
4709
4710 int nf_msecs_to_jiffies64(const struct nlattr *nla, u64 *result)
4711 {
4712         u64 ms = be64_to_cpu(nla_get_be64(nla));
4713         u64 max = (u64)(~((u64)0));
4714
4715         max = div_u64(max, NSEC_PER_MSEC);
4716         if (ms >= max)
4717                 return -ERANGE;
4718
4719         ms *= NSEC_PER_MSEC;
4720         *result = nsecs_to_jiffies64(ms) ? : !!ms;
4721         return 0;
4722 }
4723
4724 __be64 nf_jiffies64_to_msecs(u64 input)
4725 {
4726         return cpu_to_be64(jiffies64_to_msecs(input));
4727 }
4728
4729 static int nf_tables_fill_set_concat(struct sk_buff *skb,
4730                                      const struct nft_set *set)
4731 {
4732         struct nlattr *concat, *field;
4733         int i;
4734
4735         concat = nla_nest_start_noflag(skb, NFTA_SET_DESC_CONCAT);
4736         if (!concat)
4737                 return -ENOMEM;
4738
4739         for (i = 0; i < set->field_count; i++) {
4740                 field = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
4741                 if (!field)
4742                         return -ENOMEM;
4743
4744                 if (nla_put_be32(skb, NFTA_SET_FIELD_LEN,
4745                                  htonl(set->field_len[i])))
4746                         return -ENOMEM;
4747
4748                 nla_nest_end(skb, field);
4749         }
4750
4751         nla_nest_end(skb, concat);
4752
4753         return 0;
4754 }
4755
4756 static int nf_tables_fill_set(struct sk_buff *skb, const struct nft_ctx *ctx,
4757                               const struct nft_set *set, u16 event, u16 flags)
4758 {
4759         u64 timeout = READ_ONCE(set->timeout);
4760         u32 gc_int = READ_ONCE(set->gc_int);
4761         u32 portid = ctx->portid;
4762         struct nlmsghdr *nlh;
4763         struct nlattr *nest;
4764         u32 seq = ctx->seq;
4765         int i;
4766
4767         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
4768         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
4769                            NFNETLINK_V0, nft_base_seq(ctx->net));
4770         if (!nlh)
4771                 goto nla_put_failure;
4772
4773         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
4774                 goto nla_put_failure;
4775         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
4776                 goto nla_put_failure;
4777         if (nla_put_be64(skb, NFTA_SET_HANDLE, cpu_to_be64(set->handle),
4778                          NFTA_SET_PAD))
4779                 goto nla_put_failure;
4780
4781         if (event == NFT_MSG_DELSET) {
4782                 nlmsg_end(skb, nlh);
4783                 return 0;
4784         }
4785
4786         if (set->flags != 0)
4787                 if (nla_put_be32(skb, NFTA_SET_FLAGS, htonl(set->flags)))
4788                         goto nla_put_failure;
4789
4790         if (nla_put_be32(skb, NFTA_SET_KEY_TYPE, htonl(set->ktype)))
4791                 goto nla_put_failure;
4792         if (nla_put_be32(skb, NFTA_SET_KEY_LEN, htonl(set->klen)))
4793                 goto nla_put_failure;
4794         if (set->flags & NFT_SET_MAP) {
4795                 if (nla_put_be32(skb, NFTA_SET_DATA_TYPE, htonl(set->dtype)))
4796                         goto nla_put_failure;
4797                 if (nla_put_be32(skb, NFTA_SET_DATA_LEN, htonl(set->dlen)))
4798                         goto nla_put_failure;
4799         }
4800         if (set->flags & NFT_SET_OBJECT &&
4801             nla_put_be32(skb, NFTA_SET_OBJ_TYPE, htonl(set->objtype)))
4802                 goto nla_put_failure;
4803
4804         if (timeout &&
4805             nla_put_be64(skb, NFTA_SET_TIMEOUT,
4806                          nf_jiffies64_to_msecs(timeout),
4807                          NFTA_SET_PAD))
4808                 goto nla_put_failure;
4809         if (gc_int &&
4810             nla_put_be32(skb, NFTA_SET_GC_INTERVAL, htonl(gc_int)))
4811                 goto nla_put_failure;
4812
4813         if (set->policy != NFT_SET_POL_PERFORMANCE) {
4814                 if (nla_put_be32(skb, NFTA_SET_POLICY, htonl(set->policy)))
4815                         goto nla_put_failure;
4816         }
4817
4818         if (set->udata &&
4819             nla_put(skb, NFTA_SET_USERDATA, set->udlen, set->udata))
4820                 goto nla_put_failure;
4821
4822         nest = nla_nest_start_noflag(skb, NFTA_SET_DESC);
4823         if (!nest)
4824                 goto nla_put_failure;
4825         if (set->size &&
4826             nla_put_be32(skb, NFTA_SET_DESC_SIZE, htonl(set->size)))
4827                 goto nla_put_failure;
4828
4829         if (set->field_count > 1 &&
4830             nf_tables_fill_set_concat(skb, set))
4831                 goto nla_put_failure;
4832
4833         nla_nest_end(skb, nest);
4834
4835         if (set->num_exprs == 1) {
4836                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPR);
4837                 if (nf_tables_fill_expr_info(skb, set->exprs[0], false) < 0)
4838                         goto nla_put_failure;
4839
4840                 nla_nest_end(skb, nest);
4841         } else if (set->num_exprs > 1) {
4842                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPRESSIONS);
4843                 if (nest == NULL)
4844                         goto nla_put_failure;
4845
4846                 for (i = 0; i < set->num_exprs; i++) {
4847                         if (nft_expr_dump(skb, NFTA_LIST_ELEM,
4848                                           set->exprs[i], false) < 0)
4849                                 goto nla_put_failure;
4850                 }
4851                 nla_nest_end(skb, nest);
4852         }
4853
4854         nlmsg_end(skb, nlh);
4855         return 0;
4856
4857 nla_put_failure:
4858         nlmsg_trim(skb, nlh);
4859         return -1;
4860 }
4861
4862 static void nf_tables_set_notify(const struct nft_ctx *ctx,
4863                                  const struct nft_set *set, int event,
4864                                  gfp_t gfp_flags)
4865 {
4866         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
4867         u32 portid = ctx->portid;
4868         struct sk_buff *skb;
4869         u16 flags = 0;
4870         int err;
4871
4872         if (!ctx->report &&
4873             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
4874                 return;
4875
4876         skb = nlmsg_new(NLMSG_GOODSIZE, gfp_flags);
4877         if (skb == NULL)
4878                 goto err;
4879
4880         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
4881                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
4882
4883         err = nf_tables_fill_set(skb, ctx, set, event, flags);
4884         if (err < 0) {
4885                 kfree_skb(skb);
4886                 goto err;
4887         }
4888
4889         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
4890         return;
4891 err:
4892         nfnetlink_set_err(ctx->net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
4893 }
4894
4895 static int nf_tables_dump_sets(struct sk_buff *skb, struct netlink_callback *cb)
4896 {
4897         const struct nft_set *set;
4898         unsigned int idx, s_idx = cb->args[0];
4899         struct nft_table *table, *cur_table = (struct nft_table *)cb->args[2];
4900         struct net *net = sock_net(skb->sk);
4901         struct nft_ctx *ctx = cb->data, ctx_set;
4902         struct nftables_pernet *nft_net;
4903
4904         if (cb->args[1])
4905                 return skb->len;
4906
4907         rcu_read_lock();
4908         nft_net = nft_pernet(net);
4909         cb->seq = READ_ONCE(nft_net->base_seq);
4910
4911         list_for_each_entry_rcu(table, &nft_net->tables, list) {
4912                 if (ctx->family != NFPROTO_UNSPEC &&
4913                     ctx->family != table->family)
4914                         continue;
4915
4916                 if (ctx->table && ctx->table != table)
4917                         continue;
4918
4919                 if (cur_table) {
4920                         if (cur_table != table)
4921                                 continue;
4922
4923                         cur_table = NULL;
4924                 }
4925                 idx = 0;
4926                 list_for_each_entry_rcu(set, &table->sets, list) {
4927                         if (idx < s_idx)
4928                                 goto cont;
4929                         if (!nft_is_active(net, set))
4930                                 goto cont;
4931
4932                         ctx_set = *ctx;
4933                         ctx_set.table = table;
4934                         ctx_set.family = table->family;
4935
4936                         if (nf_tables_fill_set(skb, &ctx_set, set,
4937                                                NFT_MSG_NEWSET,
4938                                                NLM_F_MULTI) < 0) {
4939                                 cb->args[0] = idx;
4940                                 cb->args[2] = (unsigned long) table;
4941                                 goto done;
4942                         }
4943                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
4944 cont:
4945                         idx++;
4946                 }
4947                 if (s_idx)
4948                         s_idx = 0;
4949         }
4950         cb->args[1] = 1;
4951 done:
4952         rcu_read_unlock();
4953         return skb->len;
4954 }
4955
4956 static int nf_tables_dump_sets_start(struct netlink_callback *cb)
4957 {
4958         struct nft_ctx *ctx_dump = NULL;
4959
4960         ctx_dump = kmemdup(cb->data, sizeof(*ctx_dump), GFP_ATOMIC);
4961         if (ctx_dump == NULL)
4962                 return -ENOMEM;
4963
4964         cb->data = ctx_dump;
4965         return 0;
4966 }
4967
4968 static int nf_tables_dump_sets_done(struct netlink_callback *cb)
4969 {
4970         kfree(cb->data);
4971         return 0;
4972 }
4973
4974 /* called with rcu_read_lock held */
4975 static int nf_tables_getset(struct sk_buff *skb, const struct nfnl_info *info,
4976                             const struct nlattr * const nla[])
4977 {
4978         struct netlink_ext_ack *extack = info->extack;
4979         u8 genmask = nft_genmask_cur(info->net);
4980         u8 family = info->nfmsg->nfgen_family;
4981         struct nft_table *table = NULL;
4982         struct net *net = info->net;
4983         const struct nft_set *set;
4984         struct sk_buff *skb2;
4985         struct nft_ctx ctx;
4986         int err;
4987
4988         if (nla[NFTA_SET_TABLE]) {
4989                 table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
4990                                          genmask, 0);
4991                 if (IS_ERR(table)) {
4992                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
4993                         return PTR_ERR(table);
4994                 }
4995         }
4996
4997         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
4998
4999         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
5000                 struct netlink_dump_control c = {
5001                         .start = nf_tables_dump_sets_start,
5002                         .dump = nf_tables_dump_sets,
5003                         .done = nf_tables_dump_sets_done,
5004                         .data = &ctx,
5005                         .module = THIS_MODULE,
5006                 };
5007
5008                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
5009         }
5010
5011         /* Only accept unspec with dump */
5012         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
5013                 return -EAFNOSUPPORT;
5014         if (!nla[NFTA_SET_TABLE])
5015                 return -EINVAL;
5016
5017         set = nft_set_lookup(net, table, nla[NFTA_SET_NAME], genmask);
5018         if (IS_ERR(set)) {
5019                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
5020                 return PTR_ERR(set);
5021         }
5022
5023         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
5024         if (skb2 == NULL)
5025                 return -ENOMEM;
5026
5027         err = nf_tables_fill_set(skb2, &ctx, set, NFT_MSG_NEWSET, 0);
5028         if (err < 0)
5029                 goto err_fill_set_info;
5030
5031         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
5032
5033 err_fill_set_info:
5034         kfree_skb(skb2);
5035         return err;
5036 }
5037
5038 static int nft_set_desc_concat_parse(const struct nlattr *attr,
5039                                      struct nft_set_desc *desc)
5040 {
5041         struct nlattr *tb[NFTA_SET_FIELD_MAX + 1];
5042         u32 len;
5043         int err;
5044
5045         if (desc->field_count >= ARRAY_SIZE(desc->field_len))
5046                 return -E2BIG;
5047
5048         err = nla_parse_nested_deprecated(tb, NFTA_SET_FIELD_MAX, attr,
5049                                           nft_concat_policy, NULL);
5050         if (err < 0)
5051                 return err;
5052
5053         if (!tb[NFTA_SET_FIELD_LEN])
5054                 return -EINVAL;
5055
5056         len = ntohl(nla_get_be32(tb[NFTA_SET_FIELD_LEN]));
5057         if (!len || len > U8_MAX)
5058                 return -EINVAL;
5059
5060         desc->field_len[desc->field_count++] = len;
5061
5062         return 0;
5063 }
5064
5065 static int nft_set_desc_concat(struct nft_set_desc *desc,
5066                                const struct nlattr *nla)
5067 {
5068         u32 num_regs = 0, key_num_regs = 0;
5069         struct nlattr *attr;
5070         int rem, err, i;
5071
5072         nla_for_each_nested(attr, nla, rem) {
5073                 if (nla_type(attr) != NFTA_LIST_ELEM)
5074                         return -EINVAL;
5075
5076                 err = nft_set_desc_concat_parse(attr, desc);
5077                 if (err < 0)
5078                         return err;
5079         }
5080
5081         for (i = 0; i < desc->field_count; i++)
5082                 num_regs += DIV_ROUND_UP(desc->field_len[i], sizeof(u32));
5083
5084         key_num_regs = DIV_ROUND_UP(desc->klen, sizeof(u32));
5085         if (key_num_regs != num_regs)
5086                 return -EINVAL;
5087
5088         if (num_regs > NFT_REG32_COUNT)
5089                 return -E2BIG;
5090
5091         return 0;
5092 }
5093
5094 static int nf_tables_set_desc_parse(struct nft_set_desc *desc,
5095                                     const struct nlattr *nla)
5096 {
5097         struct nlattr *da[NFTA_SET_DESC_MAX + 1];
5098         int err;
5099
5100         err = nla_parse_nested_deprecated(da, NFTA_SET_DESC_MAX, nla,
5101                                           nft_set_desc_policy, NULL);
5102         if (err < 0)
5103                 return err;
5104
5105         if (da[NFTA_SET_DESC_SIZE] != NULL)
5106                 desc->size = ntohl(nla_get_be32(da[NFTA_SET_DESC_SIZE]));
5107         if (da[NFTA_SET_DESC_CONCAT])
5108                 err = nft_set_desc_concat(desc, da[NFTA_SET_DESC_CONCAT]);
5109
5110         return err;
5111 }
5112
5113 static int nft_set_expr_alloc(struct nft_ctx *ctx, struct nft_set *set,
5114                               const struct nlattr * const *nla,
5115                               struct nft_expr **exprs, int *num_exprs,
5116                               u32 flags)
5117 {
5118         struct nft_expr *expr;
5119         int err, i;
5120
5121         if (nla[NFTA_SET_EXPR]) {
5122                 expr = nft_set_elem_expr_alloc(ctx, set, nla[NFTA_SET_EXPR]);
5123                 if (IS_ERR(expr)) {
5124                         err = PTR_ERR(expr);
5125                         goto err_set_expr_alloc;
5126                 }
5127                 exprs[0] = expr;
5128                 (*num_exprs)++;
5129         } else if (nla[NFTA_SET_EXPRESSIONS]) {
5130                 struct nlattr *tmp;
5131                 int left;
5132
5133                 if (!(flags & NFT_SET_EXPR)) {
5134                         err = -EINVAL;
5135                         goto err_set_expr_alloc;
5136                 }
5137                 i = 0;
5138                 nla_for_each_nested(tmp, nla[NFTA_SET_EXPRESSIONS], left) {
5139                         if (i == NFT_SET_EXPR_MAX) {
5140                                 err = -E2BIG;
5141                                 goto err_set_expr_alloc;
5142                         }
5143                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
5144                                 err = -EINVAL;
5145                                 goto err_set_expr_alloc;
5146                         }
5147                         expr = nft_set_elem_expr_alloc(ctx, set, tmp);
5148                         if (IS_ERR(expr)) {
5149                                 err = PTR_ERR(expr);
5150                                 goto err_set_expr_alloc;
5151                         }
5152                         exprs[i++] = expr;
5153                         (*num_exprs)++;
5154                 }
5155         }
5156
5157         return 0;
5158
5159 err_set_expr_alloc:
5160         for (i = 0; i < *num_exprs; i++)
5161                 nft_expr_destroy(ctx, exprs[i]);
5162
5163         return err;
5164 }
5165
5166 static bool nft_set_is_same(const struct nft_set *set,
5167                             const struct nft_set_desc *desc,
5168                             struct nft_expr *exprs[], u32 num_exprs, u32 flags)
5169 {
5170         int i;
5171
5172         if (set->ktype != desc->ktype ||
5173             set->dtype != desc->dtype ||
5174             set->flags != flags ||
5175             set->klen != desc->klen ||
5176             set->dlen != desc->dlen ||
5177             set->field_count != desc->field_count ||
5178             set->num_exprs != num_exprs)
5179                 return false;
5180
5181         for (i = 0; i < desc->field_count; i++) {
5182                 if (set->field_len[i] != desc->field_len[i])
5183                         return false;
5184         }
5185
5186         for (i = 0; i < num_exprs; i++) {
5187                 if (set->exprs[i]->ops != exprs[i]->ops)
5188                         return false;
5189         }
5190
5191         return true;
5192 }
5193
5194 static int nf_tables_newset(struct sk_buff *skb, const struct nfnl_info *info,
5195                             const struct nlattr * const nla[])
5196 {
5197         struct netlink_ext_ack *extack = info->extack;
5198         u8 genmask = nft_genmask_next(info->net);
5199         u8 family = info->nfmsg->nfgen_family;
5200         const struct nft_set_ops *ops;
5201         struct net *net = info->net;
5202         struct nft_set_desc desc;
5203         struct nft_table *table;
5204         unsigned char *udata;
5205         struct nft_set *set;
5206         struct nft_ctx ctx;
5207         size_t alloc_size;
5208         int num_exprs = 0;
5209         char *name;
5210         int err, i;
5211         u16 udlen;
5212         u32 flags;
5213         u64 size;
5214
5215         if (nla[NFTA_SET_TABLE] == NULL ||
5216             nla[NFTA_SET_NAME] == NULL ||
5217             nla[NFTA_SET_KEY_LEN] == NULL ||
5218             nla[NFTA_SET_ID] == NULL)
5219                 return -EINVAL;
5220
5221         memset(&desc, 0, sizeof(desc));
5222
5223         desc.ktype = NFT_DATA_VALUE;
5224         if (nla[NFTA_SET_KEY_TYPE] != NULL) {
5225                 desc.ktype = ntohl(nla_get_be32(nla[NFTA_SET_KEY_TYPE]));
5226                 if ((desc.ktype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK)
5227                         return -EINVAL;
5228         }
5229
5230         desc.klen = ntohl(nla_get_be32(nla[NFTA_SET_KEY_LEN]));
5231         if (desc.klen == 0 || desc.klen > NFT_DATA_VALUE_MAXLEN)
5232                 return -EINVAL;
5233
5234         flags = 0;
5235         if (nla[NFTA_SET_FLAGS] != NULL) {
5236                 flags = ntohl(nla_get_be32(nla[NFTA_SET_FLAGS]));
5237                 if (flags & ~(NFT_SET_ANONYMOUS | NFT_SET_CONSTANT |
5238                               NFT_SET_INTERVAL | NFT_SET_TIMEOUT |
5239                               NFT_SET_MAP | NFT_SET_EVAL |
5240                               NFT_SET_OBJECT | NFT_SET_CONCAT | NFT_SET_EXPR))
5241                         return -EOPNOTSUPP;
5242                 /* Only one of these operations is supported */
5243                 if ((flags & (NFT_SET_MAP | NFT_SET_OBJECT)) ==
5244                              (NFT_SET_MAP | NFT_SET_OBJECT))
5245                         return -EOPNOTSUPP;
5246                 if ((flags & (NFT_SET_EVAL | NFT_SET_OBJECT)) ==
5247                              (NFT_SET_EVAL | NFT_SET_OBJECT))
5248                         return -EOPNOTSUPP;
5249                 if ((flags & (NFT_SET_ANONYMOUS | NFT_SET_TIMEOUT | NFT_SET_EVAL)) ==
5250                              (NFT_SET_ANONYMOUS | NFT_SET_TIMEOUT))
5251                         return -EOPNOTSUPP;
5252                 if ((flags & (NFT_SET_CONSTANT | NFT_SET_TIMEOUT)) ==
5253                              (NFT_SET_CONSTANT | NFT_SET_TIMEOUT))
5254                         return -EOPNOTSUPP;
5255         }
5256
5257         desc.dtype = 0;
5258         if (nla[NFTA_SET_DATA_TYPE] != NULL) {
5259                 if (!(flags & NFT_SET_MAP))
5260                         return -EINVAL;
5261
5262                 desc.dtype = ntohl(nla_get_be32(nla[NFTA_SET_DATA_TYPE]));
5263                 if ((desc.dtype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK &&
5264                     desc.dtype != NFT_DATA_VERDICT)
5265                         return -EINVAL;
5266
5267                 if (desc.dtype != NFT_DATA_VERDICT) {
5268                         if (nla[NFTA_SET_DATA_LEN] == NULL)
5269                                 return -EINVAL;
5270                         desc.dlen = ntohl(nla_get_be32(nla[NFTA_SET_DATA_LEN]));
5271                         if (desc.dlen == 0 || desc.dlen > NFT_DATA_VALUE_MAXLEN)
5272                                 return -EINVAL;
5273                 } else
5274                         desc.dlen = sizeof(struct nft_verdict);
5275         } else if (flags & NFT_SET_MAP)
5276                 return -EINVAL;
5277
5278         if (nla[NFTA_SET_OBJ_TYPE] != NULL) {
5279                 if (!(flags & NFT_SET_OBJECT))
5280                         return -EINVAL;
5281
5282                 desc.objtype = ntohl(nla_get_be32(nla[NFTA_SET_OBJ_TYPE]));
5283                 if (desc.objtype == NFT_OBJECT_UNSPEC ||
5284                     desc.objtype > NFT_OBJECT_MAX)
5285                         return -EOPNOTSUPP;
5286         } else if (flags & NFT_SET_OBJECT)
5287                 return -EINVAL;
5288         else
5289                 desc.objtype = NFT_OBJECT_UNSPEC;
5290
5291         desc.timeout = 0;
5292         if (nla[NFTA_SET_TIMEOUT] != NULL) {
5293                 if (!(flags & NFT_SET_TIMEOUT))
5294                         return -EINVAL;
5295
5296                 if (flags & NFT_SET_ANONYMOUS)
5297                         return -EOPNOTSUPP;
5298
5299                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_TIMEOUT], &desc.timeout);
5300                 if (err)
5301                         return err;
5302         }
5303         desc.gc_int = 0;
5304         if (nla[NFTA_SET_GC_INTERVAL] != NULL) {
5305                 if (!(flags & NFT_SET_TIMEOUT))
5306                         return -EINVAL;
5307
5308                 if (flags & NFT_SET_ANONYMOUS)
5309                         return -EOPNOTSUPP;
5310
5311                 desc.gc_int = ntohl(nla_get_be32(nla[NFTA_SET_GC_INTERVAL]));
5312         }
5313
5314         desc.policy = NFT_SET_POL_PERFORMANCE;
5315         if (nla[NFTA_SET_POLICY] != NULL) {
5316                 desc.policy = ntohl(nla_get_be32(nla[NFTA_SET_POLICY]));
5317                 switch (desc.policy) {
5318                 case NFT_SET_POL_PERFORMANCE:
5319                 case NFT_SET_POL_MEMORY:
5320                         break;
5321                 default:
5322                         return -EOPNOTSUPP;
5323                 }
5324         }
5325
5326         if (nla[NFTA_SET_DESC] != NULL) {
5327                 err = nf_tables_set_desc_parse(&desc, nla[NFTA_SET_DESC]);
5328                 if (err < 0)
5329                         return err;
5330
5331                 if (desc.field_count > 1) {
5332                         if (!(flags & NFT_SET_CONCAT))
5333                                 return -EINVAL;
5334                 } else if (flags & NFT_SET_CONCAT) {
5335                         return -EINVAL;
5336                 }
5337         } else if (flags & NFT_SET_CONCAT) {
5338                 return -EINVAL;
5339         }
5340
5341         if (nla[NFTA_SET_EXPR] || nla[NFTA_SET_EXPRESSIONS])
5342                 desc.expr = true;
5343
5344         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family, genmask,
5345                                  NETLINK_CB(skb).portid);
5346         if (IS_ERR(table)) {
5347                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
5348                 return PTR_ERR(table);
5349         }
5350
5351         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
5352
5353         set = nft_set_lookup(net, table, nla[NFTA_SET_NAME], genmask);
5354         if (IS_ERR(set)) {
5355                 if (PTR_ERR(set) != -ENOENT) {
5356                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
5357                         return PTR_ERR(set);
5358                 }
5359         } else {
5360                 struct nft_expr *exprs[NFT_SET_EXPR_MAX] = {};
5361
5362                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
5363                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
5364                         return -EEXIST;
5365                 }
5366                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
5367                         return -EOPNOTSUPP;
5368
5369                 if (nft_set_is_anonymous(set))
5370                         return -EOPNOTSUPP;
5371
5372                 err = nft_set_expr_alloc(&ctx, set, nla, exprs, &num_exprs, flags);
5373                 if (err < 0)
5374                         return err;
5375
5376                 err = 0;
5377                 if (!nft_set_is_same(set, &desc, exprs, num_exprs, flags)) {
5378                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
5379                         err = -EEXIST;
5380                 }
5381
5382                 for (i = 0; i < num_exprs; i++)
5383                         nft_expr_destroy(&ctx, exprs[i]);
5384
5385                 if (err < 0)
5386                         return err;
5387
5388                 return __nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set, &desc);
5389         }
5390
5391         if (!(info->nlh->nlmsg_flags & NLM_F_CREATE))
5392                 return -ENOENT;
5393
5394         ops = nft_select_set_ops(&ctx, flags, &desc);
5395         if (IS_ERR(ops))
5396                 return PTR_ERR(ops);
5397
5398         udlen = 0;
5399         if (nla[NFTA_SET_USERDATA])
5400                 udlen = nla_len(nla[NFTA_SET_USERDATA]);
5401
5402         size = 0;
5403         if (ops->privsize != NULL)
5404                 size = ops->privsize(nla, &desc);
5405         alloc_size = sizeof(*set) + size + udlen;
5406         if (alloc_size < size || alloc_size > INT_MAX)
5407                 return -ENOMEM;
5408
5409         if (!nft_use_inc(&table->use))
5410                 return -EMFILE;
5411
5412         set = kvzalloc(alloc_size, GFP_KERNEL_ACCOUNT);
5413         if (!set) {
5414                 err = -ENOMEM;
5415                 goto err_alloc;
5416         }
5417
5418         name = nla_strdup(nla[NFTA_SET_NAME], GFP_KERNEL_ACCOUNT);
5419         if (!name) {
5420                 err = -ENOMEM;
5421                 goto err_set_name;
5422         }
5423
5424         err = nf_tables_set_alloc_name(&ctx, set, name);
5425         kfree(name);
5426         if (err < 0)
5427                 goto err_set_name;
5428
5429         udata = NULL;
5430         if (udlen) {
5431                 udata = set->data + size;
5432                 nla_memcpy(udata, nla[NFTA_SET_USERDATA], udlen);
5433         }
5434
5435         INIT_LIST_HEAD(&set->bindings);
5436         INIT_LIST_HEAD(&set->catchall_list);
5437         refcount_set(&set->refs, 1);
5438         set->table = table;
5439         write_pnet(&set->net, net);
5440         set->ops = ops;
5441         set->ktype = desc.ktype;
5442         set->klen = desc.klen;
5443         set->dtype = desc.dtype;
5444         set->objtype = desc.objtype;
5445         set->dlen = desc.dlen;
5446         set->flags = flags;
5447         set->size = desc.size;
5448         set->policy = desc.policy;
5449         set->udlen = udlen;
5450         set->udata = udata;
5451         set->timeout = desc.timeout;
5452         set->gc_int = desc.gc_int;
5453
5454         set->field_count = desc.field_count;
5455         for (i = 0; i < desc.field_count; i++)
5456                 set->field_len[i] = desc.field_len[i];
5457
5458         err = ops->init(set, &desc, nla);
5459         if (err < 0)
5460                 goto err_set_init;
5461
5462         err = nft_set_expr_alloc(&ctx, set, nla, set->exprs, &num_exprs, flags);
5463         if (err < 0)
5464                 goto err_set_destroy;
5465
5466         set->num_exprs = num_exprs;
5467         set->handle = nf_tables_alloc_handle(table);
5468         INIT_LIST_HEAD(&set->pending_update);
5469
5470         err = nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set);
5471         if (err < 0)
5472                 goto err_set_expr_alloc;
5473
5474         list_add_tail_rcu(&set->list, &table->sets);
5475
5476         return 0;
5477
5478 err_set_expr_alloc:
5479         for (i = 0; i < set->num_exprs; i++)
5480                 nft_expr_destroy(&ctx, set->exprs[i]);
5481 err_set_destroy:
5482         ops->destroy(&ctx, set);
5483 err_set_init:
5484         kfree(set->name);
5485 err_set_name:
5486         kvfree(set);
5487 err_alloc:
5488         nft_use_dec_restore(&table->use);
5489
5490         return err;
5491 }
5492
5493 static void nft_set_catchall_destroy(const struct nft_ctx *ctx,
5494                                      struct nft_set *set)
5495 {
5496         struct nft_set_elem_catchall *next, *catchall;
5497
5498         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
5499                 list_del_rcu(&catchall->list);
5500                 nf_tables_set_elem_destroy(ctx, set, catchall->elem);
5501                 kfree_rcu(catchall, rcu);
5502         }
5503 }
5504
5505 static void nft_set_put(struct nft_set *set)
5506 {
5507         if (refcount_dec_and_test(&set->refs)) {
5508                 kfree(set->name);
5509                 kvfree(set);
5510         }
5511 }
5512
5513 static void nft_set_destroy(const struct nft_ctx *ctx, struct nft_set *set)
5514 {
5515         int i;
5516
5517         if (WARN_ON(set->use > 0))
5518                 return;
5519
5520         for (i = 0; i < set->num_exprs; i++)
5521                 nft_expr_destroy(ctx, set->exprs[i]);
5522
5523         set->ops->destroy(ctx, set);
5524         nft_set_catchall_destroy(ctx, set);
5525         nft_set_put(set);
5526 }
5527
5528 static int nf_tables_delset(struct sk_buff *skb, const struct nfnl_info *info,
5529                             const struct nlattr * const nla[])
5530 {
5531         struct netlink_ext_ack *extack = info->extack;
5532         u8 genmask = nft_genmask_next(info->net);
5533         u8 family = info->nfmsg->nfgen_family;
5534         struct net *net = info->net;
5535         const struct nlattr *attr;
5536         struct nft_table *table;
5537         struct nft_set *set;
5538         struct nft_ctx ctx;
5539
5540         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
5541                 return -EAFNOSUPPORT;
5542
5543         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
5544                                  genmask, NETLINK_CB(skb).portid);
5545         if (IS_ERR(table)) {
5546                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
5547                 return PTR_ERR(table);
5548         }
5549
5550         if (nla[NFTA_SET_HANDLE]) {
5551                 attr = nla[NFTA_SET_HANDLE];
5552                 set = nft_set_lookup_byhandle(table, attr, genmask);
5553         } else {
5554                 attr = nla[NFTA_SET_NAME];
5555                 set = nft_set_lookup(net, table, attr, genmask);
5556         }
5557
5558         if (IS_ERR(set)) {
5559                 if (PTR_ERR(set) == -ENOENT &&
5560                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYSET)
5561                         return 0;
5562
5563                 NL_SET_BAD_ATTR(extack, attr);
5564                 return PTR_ERR(set);
5565         }
5566         if (set->use ||
5567             (info->nlh->nlmsg_flags & NLM_F_NONREC &&
5568              atomic_read(&set->nelems) > 0)) {
5569                 NL_SET_BAD_ATTR(extack, attr);
5570                 return -EBUSY;
5571         }
5572
5573         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
5574
5575         return nft_delset(&ctx, set);
5576 }
5577
5578 static int nft_validate_register_store(const struct nft_ctx *ctx,
5579                                        enum nft_registers reg,
5580                                        const struct nft_data *data,
5581                                        enum nft_data_types type,
5582                                        unsigned int len);
5583
5584 static int nft_setelem_data_validate(const struct nft_ctx *ctx,
5585                                      struct nft_set *set,
5586                                      struct nft_elem_priv *elem_priv)
5587 {
5588         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
5589         enum nft_registers dreg;
5590
5591         dreg = nft_type_to_reg(set->dtype);
5592         return nft_validate_register_store(ctx, dreg, nft_set_ext_data(ext),
5593                                            set->dtype == NFT_DATA_VERDICT ?
5594                                            NFT_DATA_VERDICT : NFT_DATA_VALUE,
5595                                            set->dlen);
5596 }
5597
5598 static int nf_tables_bind_check_setelem(const struct nft_ctx *ctx,
5599                                         struct nft_set *set,
5600                                         const struct nft_set_iter *iter,
5601                                         struct nft_elem_priv *elem_priv)
5602 {
5603         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
5604
5605         if (!nft_set_elem_active(ext, iter->genmask))
5606                 return 0;
5607
5608         return nft_setelem_data_validate(ctx, set, elem_priv);
5609 }
5610
5611 static int nft_set_catchall_bind_check(const struct nft_ctx *ctx,
5612                                        struct nft_set *set)
5613 {
5614         u8 genmask = nft_genmask_next(ctx->net);
5615         struct nft_set_elem_catchall *catchall;
5616         struct nft_set_ext *ext;
5617         int ret = 0;
5618
5619         list_for_each_entry_rcu(catchall, &set->catchall_list, list,
5620                                 lockdep_commit_lock_is_held(ctx->net)) {
5621                 ext = nft_set_elem_ext(set, catchall->elem);
5622                 if (!nft_set_elem_active(ext, genmask))
5623                         continue;
5624
5625                 ret = nft_setelem_data_validate(ctx, set, catchall->elem);
5626                 if (ret < 0)
5627                         break;
5628         }
5629
5630         return ret;
5631 }
5632
5633 int nf_tables_bind_set(const struct nft_ctx *ctx, struct nft_set *set,
5634                        struct nft_set_binding *binding)
5635 {
5636         struct nft_set_binding *i;
5637         struct nft_set_iter iter;
5638
5639         if (!list_empty(&set->bindings) && nft_set_is_anonymous(set))
5640                 return -EBUSY;
5641
5642         if (binding->flags & NFT_SET_MAP) {
5643                 /* If the set is already bound to the same chain all
5644                  * jumps are already validated for that chain.
5645                  */
5646                 list_for_each_entry(i, &set->bindings, list) {
5647                         if (i->flags & NFT_SET_MAP &&
5648                             i->chain == binding->chain)
5649                                 goto bind;
5650                 }
5651
5652                 iter.genmask    = nft_genmask_next(ctx->net);
5653                 iter.type       = NFT_ITER_UPDATE;
5654                 iter.skip       = 0;
5655                 iter.count      = 0;
5656                 iter.err        = 0;
5657                 iter.fn         = nf_tables_bind_check_setelem;
5658
5659                 set->ops->walk(ctx, set, &iter);
5660                 if (!iter.err)
5661                         iter.err = nft_set_catchall_bind_check(ctx, set);
5662
5663                 if (iter.err < 0)
5664                         return iter.err;
5665         }
5666 bind:
5667         if (!nft_use_inc(&set->use))
5668                 return -EMFILE;
5669
5670         binding->chain = ctx->chain;
5671         list_add_tail_rcu(&binding->list, &set->bindings);
5672         nft_set_trans_bind(ctx, set);
5673
5674         return 0;
5675 }
5676 EXPORT_SYMBOL_GPL(nf_tables_bind_set);
5677
5678 static void nf_tables_unbind_set(const struct nft_ctx *ctx, struct nft_set *set,
5679                                  struct nft_set_binding *binding, bool event)
5680 {
5681         list_del_rcu(&binding->list);
5682
5683         if (list_empty(&set->bindings) && nft_set_is_anonymous(set)) {
5684                 list_del_rcu(&set->list);
5685                 set->dead = 1;
5686                 if (event)
5687                         nf_tables_set_notify(ctx, set, NFT_MSG_DELSET,
5688                                              GFP_KERNEL);
5689         }
5690 }
5691
5692 static void nft_setelem_data_activate(const struct net *net,
5693                                       const struct nft_set *set,
5694                                       struct nft_elem_priv *elem_priv);
5695
5696 static int nft_mapelem_activate(const struct nft_ctx *ctx,
5697                                 struct nft_set *set,
5698                                 const struct nft_set_iter *iter,
5699                                 struct nft_elem_priv *elem_priv)
5700 {
5701         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
5702
5703         /* called from abort path, reverse check to undo changes. */
5704         if (nft_set_elem_active(ext, iter->genmask))
5705                 return 0;
5706
5707         nft_clear(ctx->net, ext);
5708         nft_setelem_data_activate(ctx->net, set, elem_priv);
5709
5710         return 0;
5711 }
5712
5713 static void nft_map_catchall_activate(const struct nft_ctx *ctx,
5714                                       struct nft_set *set)
5715 {
5716         u8 genmask = nft_genmask_next(ctx->net);
5717         struct nft_set_elem_catchall *catchall;
5718         struct nft_set_ext *ext;
5719
5720         list_for_each_entry(catchall, &set->catchall_list, list) {
5721                 ext = nft_set_elem_ext(set, catchall->elem);
5722                 if (!nft_set_elem_active(ext, genmask))
5723                         continue;
5724
5725                 nft_clear(ctx->net, ext);
5726                 nft_setelem_data_activate(ctx->net, set, catchall->elem);
5727                 break;
5728         }
5729 }
5730
5731 static void nft_map_activate(const struct nft_ctx *ctx, struct nft_set *set)
5732 {
5733         struct nft_set_iter iter = {
5734                 .genmask        = nft_genmask_next(ctx->net),
5735                 .type           = NFT_ITER_UPDATE,
5736                 .fn             = nft_mapelem_activate,
5737         };
5738
5739         set->ops->walk(ctx, set, &iter);
5740         WARN_ON_ONCE(iter.err);
5741
5742         nft_map_catchall_activate(ctx, set);
5743 }
5744
5745 void nf_tables_activate_set(const struct nft_ctx *ctx, struct nft_set *set)
5746 {
5747         if (nft_set_is_anonymous(set)) {
5748                 if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
5749                         nft_map_activate(ctx, set);
5750
5751                 nft_clear(ctx->net, set);
5752         }
5753
5754         nft_use_inc_restore(&set->use);
5755 }
5756 EXPORT_SYMBOL_GPL(nf_tables_activate_set);
5757
5758 void nf_tables_deactivate_set(const struct nft_ctx *ctx, struct nft_set *set,
5759                               struct nft_set_binding *binding,
5760                               enum nft_trans_phase phase)
5761 {
5762         lockdep_commit_lock_is_held(ctx->net);
5763
5764         switch (phase) {
5765         case NFT_TRANS_PREPARE_ERROR:
5766                 nft_set_trans_unbind(ctx, set);
5767                 if (nft_set_is_anonymous(set))
5768                         nft_deactivate_next(ctx->net, set);
5769                 else
5770                         list_del_rcu(&binding->list);
5771
5772                 nft_use_dec(&set->use);
5773                 break;
5774         case NFT_TRANS_PREPARE:
5775                 if (nft_set_is_anonymous(set)) {
5776                         if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
5777                                 nft_map_deactivate(ctx, set);
5778
5779                         nft_deactivate_next(ctx->net, set);
5780                 }
5781                 nft_use_dec(&set->use);
5782                 return;
5783         case NFT_TRANS_ABORT:
5784         case NFT_TRANS_RELEASE:
5785                 if (nft_set_is_anonymous(set) &&
5786                     set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
5787                         nft_map_deactivate(ctx, set);
5788
5789                 nft_use_dec(&set->use);
5790                 fallthrough;
5791         default:
5792                 nf_tables_unbind_set(ctx, set, binding,
5793                                      phase == NFT_TRANS_COMMIT);
5794         }
5795 }
5796 EXPORT_SYMBOL_GPL(nf_tables_deactivate_set);
5797
5798 void nf_tables_destroy_set(const struct nft_ctx *ctx, struct nft_set *set)
5799 {
5800         if (list_empty(&set->bindings) && nft_set_is_anonymous(set))
5801                 nft_set_destroy(ctx, set);
5802 }
5803 EXPORT_SYMBOL_GPL(nf_tables_destroy_set);
5804
5805 const struct nft_set_ext_type nft_set_ext_types[] = {
5806         [NFT_SET_EXT_KEY]               = {
5807                 .align  = __alignof__(u32),
5808         },
5809         [NFT_SET_EXT_DATA]              = {
5810                 .align  = __alignof__(u32),
5811         },
5812         [NFT_SET_EXT_EXPRESSIONS]       = {
5813                 .align  = __alignof__(struct nft_set_elem_expr),
5814         },
5815         [NFT_SET_EXT_OBJREF]            = {
5816                 .len    = sizeof(struct nft_object *),
5817                 .align  = __alignof__(struct nft_object *),
5818         },
5819         [NFT_SET_EXT_FLAGS]             = {
5820                 .len    = sizeof(u8),
5821                 .align  = __alignof__(u8),
5822         },
5823         [NFT_SET_EXT_TIMEOUT]           = {
5824                 .len    = sizeof(struct nft_timeout),
5825                 .align  = __alignof__(struct nft_timeout),
5826         },
5827         [NFT_SET_EXT_USERDATA]          = {
5828                 .len    = sizeof(struct nft_userdata),
5829                 .align  = __alignof__(struct nft_userdata),
5830         },
5831         [NFT_SET_EXT_KEY_END]           = {
5832                 .align  = __alignof__(u32),
5833         },
5834 };
5835
5836 /*
5837  * Set elements
5838  */
5839
5840 static const struct nla_policy nft_set_elem_policy[NFTA_SET_ELEM_MAX + 1] = {
5841         [NFTA_SET_ELEM_KEY]             = { .type = NLA_NESTED },
5842         [NFTA_SET_ELEM_DATA]            = { .type = NLA_NESTED },
5843         [NFTA_SET_ELEM_FLAGS]           = { .type = NLA_U32 },
5844         [NFTA_SET_ELEM_TIMEOUT]         = { .type = NLA_U64 },
5845         [NFTA_SET_ELEM_EXPIRATION]      = { .type = NLA_U64 },
5846         [NFTA_SET_ELEM_USERDATA]        = { .type = NLA_BINARY,
5847                                             .len = NFT_USERDATA_MAXLEN },
5848         [NFTA_SET_ELEM_EXPR]            = { .type = NLA_NESTED },
5849         [NFTA_SET_ELEM_OBJREF]          = { .type = NLA_STRING,
5850                                             .len = NFT_OBJ_MAXNAMELEN - 1 },
5851         [NFTA_SET_ELEM_KEY_END]         = { .type = NLA_NESTED },
5852         [NFTA_SET_ELEM_EXPRESSIONS]     = NLA_POLICY_NESTED_ARRAY(nft_expr_policy),
5853 };
5854
5855 static const struct nla_policy nft_set_elem_list_policy[NFTA_SET_ELEM_LIST_MAX + 1] = {
5856         [NFTA_SET_ELEM_LIST_TABLE]      = { .type = NLA_STRING,
5857                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
5858         [NFTA_SET_ELEM_LIST_SET]        = { .type = NLA_STRING,
5859                                             .len = NFT_SET_MAXNAMELEN - 1 },
5860         [NFTA_SET_ELEM_LIST_ELEMENTS]   = NLA_POLICY_NESTED_ARRAY(nft_set_elem_policy),
5861         [NFTA_SET_ELEM_LIST_SET_ID]     = { .type = NLA_U32 },
5862 };
5863
5864 static int nft_set_elem_expr_dump(struct sk_buff *skb,
5865                                   const struct nft_set *set,
5866                                   const struct nft_set_ext *ext,
5867                                   bool reset)
5868 {
5869         struct nft_set_elem_expr *elem_expr;
5870         u32 size, num_exprs = 0;
5871         struct nft_expr *expr;
5872         struct nlattr *nest;
5873
5874         elem_expr = nft_set_ext_expr(ext);
5875         nft_setelem_expr_foreach(expr, elem_expr, size)
5876                 num_exprs++;
5877
5878         if (num_exprs == 1) {
5879                 expr = nft_setelem_expr_at(elem_expr, 0);
5880                 if (nft_expr_dump(skb, NFTA_SET_ELEM_EXPR, expr, reset) < 0)
5881                         return -1;
5882
5883                 return 0;
5884         } else if (num_exprs > 1) {
5885                 nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_EXPRESSIONS);
5886                 if (nest == NULL)
5887                         goto nla_put_failure;
5888
5889                 nft_setelem_expr_foreach(expr, elem_expr, size) {
5890                         expr = nft_setelem_expr_at(elem_expr, size);
5891                         if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr, reset) < 0)
5892                                 goto nla_put_failure;
5893                 }
5894                 nla_nest_end(skb, nest);
5895         }
5896         return 0;
5897
5898 nla_put_failure:
5899         return -1;
5900 }
5901
5902 static int nf_tables_fill_setelem(struct sk_buff *skb,
5903                                   const struct nft_set *set,
5904                                   const struct nft_elem_priv *elem_priv,
5905                                   bool reset)
5906 {
5907         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
5908         unsigned char *b = skb_tail_pointer(skb);
5909         struct nlattr *nest;
5910
5911         nest = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
5912         if (nest == NULL)
5913                 goto nla_put_failure;
5914
5915         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
5916             nft_data_dump(skb, NFTA_SET_ELEM_KEY, nft_set_ext_key(ext),
5917                           NFT_DATA_VALUE, set->klen) < 0)
5918                 goto nla_put_failure;
5919
5920         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
5921             nft_data_dump(skb, NFTA_SET_ELEM_KEY_END, nft_set_ext_key_end(ext),
5922                           NFT_DATA_VALUE, set->klen) < 0)
5923                 goto nla_put_failure;
5924
5925         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
5926             nft_data_dump(skb, NFTA_SET_ELEM_DATA, nft_set_ext_data(ext),
5927                           nft_set_datatype(set), set->dlen) < 0)
5928                 goto nla_put_failure;
5929
5930         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS) &&
5931             nft_set_elem_expr_dump(skb, set, ext, reset))
5932                 goto nla_put_failure;
5933
5934         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
5935             nla_put_string(skb, NFTA_SET_ELEM_OBJREF,
5936                            (*nft_set_ext_obj(ext))->key.name) < 0)
5937                 goto nla_put_failure;
5938
5939         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
5940             nla_put_be32(skb, NFTA_SET_ELEM_FLAGS,
5941                          htonl(*nft_set_ext_flags(ext))))
5942                 goto nla_put_failure;
5943
5944         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT)) {
5945                 u64 timeout = READ_ONCE(nft_set_ext_timeout(ext)->timeout);
5946                 u64 set_timeout = READ_ONCE(set->timeout);
5947                 __be64 msecs = 0;
5948
5949                 if (set_timeout != timeout) {
5950                         msecs = nf_jiffies64_to_msecs(timeout);
5951                         if (nla_put_be64(skb, NFTA_SET_ELEM_TIMEOUT, msecs,
5952                                          NFTA_SET_ELEM_PAD))
5953                                 goto nla_put_failure;
5954                 }
5955
5956                 if (timeout > 0) {
5957                         u64 expires, now = get_jiffies_64();
5958
5959                         expires = READ_ONCE(nft_set_ext_timeout(ext)->expiration);
5960                         if (time_before64(now, expires))
5961                                 expires -= now;
5962                         else
5963                                 expires = 0;
5964
5965                         if (nla_put_be64(skb, NFTA_SET_ELEM_EXPIRATION,
5966                                          nf_jiffies64_to_msecs(expires),
5967                                          NFTA_SET_ELEM_PAD))
5968                                 goto nla_put_failure;
5969                 }
5970         }
5971
5972         if (nft_set_ext_exists(ext, NFT_SET_EXT_USERDATA)) {
5973                 struct nft_userdata *udata;
5974
5975                 udata = nft_set_ext_userdata(ext);
5976                 if (nla_put(skb, NFTA_SET_ELEM_USERDATA,
5977                             udata->len + 1, udata->data))
5978                         goto nla_put_failure;
5979         }
5980
5981         nla_nest_end(skb, nest);
5982         return 0;
5983
5984 nla_put_failure:
5985         nlmsg_trim(skb, b);
5986         return -EMSGSIZE;
5987 }
5988
5989 struct nft_set_dump_args {
5990         const struct netlink_callback   *cb;
5991         struct nft_set_iter             iter;
5992         struct sk_buff                  *skb;
5993         bool                            reset;
5994 };
5995
5996 static int nf_tables_dump_setelem(const struct nft_ctx *ctx,
5997                                   struct nft_set *set,
5998                                   const struct nft_set_iter *iter,
5999                                   struct nft_elem_priv *elem_priv)
6000 {
6001         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
6002         struct nft_set_dump_args *args;
6003
6004         if (!nft_set_elem_active(ext, iter->genmask))
6005                 return 0;
6006
6007         if (nft_set_elem_expired(ext) || nft_set_elem_is_dead(ext))
6008                 return 0;
6009
6010         args = container_of(iter, struct nft_set_dump_args, iter);
6011         return nf_tables_fill_setelem(args->skb, set, elem_priv, args->reset);
6012 }
6013
6014 static void audit_log_nft_set_reset(const struct nft_table *table,
6015                                     unsigned int base_seq,
6016                                     unsigned int nentries)
6017 {
6018         char *buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, base_seq);
6019
6020         audit_log_nfcfg(buf, table->family, nentries,
6021                         AUDIT_NFT_OP_SETELEM_RESET, GFP_ATOMIC);
6022         kfree(buf);
6023 }
6024
6025 struct nft_set_dump_ctx {
6026         const struct nft_set    *set;
6027         struct nft_ctx          ctx;
6028         bool                    reset;
6029 };
6030
6031 static int nft_set_catchall_dump(struct net *net, struct sk_buff *skb,
6032                                  const struct nft_set *set, bool reset,
6033                                  unsigned int base_seq)
6034 {
6035         struct nft_set_elem_catchall *catchall;
6036         u8 genmask = nft_genmask_cur(net);
6037         struct nft_set_ext *ext;
6038         int ret = 0;
6039
6040         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
6041                 ext = nft_set_elem_ext(set, catchall->elem);
6042                 if (!nft_set_elem_active(ext, genmask) ||
6043                     nft_set_elem_expired(ext))
6044                         continue;
6045
6046                 ret = nf_tables_fill_setelem(skb, set, catchall->elem, reset);
6047                 if (reset && !ret)
6048                         audit_log_nft_set_reset(set->table, base_seq, 1);
6049                 break;
6050         }
6051
6052         return ret;
6053 }
6054
6055 static int nf_tables_dump_set(struct sk_buff *skb, struct netlink_callback *cb)
6056 {
6057         struct nft_set_dump_ctx *dump_ctx = cb->data;
6058         struct net *net = sock_net(skb->sk);
6059         struct nftables_pernet *nft_net;
6060         struct nft_table *table;
6061         struct nft_set *set;
6062         struct nft_set_dump_args args;
6063         bool set_found = false;
6064         struct nlmsghdr *nlh;
6065         struct nlattr *nest;
6066         u32 portid, seq;
6067         int event;
6068
6069         rcu_read_lock();
6070         nft_net = nft_pernet(net);
6071         cb->seq = READ_ONCE(nft_net->base_seq);
6072
6073         list_for_each_entry_rcu(table, &nft_net->tables, list) {
6074                 if (dump_ctx->ctx.family != NFPROTO_UNSPEC &&
6075                     dump_ctx->ctx.family != table->family)
6076                         continue;
6077
6078                 if (table != dump_ctx->ctx.table)
6079                         continue;
6080
6081                 list_for_each_entry_rcu(set, &table->sets, list) {
6082                         if (set == dump_ctx->set) {
6083                                 set_found = true;
6084                                 break;
6085                         }
6086                 }
6087                 break;
6088         }
6089
6090         if (!set_found) {
6091                 rcu_read_unlock();
6092                 return -ENOENT;
6093         }
6094
6095         event  = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWSETELEM);
6096         portid = NETLINK_CB(cb->skb).portid;
6097         seq    = cb->nlh->nlmsg_seq;
6098
6099         nlh = nfnl_msg_put(skb, portid, seq, event, NLM_F_MULTI,
6100                            table->family, NFNETLINK_V0, nft_base_seq(net));
6101         if (!nlh)
6102                 goto nla_put_failure;
6103
6104         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_TABLE, table->name))
6105                 goto nla_put_failure;
6106         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_SET, set->name))
6107                 goto nla_put_failure;
6108
6109         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
6110         if (nest == NULL)
6111                 goto nla_put_failure;
6112
6113         args.cb                 = cb;
6114         args.skb                = skb;
6115         args.reset              = dump_ctx->reset;
6116         args.iter.genmask       = nft_genmask_cur(net);
6117         args.iter.type          = NFT_ITER_READ;
6118         args.iter.skip          = cb->args[0];
6119         args.iter.count         = 0;
6120         args.iter.err           = 0;
6121         args.iter.fn            = nf_tables_dump_setelem;
6122         set->ops->walk(&dump_ctx->ctx, set, &args.iter);
6123
6124         if (!args.iter.err && args.iter.count == cb->args[0])
6125                 args.iter.err = nft_set_catchall_dump(net, skb, set,
6126                                                       dump_ctx->reset, cb->seq);
6127         nla_nest_end(skb, nest);
6128         nlmsg_end(skb, nlh);
6129
6130         rcu_read_unlock();
6131
6132         if (args.iter.err && args.iter.err != -EMSGSIZE)
6133                 return args.iter.err;
6134         if (args.iter.count == cb->args[0])
6135                 return 0;
6136
6137         cb->args[0] = args.iter.count;
6138         return skb->len;
6139
6140 nla_put_failure:
6141         rcu_read_unlock();
6142         return -ENOSPC;
6143 }
6144
6145 static int nf_tables_dumpreset_set(struct sk_buff *skb,
6146                                    struct netlink_callback *cb)
6147 {
6148         struct nftables_pernet *nft_net = nft_pernet(sock_net(skb->sk));
6149         struct nft_set_dump_ctx *dump_ctx = cb->data;
6150         int ret, skip = cb->args[0];
6151
6152         mutex_lock(&nft_net->commit_mutex);
6153
6154         ret = nf_tables_dump_set(skb, cb);
6155
6156         if (cb->args[0] > skip)
6157                 audit_log_nft_set_reset(dump_ctx->ctx.table, cb->seq,
6158                                         cb->args[0] - skip);
6159
6160         mutex_unlock(&nft_net->commit_mutex);
6161
6162         return ret;
6163 }
6164
6165 static int nf_tables_dump_set_start(struct netlink_callback *cb)
6166 {
6167         struct nft_set_dump_ctx *dump_ctx = cb->data;
6168
6169         cb->data = kmemdup(dump_ctx, sizeof(*dump_ctx), GFP_ATOMIC);
6170
6171         return cb->data ? 0 : -ENOMEM;
6172 }
6173
6174 static int nf_tables_dump_set_done(struct netlink_callback *cb)
6175 {
6176         kfree(cb->data);
6177         return 0;
6178 }
6179
6180 static int nf_tables_fill_setelem_info(struct sk_buff *skb,
6181                                        const struct nft_ctx *ctx, u32 seq,
6182                                        u32 portid, int event, u16 flags,
6183                                        const struct nft_set *set,
6184                                        const struct nft_elem_priv *elem_priv,
6185                                        bool reset)
6186 {
6187         struct nlmsghdr *nlh;
6188         struct nlattr *nest;
6189         int err;
6190
6191         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
6192         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
6193                            NFNETLINK_V0, nft_base_seq(ctx->net));
6194         if (!nlh)
6195                 goto nla_put_failure;
6196
6197         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
6198                 goto nla_put_failure;
6199         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
6200                 goto nla_put_failure;
6201
6202         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
6203         if (nest == NULL)
6204                 goto nla_put_failure;
6205
6206         err = nf_tables_fill_setelem(skb, set, elem_priv, reset);
6207         if (err < 0)
6208                 goto nla_put_failure;
6209
6210         nla_nest_end(skb, nest);
6211
6212         nlmsg_end(skb, nlh);
6213         return 0;
6214
6215 nla_put_failure:
6216         nlmsg_trim(skb, nlh);
6217         return -1;
6218 }
6219
6220 static int nft_setelem_parse_flags(const struct nft_set *set,
6221                                    const struct nlattr *attr, u32 *flags)
6222 {
6223         if (attr == NULL)
6224                 return 0;
6225
6226         *flags = ntohl(nla_get_be32(attr));
6227         if (*flags & ~(NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
6228                 return -EOPNOTSUPP;
6229         if (!(set->flags & NFT_SET_INTERVAL) &&
6230             *flags & NFT_SET_ELEM_INTERVAL_END)
6231                 return -EINVAL;
6232         if ((*flags & (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL)) ==
6233             (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
6234                 return -EINVAL;
6235
6236         return 0;
6237 }
6238
6239 static int nft_setelem_parse_key(struct nft_ctx *ctx, const struct nft_set *set,
6240                                  struct nft_data *key, struct nlattr *attr)
6241 {
6242         struct nft_data_desc desc = {
6243                 .type   = NFT_DATA_VALUE,
6244                 .size   = NFT_DATA_VALUE_MAXLEN,
6245                 .len    = set->klen,
6246         };
6247
6248         return nft_data_init(ctx, key, &desc, attr);
6249 }
6250
6251 static int nft_setelem_parse_data(struct nft_ctx *ctx, struct nft_set *set,
6252                                   struct nft_data_desc *desc,
6253                                   struct nft_data *data,
6254                                   struct nlattr *attr)
6255 {
6256         u32 dtype;
6257
6258         if (set->dtype == NFT_DATA_VERDICT)
6259                 dtype = NFT_DATA_VERDICT;
6260         else
6261                 dtype = NFT_DATA_VALUE;
6262
6263         desc->type = dtype;
6264         desc->size = NFT_DATA_VALUE_MAXLEN;
6265         desc->len = set->dlen;
6266         desc->flags = NFT_DATA_DESC_SETELEM;
6267
6268         return nft_data_init(ctx, data, desc, attr);
6269 }
6270
6271 static void *nft_setelem_catchall_get(const struct net *net,
6272                                       const struct nft_set *set)
6273 {
6274         struct nft_set_elem_catchall *catchall;
6275         u8 genmask = nft_genmask_cur(net);
6276         struct nft_set_ext *ext;
6277         void *priv = NULL;
6278
6279         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
6280                 ext = nft_set_elem_ext(set, catchall->elem);
6281                 if (!nft_set_elem_active(ext, genmask) ||
6282                     nft_set_elem_expired(ext))
6283                         continue;
6284
6285                 priv = catchall->elem;
6286                 break;
6287         }
6288
6289         return priv;
6290 }
6291
6292 static int nft_setelem_get(struct nft_ctx *ctx, const struct nft_set *set,
6293                            struct nft_set_elem *elem, u32 flags)
6294 {
6295         void *priv;
6296
6297         if (!(flags & NFT_SET_ELEM_CATCHALL)) {
6298                 priv = set->ops->get(ctx->net, set, elem, flags);
6299                 if (IS_ERR(priv))
6300                         return PTR_ERR(priv);
6301         } else {
6302                 priv = nft_setelem_catchall_get(ctx->net, set);
6303                 if (!priv)
6304                         return -ENOENT;
6305         }
6306         elem->priv = priv;
6307
6308         return 0;
6309 }
6310
6311 static int nft_get_set_elem(struct nft_ctx *ctx, const struct nft_set *set,
6312                             const struct nlattr *attr, bool reset)
6313 {
6314         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
6315         struct nft_set_elem elem;
6316         struct sk_buff *skb;
6317         uint32_t flags = 0;
6318         int err;
6319
6320         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
6321                                           nft_set_elem_policy, NULL);
6322         if (err < 0)
6323                 return err;
6324
6325         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
6326         if (err < 0)
6327                 return err;
6328
6329         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
6330                 return -EINVAL;
6331
6332         if (nla[NFTA_SET_ELEM_KEY]) {
6333                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
6334                                             nla[NFTA_SET_ELEM_KEY]);
6335                 if (err < 0)
6336                         return err;
6337         }
6338
6339         if (nla[NFTA_SET_ELEM_KEY_END]) {
6340                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
6341                                             nla[NFTA_SET_ELEM_KEY_END]);
6342                 if (err < 0)
6343                         return err;
6344         }
6345
6346         err = nft_setelem_get(ctx, set, &elem, flags);
6347         if (err < 0)
6348                 return err;
6349
6350         err = -ENOMEM;
6351         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_ATOMIC);
6352         if (skb == NULL)
6353                 return err;
6354
6355         err = nf_tables_fill_setelem_info(skb, ctx, ctx->seq, ctx->portid,
6356                                           NFT_MSG_NEWSETELEM, 0, set, elem.priv,
6357                                           reset);
6358         if (err < 0)
6359                 goto err_fill_setelem;
6360
6361         return nfnetlink_unicast(skb, ctx->net, ctx->portid);
6362
6363 err_fill_setelem:
6364         kfree_skb(skb);
6365         return err;
6366 }
6367
6368 static int nft_set_dump_ctx_init(struct nft_set_dump_ctx *dump_ctx,
6369                                  const struct sk_buff *skb,
6370                                  const struct nfnl_info *info,
6371                                  const struct nlattr * const nla[],
6372                                  bool reset)
6373 {
6374         struct netlink_ext_ack *extack = info->extack;
6375         u8 genmask = nft_genmask_cur(info->net);
6376         u8 family = info->nfmsg->nfgen_family;
6377         struct net *net = info->net;
6378         struct nft_table *table;
6379         struct nft_set *set;
6380
6381         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
6382                                  genmask, 0);
6383         if (IS_ERR(table)) {
6384                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
6385                 return PTR_ERR(table);
6386         }
6387
6388         set = nft_set_lookup(net, table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
6389         if (IS_ERR(set)) {
6390                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_SET]);
6391                 return PTR_ERR(set);
6392         }
6393
6394         nft_ctx_init(&dump_ctx->ctx, net, skb,
6395                      info->nlh, family, table, NULL, nla);
6396         dump_ctx->set = set;
6397         dump_ctx->reset = reset;
6398         return 0;
6399 }
6400
6401 /* called with rcu_read_lock held */
6402 static int nf_tables_getsetelem(struct sk_buff *skb,
6403                                 const struct nfnl_info *info,
6404                                 const struct nlattr * const nla[])
6405 {
6406         struct netlink_ext_ack *extack = info->extack;
6407         struct nft_set_dump_ctx dump_ctx;
6408         struct nlattr *attr;
6409         int rem, err = 0;
6410
6411         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
6412                 struct netlink_dump_control c = {
6413                         .start = nf_tables_dump_set_start,
6414                         .dump = nf_tables_dump_set,
6415                         .done = nf_tables_dump_set_done,
6416                         .module = THIS_MODULE,
6417                 };
6418
6419                 err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, false);
6420                 if (err)
6421                         return err;
6422
6423                 c.data = &dump_ctx;
6424                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
6425         }
6426
6427         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
6428                 return -EINVAL;
6429
6430         err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, false);
6431         if (err)
6432                 return err;
6433
6434         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
6435                 err = nft_get_set_elem(&dump_ctx.ctx, dump_ctx.set, attr, false);
6436                 if (err < 0) {
6437                         NL_SET_BAD_ATTR(extack, attr);
6438                         break;
6439                 }
6440         }
6441
6442         return err;
6443 }
6444
6445 static int nf_tables_getsetelem_reset(struct sk_buff *skb,
6446                                       const struct nfnl_info *info,
6447                                       const struct nlattr * const nla[])
6448 {
6449         struct nftables_pernet *nft_net = nft_pernet(info->net);
6450         struct netlink_ext_ack *extack = info->extack;
6451         struct nft_set_dump_ctx dump_ctx;
6452         int rem, err = 0, nelems = 0;
6453         struct nlattr *attr;
6454
6455         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
6456                 struct netlink_dump_control c = {
6457                         .start = nf_tables_dump_set_start,
6458                         .dump = nf_tables_dumpreset_set,
6459                         .done = nf_tables_dump_set_done,
6460                         .module = THIS_MODULE,
6461                 };
6462
6463                 err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, true);
6464                 if (err)
6465                         return err;
6466
6467                 c.data = &dump_ctx;
6468                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
6469         }
6470
6471         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
6472                 return -EINVAL;
6473
6474         if (!try_module_get(THIS_MODULE))
6475                 return -EINVAL;
6476         rcu_read_unlock();
6477         mutex_lock(&nft_net->commit_mutex);
6478         rcu_read_lock();
6479
6480         err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, true);
6481         if (err)
6482                 goto out_unlock;
6483
6484         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
6485                 err = nft_get_set_elem(&dump_ctx.ctx, dump_ctx.set, attr, true);
6486                 if (err < 0) {
6487                         NL_SET_BAD_ATTR(extack, attr);
6488                         break;
6489                 }
6490                 nelems++;
6491         }
6492         audit_log_nft_set_reset(dump_ctx.ctx.table, nft_net->base_seq, nelems);
6493
6494 out_unlock:
6495         rcu_read_unlock();
6496         mutex_unlock(&nft_net->commit_mutex);
6497         rcu_read_lock();
6498         module_put(THIS_MODULE);
6499
6500         return err;
6501 }
6502
6503 static void nf_tables_setelem_notify(const struct nft_ctx *ctx,
6504                                      const struct nft_set *set,
6505                                      const struct nft_elem_priv *elem_priv,
6506                                      int event)
6507 {
6508         struct nftables_pernet *nft_net;
6509         struct net *net = ctx->net;
6510         u32 portid = ctx->portid;
6511         struct sk_buff *skb;
6512         u16 flags = 0;
6513         int err;
6514
6515         if (!ctx->report && !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
6516                 return;
6517
6518         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
6519         if (skb == NULL)
6520                 goto err;
6521
6522         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
6523                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
6524
6525         err = nf_tables_fill_setelem_info(skb, ctx, 0, portid, event, flags,
6526                                           set, elem_priv, false);
6527         if (err < 0) {
6528                 kfree_skb(skb);
6529                 goto err;
6530         }
6531
6532         nft_net = nft_pernet(net);
6533         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
6534         return;
6535 err:
6536         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
6537 }
6538
6539 static struct nft_trans *nft_trans_elem_alloc(const struct nft_ctx *ctx,
6540                                               int msg_type,
6541                                               struct nft_set *set)
6542 {
6543         struct nft_trans_elem *te;
6544         struct nft_trans *trans;
6545
6546         trans = nft_trans_alloc(ctx, msg_type, struct_size(te, elems, 1));
6547         if (trans == NULL)
6548                 return NULL;
6549
6550         te = nft_trans_container_elem(trans);
6551         te->nelems = 1;
6552         te->set = set;
6553
6554         return trans;
6555 }
6556
6557 struct nft_expr *nft_set_elem_expr_alloc(const struct nft_ctx *ctx,
6558                                          const struct nft_set *set,
6559                                          const struct nlattr *attr)
6560 {
6561         struct nft_expr *expr;
6562         int err;
6563
6564         expr = nft_expr_init(ctx, attr);
6565         if (IS_ERR(expr))
6566                 return expr;
6567
6568         err = -EOPNOTSUPP;
6569         if (expr->ops->type->flags & NFT_EXPR_GC) {
6570                 if (set->flags & NFT_SET_TIMEOUT)
6571                         goto err_set_elem_expr;
6572                 if (!set->ops->gc_init)
6573                         goto err_set_elem_expr;
6574                 set->ops->gc_init(set);
6575         }
6576
6577         return expr;
6578
6579 err_set_elem_expr:
6580         nft_expr_destroy(ctx, expr);
6581         return ERR_PTR(err);
6582 }
6583
6584 static int nft_set_ext_check(const struct nft_set_ext_tmpl *tmpl, u8 id, u32 len)
6585 {
6586         len += nft_set_ext_types[id].len;
6587         if (len > tmpl->ext_len[id] ||
6588             len > U8_MAX)
6589                 return -1;
6590
6591         return 0;
6592 }
6593
6594 static int nft_set_ext_memcpy(const struct nft_set_ext_tmpl *tmpl, u8 id,
6595                               void *to, const void *from, u32 len)
6596 {
6597         if (nft_set_ext_check(tmpl, id, len) < 0)
6598                 return -1;
6599
6600         memcpy(to, from, len);
6601
6602         return 0;
6603 }
6604
6605 struct nft_elem_priv *nft_set_elem_init(const struct nft_set *set,
6606                                         const struct nft_set_ext_tmpl *tmpl,
6607                                         const u32 *key, const u32 *key_end,
6608                                         const u32 *data,
6609                                         u64 timeout, u64 expiration, gfp_t gfp)
6610 {
6611         struct nft_set_ext *ext;
6612         void *elem;
6613
6614         elem = kzalloc(set->ops->elemsize + tmpl->len, gfp);
6615         if (elem == NULL)
6616                 return ERR_PTR(-ENOMEM);
6617
6618         ext = nft_set_elem_ext(set, elem);
6619         nft_set_ext_init(ext, tmpl);
6620
6621         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
6622             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY,
6623                                nft_set_ext_key(ext), key, set->klen) < 0)
6624                 goto err_ext_check;
6625
6626         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
6627             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY_END,
6628                                nft_set_ext_key_end(ext), key_end, set->klen) < 0)
6629                 goto err_ext_check;
6630
6631         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
6632             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_DATA,
6633                                nft_set_ext_data(ext), data, set->dlen) < 0)
6634                 goto err_ext_check;
6635
6636         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT)) {
6637                 nft_set_ext_timeout(ext)->timeout = timeout;
6638
6639                 if (expiration == 0)
6640                         expiration = timeout;
6641
6642                 nft_set_ext_timeout(ext)->expiration = get_jiffies_64() + expiration;
6643         }
6644
6645         return elem;
6646
6647 err_ext_check:
6648         kfree(elem);
6649
6650         return ERR_PTR(-EINVAL);
6651 }
6652
6653 static void __nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
6654                                         struct nft_expr *expr)
6655 {
6656         if (expr->ops->destroy_clone) {
6657                 expr->ops->destroy_clone(ctx, expr);
6658                 module_put(expr->ops->type->owner);
6659         } else {
6660                 nf_tables_expr_destroy(ctx, expr);
6661         }
6662 }
6663
6664 static void nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
6665                                       struct nft_set_elem_expr *elem_expr)
6666 {
6667         struct nft_expr *expr;
6668         u32 size;
6669
6670         nft_setelem_expr_foreach(expr, elem_expr, size)
6671                 __nft_set_elem_expr_destroy(ctx, expr);
6672 }
6673
6674 /* Drop references and destroy. Called from gc, dynset and abort path. */
6675 static void __nft_set_elem_destroy(const struct nft_ctx *ctx,
6676                                    const struct nft_set *set,
6677                                    const struct nft_elem_priv *elem_priv,
6678                                    bool destroy_expr)
6679 {
6680         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
6681
6682         nft_data_release(nft_set_ext_key(ext), NFT_DATA_VALUE);
6683         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
6684                 nft_data_release(nft_set_ext_data(ext), set->dtype);
6685         if (destroy_expr && nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
6686                 nft_set_elem_expr_destroy(ctx, nft_set_ext_expr(ext));
6687         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
6688                 nft_use_dec(&(*nft_set_ext_obj(ext))->use);
6689
6690         kfree(elem_priv);
6691 }
6692
6693 /* Drop references and destroy. Called from gc and dynset. */
6694 void nft_set_elem_destroy(const struct nft_set *set,
6695                           const struct nft_elem_priv *elem_priv,
6696                           bool destroy_expr)
6697 {
6698         struct nft_ctx ctx = {
6699                 .net    = read_pnet(&set->net),
6700                 .family = set->table->family,
6701         };
6702
6703         __nft_set_elem_destroy(&ctx, set, elem_priv, destroy_expr);
6704 }
6705 EXPORT_SYMBOL_GPL(nft_set_elem_destroy);
6706
6707 /* Drop references and destroy. Called from abort path. */
6708 static void nft_trans_set_elem_destroy(const struct nft_ctx *ctx, struct nft_trans_elem *te)
6709 {
6710         int i;
6711
6712         for (i = 0; i < te->nelems; i++) {
6713                 /* skip update request, see nft_trans_elems_new_abort() */
6714                 if (!te->elems[i].priv)
6715                         continue;
6716
6717                 __nft_set_elem_destroy(ctx, te->set, te->elems[i].priv, true);
6718         }
6719 }
6720
6721 /* Destroy element. References have been already dropped in the preparation
6722  * path via nft_setelem_data_deactivate().
6723  */
6724 void nf_tables_set_elem_destroy(const struct nft_ctx *ctx,
6725                                 const struct nft_set *set,
6726                                 const struct nft_elem_priv *elem_priv)
6727 {
6728         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
6729
6730         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
6731                 nft_set_elem_expr_destroy(ctx, nft_set_ext_expr(ext));
6732
6733         kfree(elem_priv);
6734 }
6735
6736 static void nft_trans_elems_destroy(const struct nft_ctx *ctx,
6737                                     const struct nft_trans_elem *te)
6738 {
6739         int i;
6740
6741         for (i = 0; i < te->nelems; i++)
6742                 nf_tables_set_elem_destroy(ctx, te->set, te->elems[i].priv);
6743 }
6744
6745 int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set,
6746                             struct nft_expr *expr_array[])
6747 {
6748         struct nft_expr *expr;
6749         int err, i, k;
6750
6751         for (i = 0; i < set->num_exprs; i++) {
6752                 expr = kzalloc(set->exprs[i]->ops->size, GFP_KERNEL_ACCOUNT);
6753                 if (!expr)
6754                         goto err_expr;
6755
6756                 err = nft_expr_clone(expr, set->exprs[i], GFP_KERNEL_ACCOUNT);
6757                 if (err < 0) {
6758                         kfree(expr);
6759                         goto err_expr;
6760                 }
6761                 expr_array[i] = expr;
6762         }
6763
6764         return 0;
6765
6766 err_expr:
6767         for (k = i - 1; k >= 0; k--)
6768                 nft_expr_destroy(ctx, expr_array[k]);
6769
6770         return -ENOMEM;
6771 }
6772
6773 static int nft_set_elem_expr_setup(struct nft_ctx *ctx,
6774                                    const struct nft_set_ext_tmpl *tmpl,
6775                                    const struct nft_set_ext *ext,
6776                                    struct nft_expr *expr_array[],
6777                                    u32 num_exprs)
6778 {
6779         struct nft_set_elem_expr *elem_expr = nft_set_ext_expr(ext);
6780         u32 len = sizeof(struct nft_set_elem_expr);
6781         struct nft_expr *expr;
6782         int i, err;
6783
6784         if (num_exprs == 0)
6785                 return 0;
6786
6787         for (i = 0; i < num_exprs; i++)
6788                 len += expr_array[i]->ops->size;
6789
6790         if (nft_set_ext_check(tmpl, NFT_SET_EXT_EXPRESSIONS, len) < 0)
6791                 return -EINVAL;
6792
6793         for (i = 0; i < num_exprs; i++) {
6794                 expr = nft_setelem_expr_at(elem_expr, elem_expr->size);
6795                 err = nft_expr_clone(expr, expr_array[i], GFP_KERNEL_ACCOUNT);
6796                 if (err < 0)
6797                         goto err_elem_expr_setup;
6798
6799                 elem_expr->size += expr_array[i]->ops->size;
6800                 nft_expr_destroy(ctx, expr_array[i]);
6801                 expr_array[i] = NULL;
6802         }
6803
6804         return 0;
6805
6806 err_elem_expr_setup:
6807         for (; i < num_exprs; i++) {
6808                 nft_expr_destroy(ctx, expr_array[i]);
6809                 expr_array[i] = NULL;
6810         }
6811
6812         return -ENOMEM;
6813 }
6814
6815 struct nft_set_ext *nft_set_catchall_lookup(const struct net *net,
6816                                             const struct nft_set *set)
6817 {
6818         struct nft_set_elem_catchall *catchall;
6819         u8 genmask = nft_genmask_cur(net);
6820         struct nft_set_ext *ext;
6821
6822         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
6823                 ext = nft_set_elem_ext(set, catchall->elem);
6824                 if (nft_set_elem_active(ext, genmask) &&
6825                     !nft_set_elem_expired(ext) &&
6826                     !nft_set_elem_is_dead(ext))
6827                         return ext;
6828         }
6829
6830         return NULL;
6831 }
6832 EXPORT_SYMBOL_GPL(nft_set_catchall_lookup);
6833
6834 static int nft_setelem_catchall_insert(const struct net *net,
6835                                        struct nft_set *set,
6836                                        const struct nft_set_elem *elem,
6837                                        struct nft_elem_priv **priv)
6838 {
6839         struct nft_set_elem_catchall *catchall;
6840         u8 genmask = nft_genmask_next(net);
6841         struct nft_set_ext *ext;
6842
6843         list_for_each_entry(catchall, &set->catchall_list, list) {
6844                 ext = nft_set_elem_ext(set, catchall->elem);
6845                 if (nft_set_elem_active(ext, genmask)) {
6846                         *priv = catchall->elem;
6847                         return -EEXIST;
6848                 }
6849         }
6850
6851         catchall = kmalloc(sizeof(*catchall), GFP_KERNEL_ACCOUNT);
6852         if (!catchall)
6853                 return -ENOMEM;
6854
6855         catchall->elem = elem->priv;
6856         list_add_tail_rcu(&catchall->list, &set->catchall_list);
6857
6858         return 0;
6859 }
6860
6861 static int nft_setelem_insert(const struct net *net,
6862                               struct nft_set *set,
6863                               const struct nft_set_elem *elem,
6864                               struct nft_elem_priv **elem_priv,
6865                               unsigned int flags)
6866 {
6867         int ret;
6868
6869         if (flags & NFT_SET_ELEM_CATCHALL)
6870                 ret = nft_setelem_catchall_insert(net, set, elem, elem_priv);
6871         else
6872                 ret = set->ops->insert(net, set, elem, elem_priv);
6873
6874         return ret;
6875 }
6876
6877 static bool nft_setelem_is_catchall(const struct nft_set *set,
6878                                     const struct nft_elem_priv *elem_priv)
6879 {
6880         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
6881
6882         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
6883             *nft_set_ext_flags(ext) & NFT_SET_ELEM_CATCHALL)
6884                 return true;
6885
6886         return false;
6887 }
6888
6889 static void nft_setelem_activate(struct net *net, struct nft_set *set,
6890                                  struct nft_elem_priv *elem_priv)
6891 {
6892         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
6893
6894         if (nft_setelem_is_catchall(set, elem_priv)) {
6895                 nft_clear(net, ext);
6896         } else {
6897                 set->ops->activate(net, set, elem_priv);
6898         }
6899 }
6900
6901 static void nft_trans_elem_update(const struct nft_set *set,
6902                                   const struct nft_trans_one_elem *elem)
6903 {
6904         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6905         const struct nft_elem_update *update = elem->update;
6906
6907         if (update->flags & NFT_TRANS_UPD_TIMEOUT)
6908                 WRITE_ONCE(nft_set_ext_timeout(ext)->timeout, update->timeout);
6909
6910         if (update->flags & NFT_TRANS_UPD_EXPIRATION)
6911                 WRITE_ONCE(nft_set_ext_timeout(ext)->expiration, get_jiffies_64() + update->expiration);
6912 }
6913
6914 static void nft_trans_elems_add(const struct nft_ctx *ctx,
6915                                 struct nft_trans_elem *te)
6916 {
6917         int i;
6918
6919         for (i = 0; i < te->nelems; i++) {
6920                 struct nft_trans_one_elem *elem = &te->elems[i];
6921
6922                 if (elem->update)
6923                         nft_trans_elem_update(te->set, elem);
6924                 else
6925                         nft_setelem_activate(ctx->net, te->set, elem->priv);
6926
6927                 nf_tables_setelem_notify(ctx, te->set, elem->priv,
6928                                          NFT_MSG_NEWSETELEM);
6929                 kfree(elem->update);
6930         }
6931 }
6932
6933 static int nft_setelem_catchall_deactivate(const struct net *net,
6934                                            struct nft_set *set,
6935                                            struct nft_set_elem *elem)
6936 {
6937         struct nft_set_elem_catchall *catchall;
6938         struct nft_set_ext *ext;
6939
6940         list_for_each_entry(catchall, &set->catchall_list, list) {
6941                 ext = nft_set_elem_ext(set, catchall->elem);
6942                 if (!nft_is_active_next(net, ext))
6943                         continue;
6944
6945                 kfree(elem->priv);
6946                 elem->priv = catchall->elem;
6947                 nft_set_elem_change_active(net, set, ext);
6948                 return 0;
6949         }
6950
6951         return -ENOENT;
6952 }
6953
6954 static int __nft_setelem_deactivate(const struct net *net,
6955                                     struct nft_set *set,
6956                                     struct nft_set_elem *elem)
6957 {
6958         void *priv;
6959
6960         priv = set->ops->deactivate(net, set, elem);
6961         if (!priv)
6962                 return -ENOENT;
6963
6964         kfree(elem->priv);
6965         elem->priv = priv;
6966         set->ndeact++;
6967
6968         return 0;
6969 }
6970
6971 static int nft_setelem_deactivate(const struct net *net,
6972                                   struct nft_set *set,
6973                                   struct nft_set_elem *elem, u32 flags)
6974 {
6975         int ret;
6976
6977         if (flags & NFT_SET_ELEM_CATCHALL)
6978                 ret = nft_setelem_catchall_deactivate(net, set, elem);
6979         else
6980                 ret = __nft_setelem_deactivate(net, set, elem);
6981
6982         return ret;
6983 }
6984
6985 static void nft_setelem_catchall_destroy(struct nft_set_elem_catchall *catchall)
6986 {
6987         list_del_rcu(&catchall->list);
6988         kfree_rcu(catchall, rcu);
6989 }
6990
6991 static void nft_setelem_catchall_remove(const struct net *net,
6992                                         const struct nft_set *set,
6993                                         struct nft_elem_priv *elem_priv)
6994 {
6995         struct nft_set_elem_catchall *catchall, *next;
6996
6997         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
6998                 if (catchall->elem == elem_priv) {
6999                         nft_setelem_catchall_destroy(catchall);
7000                         break;
7001                 }
7002         }
7003 }
7004
7005 static void nft_setelem_remove(const struct net *net,
7006                                const struct nft_set *set,
7007                                struct nft_elem_priv *elem_priv)
7008 {
7009         if (nft_setelem_is_catchall(set, elem_priv))
7010                 nft_setelem_catchall_remove(net, set, elem_priv);
7011         else
7012                 set->ops->remove(net, set, elem_priv);
7013 }
7014
7015 static void nft_trans_elems_remove(const struct nft_ctx *ctx,
7016                                    const struct nft_trans_elem *te)
7017 {
7018         int i;
7019
7020         for (i = 0; i < te->nelems; i++) {
7021                 WARN_ON_ONCE(te->elems[i].update);
7022
7023                 nf_tables_setelem_notify(ctx, te->set,
7024                                          te->elems[i].priv,
7025                                          te->nft_trans.msg_type);
7026
7027                 nft_setelem_remove(ctx->net, te->set, te->elems[i].priv);
7028                 if (!nft_setelem_is_catchall(te->set, te->elems[i].priv)) {
7029                         atomic_dec(&te->set->nelems);
7030                         te->set->ndeact--;
7031                 }
7032         }
7033 }
7034
7035 static bool nft_setelem_valid_key_end(const struct nft_set *set,
7036                                       struct nlattr **nla, u32 flags)
7037 {
7038         if ((set->flags & (NFT_SET_CONCAT | NFT_SET_INTERVAL)) ==
7039                           (NFT_SET_CONCAT | NFT_SET_INTERVAL)) {
7040                 if (flags & NFT_SET_ELEM_INTERVAL_END)
7041                         return false;
7042
7043                 if (nla[NFTA_SET_ELEM_KEY_END] &&
7044                     flags & NFT_SET_ELEM_CATCHALL)
7045                         return false;
7046         } else {
7047                 if (nla[NFTA_SET_ELEM_KEY_END])
7048                         return false;
7049         }
7050
7051         return true;
7052 }
7053
7054 static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set,
7055                             const struct nlattr *attr, u32 nlmsg_flags)
7056 {
7057         struct nft_expr *expr_array[NFT_SET_EXPR_MAX] = {};
7058         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
7059         u8 genmask = nft_genmask_next(ctx->net);
7060         u32 flags = 0, size = 0, num_exprs = 0;
7061         struct nft_set_ext_tmpl tmpl;
7062         struct nft_set_ext *ext, *ext2;
7063         struct nft_set_elem elem;
7064         struct nft_set_binding *binding;
7065         struct nft_elem_priv *elem_priv;
7066         struct nft_object *obj = NULL;
7067         struct nft_userdata *udata;
7068         struct nft_data_desc desc;
7069         enum nft_registers dreg;
7070         struct nft_trans *trans;
7071         u64 expiration;
7072         u64 timeout;
7073         int err, i;
7074         u8 ulen;
7075
7076         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
7077                                           nft_set_elem_policy, NULL);
7078         if (err < 0)
7079                 return err;
7080
7081         nft_set_ext_prepare(&tmpl);
7082
7083         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
7084         if (err < 0)
7085                 return err;
7086
7087         if (((flags & NFT_SET_ELEM_CATCHALL) && nla[NFTA_SET_ELEM_KEY]) ||
7088             (!(flags & NFT_SET_ELEM_CATCHALL) && !nla[NFTA_SET_ELEM_KEY]))
7089                 return -EINVAL;
7090
7091         if (flags != 0) {
7092                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
7093                 if (err < 0)
7094                         return err;
7095         }
7096
7097         if (set->flags & NFT_SET_MAP) {
7098                 if (nla[NFTA_SET_ELEM_DATA] == NULL &&
7099                     !(flags & NFT_SET_ELEM_INTERVAL_END))
7100                         return -EINVAL;
7101         } else {
7102                 if (nla[NFTA_SET_ELEM_DATA] != NULL)
7103                         return -EINVAL;
7104         }
7105
7106         if (set->flags & NFT_SET_OBJECT) {
7107                 if (!nla[NFTA_SET_ELEM_OBJREF] &&
7108                     !(flags & NFT_SET_ELEM_INTERVAL_END))
7109                         return -EINVAL;
7110         } else {
7111                 if (nla[NFTA_SET_ELEM_OBJREF])
7112                         return -EINVAL;
7113         }
7114
7115         if (!nft_setelem_valid_key_end(set, nla, flags))
7116                 return -EINVAL;
7117
7118         if ((flags & NFT_SET_ELEM_INTERVAL_END) &&
7119              (nla[NFTA_SET_ELEM_DATA] ||
7120               nla[NFTA_SET_ELEM_OBJREF] ||
7121               nla[NFTA_SET_ELEM_TIMEOUT] ||
7122               nla[NFTA_SET_ELEM_EXPIRATION] ||
7123               nla[NFTA_SET_ELEM_USERDATA] ||
7124               nla[NFTA_SET_ELEM_EXPR] ||
7125               nla[NFTA_SET_ELEM_KEY_END] ||
7126               nla[NFTA_SET_ELEM_EXPRESSIONS]))
7127                 return -EINVAL;
7128
7129         timeout = 0;
7130         if (nla[NFTA_SET_ELEM_TIMEOUT] != NULL) {
7131                 if (!(set->flags & NFT_SET_TIMEOUT))
7132                         return -EINVAL;
7133                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_TIMEOUT],
7134                                             &timeout);
7135                 if (err)
7136                         return err;
7137         } else if (set->flags & NFT_SET_TIMEOUT &&
7138                    !(flags & NFT_SET_ELEM_INTERVAL_END)) {
7139                 timeout = set->timeout;
7140         }
7141
7142         expiration = 0;
7143         if (nla[NFTA_SET_ELEM_EXPIRATION] != NULL) {
7144                 if (!(set->flags & NFT_SET_TIMEOUT))
7145                         return -EINVAL;
7146                 if (timeout == 0)
7147                         return -EOPNOTSUPP;
7148
7149                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_EXPIRATION],
7150                                             &expiration);
7151                 if (err)
7152                         return err;
7153
7154                 if (expiration > timeout)
7155                         return -ERANGE;
7156         }
7157
7158         if (nla[NFTA_SET_ELEM_EXPR]) {
7159                 struct nft_expr *expr;
7160
7161                 if (set->num_exprs && set->num_exprs != 1)
7162                         return -EOPNOTSUPP;
7163
7164                 expr = nft_set_elem_expr_alloc(ctx, set,
7165                                                nla[NFTA_SET_ELEM_EXPR]);
7166                 if (IS_ERR(expr))
7167                         return PTR_ERR(expr);
7168
7169                 expr_array[0] = expr;
7170                 num_exprs = 1;
7171
7172                 if (set->num_exprs && set->exprs[0]->ops != expr->ops) {
7173                         err = -EOPNOTSUPP;
7174                         goto err_set_elem_expr;
7175                 }
7176         } else if (nla[NFTA_SET_ELEM_EXPRESSIONS]) {
7177                 struct nft_expr *expr;
7178                 struct nlattr *tmp;
7179                 int left;
7180
7181                 i = 0;
7182                 nla_for_each_nested(tmp, nla[NFTA_SET_ELEM_EXPRESSIONS], left) {
7183                         if (i == NFT_SET_EXPR_MAX ||
7184                             (set->num_exprs && set->num_exprs == i)) {
7185                                 err = -E2BIG;
7186                                 goto err_set_elem_expr;
7187                         }
7188                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
7189                                 err = -EINVAL;
7190                                 goto err_set_elem_expr;
7191                         }
7192                         expr = nft_set_elem_expr_alloc(ctx, set, tmp);
7193                         if (IS_ERR(expr)) {
7194                                 err = PTR_ERR(expr);
7195                                 goto err_set_elem_expr;
7196                         }
7197                         expr_array[i] = expr;
7198                         num_exprs++;
7199
7200                         if (set->num_exprs && expr->ops != set->exprs[i]->ops) {
7201                                 err = -EOPNOTSUPP;
7202                                 goto err_set_elem_expr;
7203                         }
7204                         i++;
7205                 }
7206                 if (set->num_exprs && set->num_exprs != i) {
7207                         err = -EOPNOTSUPP;
7208                         goto err_set_elem_expr;
7209                 }
7210         } else if (set->num_exprs > 0 &&
7211                    !(flags & NFT_SET_ELEM_INTERVAL_END)) {
7212                 err = nft_set_elem_expr_clone(ctx, set, expr_array);
7213                 if (err < 0)
7214                         goto err_set_elem_expr_clone;
7215
7216                 num_exprs = set->num_exprs;
7217         }
7218
7219         if (nla[NFTA_SET_ELEM_KEY]) {
7220                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
7221                                             nla[NFTA_SET_ELEM_KEY]);
7222                 if (err < 0)
7223                         goto err_set_elem_expr;
7224
7225                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
7226                 if (err < 0)
7227                         goto err_parse_key;
7228         }
7229
7230         if (nla[NFTA_SET_ELEM_KEY_END]) {
7231                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
7232                                             nla[NFTA_SET_ELEM_KEY_END]);
7233                 if (err < 0)
7234                         goto err_parse_key;
7235
7236                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
7237                 if (err < 0)
7238                         goto err_parse_key_end;
7239         }
7240
7241         if (set->flags & NFT_SET_TIMEOUT) {
7242                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_TIMEOUT);
7243                 if (err < 0)
7244                         goto err_parse_key_end;
7245         }
7246
7247         if (num_exprs) {
7248                 for (i = 0; i < num_exprs; i++)
7249                         size += expr_array[i]->ops->size;
7250
7251                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_EXPRESSIONS,
7252                                              sizeof(struct nft_set_elem_expr) + size);
7253                 if (err < 0)
7254                         goto err_parse_key_end;
7255         }
7256
7257         if (nla[NFTA_SET_ELEM_OBJREF] != NULL) {
7258                 obj = nft_obj_lookup(ctx->net, ctx->table,
7259                                      nla[NFTA_SET_ELEM_OBJREF],
7260                                      set->objtype, genmask);
7261                 if (IS_ERR(obj)) {
7262                         err = PTR_ERR(obj);
7263                         obj = NULL;
7264                         goto err_parse_key_end;
7265                 }
7266
7267                 if (!nft_use_inc(&obj->use)) {
7268                         err = -EMFILE;
7269                         obj = NULL;
7270                         goto err_parse_key_end;
7271                 }
7272
7273                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_OBJREF);
7274                 if (err < 0)
7275                         goto err_parse_key_end;
7276         }
7277
7278         if (nla[NFTA_SET_ELEM_DATA] != NULL) {
7279                 err = nft_setelem_parse_data(ctx, set, &desc, &elem.data.val,
7280                                              nla[NFTA_SET_ELEM_DATA]);
7281                 if (err < 0)
7282                         goto err_parse_key_end;
7283
7284                 dreg = nft_type_to_reg(set->dtype);
7285                 list_for_each_entry(binding, &set->bindings, list) {
7286                         struct nft_ctx bind_ctx = {
7287                                 .net    = ctx->net,
7288                                 .family = ctx->family,
7289                                 .table  = ctx->table,
7290                                 .chain  = (struct nft_chain *)binding->chain,
7291                         };
7292
7293                         if (!(binding->flags & NFT_SET_MAP))
7294                                 continue;
7295
7296                         err = nft_validate_register_store(&bind_ctx, dreg,
7297                                                           &elem.data.val,
7298                                                           desc.type, desc.len);
7299                         if (err < 0)
7300                                 goto err_parse_data;
7301
7302                         if (desc.type == NFT_DATA_VERDICT &&
7303                             (elem.data.val.verdict.code == NFT_GOTO ||
7304                              elem.data.val.verdict.code == NFT_JUMP))
7305                                 nft_validate_state_update(ctx->table,
7306                                                           NFT_VALIDATE_NEED);
7307                 }
7308
7309                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_DATA, desc.len);
7310                 if (err < 0)
7311                         goto err_parse_data;
7312         }
7313
7314         /* The full maximum length of userdata can exceed the maximum
7315          * offset value (U8_MAX) for following extensions, therefor it
7316          * must be the last extension added.
7317          */
7318         ulen = 0;
7319         if (nla[NFTA_SET_ELEM_USERDATA] != NULL) {
7320                 ulen = nla_len(nla[NFTA_SET_ELEM_USERDATA]);
7321                 if (ulen > 0) {
7322                         err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_USERDATA,
7323                                                      ulen);
7324                         if (err < 0)
7325                                 goto err_parse_data;
7326                 }
7327         }
7328
7329         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
7330                                       elem.key_end.val.data, elem.data.val.data,
7331                                       timeout, expiration, GFP_KERNEL_ACCOUNT);
7332         if (IS_ERR(elem.priv)) {
7333                 err = PTR_ERR(elem.priv);
7334                 goto err_parse_data;
7335         }
7336
7337         ext = nft_set_elem_ext(set, elem.priv);
7338         if (flags)
7339                 *nft_set_ext_flags(ext) = flags;
7340
7341         if (obj)
7342                 *nft_set_ext_obj(ext) = obj;
7343
7344         if (ulen > 0) {
7345                 if (nft_set_ext_check(&tmpl, NFT_SET_EXT_USERDATA, ulen) < 0) {
7346                         err = -EINVAL;
7347                         goto err_elem_free;
7348                 }
7349                 udata = nft_set_ext_userdata(ext);
7350                 udata->len = ulen - 1;
7351                 nla_memcpy(&udata->data, nla[NFTA_SET_ELEM_USERDATA], ulen);
7352         }
7353         err = nft_set_elem_expr_setup(ctx, &tmpl, ext, expr_array, num_exprs);
7354         if (err < 0)
7355                 goto err_elem_free;
7356
7357         trans = nft_trans_elem_alloc(ctx, NFT_MSG_NEWSETELEM, set);
7358         if (trans == NULL) {
7359                 err = -ENOMEM;
7360                 goto err_elem_free;
7361         }
7362
7363         ext->genmask = nft_genmask_cur(ctx->net);
7364
7365         err = nft_setelem_insert(ctx->net, set, &elem, &elem_priv, flags);
7366         if (err) {
7367                 if (err == -EEXIST) {
7368                         ext2 = nft_set_elem_ext(set, elem_priv);
7369                         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) ^
7370                             nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) ||
7371                             nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) ^
7372                             nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF))
7373                                 goto err_element_clash;
7374                         if ((nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
7375                              nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) &&
7376                              memcmp(nft_set_ext_data(ext),
7377                                     nft_set_ext_data(ext2), set->dlen) != 0) ||
7378                             (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
7379                              nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF) &&
7380                              *nft_set_ext_obj(ext) != *nft_set_ext_obj(ext2)))
7381                                 goto err_element_clash;
7382                         else if (!(nlmsg_flags & NLM_F_EXCL)) {
7383                                 err = 0;
7384                                 if (nft_set_ext_exists(ext2, NFT_SET_EXT_TIMEOUT)) {
7385                                         struct nft_elem_update update = { };
7386
7387                                         if (timeout != nft_set_ext_timeout(ext2)->timeout) {
7388                                                 update.timeout = timeout;
7389                                                 if (expiration == 0)
7390                                                         expiration = timeout;
7391
7392                                                 update.flags |= NFT_TRANS_UPD_TIMEOUT;
7393                                         }
7394                                         if (expiration) {
7395                                                 update.expiration = expiration;
7396                                                 update.flags |= NFT_TRANS_UPD_EXPIRATION;
7397                                         }
7398
7399                                         if (update.flags) {
7400                                                 struct nft_trans_one_elem *ue;
7401
7402                                                 ue = &nft_trans_container_elem(trans)->elems[0];
7403
7404                                                 ue->update = kmemdup(&update, sizeof(update), GFP_KERNEL);
7405                                                 if (!ue->update) {
7406                                                         err = -ENOMEM;
7407                                                         goto err_element_clash;
7408                                                 }
7409
7410                                                 ue->priv = elem_priv;
7411                                                 nft_trans_commit_list_add_elem(ctx->net, trans, GFP_KERNEL);
7412                                                 goto err_elem_free;
7413                                         }
7414                                 }
7415                         }
7416                 } else if (err == -ENOTEMPTY) {
7417                         /* ENOTEMPTY reports overlapping between this element
7418                          * and an existing one.
7419                          */
7420                         err = -EEXIST;
7421                 }
7422                 goto err_element_clash;
7423         }
7424
7425         if (!(flags & NFT_SET_ELEM_CATCHALL)) {
7426                 unsigned int max = set->size ? set->size + set->ndeact : UINT_MAX;
7427
7428                 if (!atomic_add_unless(&set->nelems, 1, max)) {
7429                         err = -ENFILE;
7430                         goto err_set_full;
7431                 }
7432         }
7433
7434         nft_trans_container_elem(trans)->elems[0].priv = elem.priv;
7435         nft_trans_commit_list_add_elem(ctx->net, trans, GFP_KERNEL);
7436         return 0;
7437
7438 err_set_full:
7439         nft_setelem_remove(ctx->net, set, elem.priv);
7440 err_element_clash:
7441         kfree(trans);
7442 err_elem_free:
7443         nf_tables_set_elem_destroy(ctx, set, elem.priv);
7444 err_parse_data:
7445         if (nla[NFTA_SET_ELEM_DATA] != NULL)
7446                 nft_data_release(&elem.data.val, desc.type);
7447 err_parse_key_end:
7448         if (obj)
7449                 nft_use_dec_restore(&obj->use);
7450
7451         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
7452 err_parse_key:
7453         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
7454 err_set_elem_expr:
7455         for (i = 0; i < num_exprs && expr_array[i]; i++)
7456                 nft_expr_destroy(ctx, expr_array[i]);
7457 err_set_elem_expr_clone:
7458         return err;
7459 }
7460
7461 static int nf_tables_newsetelem(struct sk_buff *skb,
7462                                 const struct nfnl_info *info,
7463                                 const struct nlattr * const nla[])
7464 {
7465         struct netlink_ext_ack *extack = info->extack;
7466         u8 genmask = nft_genmask_next(info->net);
7467         u8 family = info->nfmsg->nfgen_family;
7468         struct net *net = info->net;
7469         const struct nlattr *attr;
7470         struct nft_table *table;
7471         struct nft_set *set;
7472         struct nft_ctx ctx;
7473         int rem, err;
7474
7475         if (nla[NFTA_SET_ELEM_LIST_ELEMENTS] == NULL)
7476                 return -EINVAL;
7477
7478         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
7479                                  genmask, NETLINK_CB(skb).portid);
7480         if (IS_ERR(table)) {
7481                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
7482                 return PTR_ERR(table);
7483         }
7484
7485         set = nft_set_lookup_global(net, table, nla[NFTA_SET_ELEM_LIST_SET],
7486                                     nla[NFTA_SET_ELEM_LIST_SET_ID], genmask);
7487         if (IS_ERR(set)) {
7488                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_SET]);
7489                 return PTR_ERR(set);
7490         }
7491
7492         if (!list_empty(&set->bindings) &&
7493             (set->flags & (NFT_SET_CONSTANT | NFT_SET_ANONYMOUS)))
7494                 return -EBUSY;
7495
7496         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7497
7498         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
7499                 err = nft_add_set_elem(&ctx, set, attr, info->nlh->nlmsg_flags);
7500                 if (err < 0) {
7501                         NL_SET_BAD_ATTR(extack, attr);
7502                         return err;
7503                 }
7504         }
7505
7506         if (table->validate_state == NFT_VALIDATE_DO)
7507                 return nft_table_validate(net, table);
7508
7509         return 0;
7510 }
7511
7512 /**
7513  *      nft_data_hold - hold a nft_data item
7514  *
7515  *      @data: struct nft_data to release
7516  *      @type: type of data
7517  *
7518  *      Hold a nft_data item. NFT_DATA_VALUE types can be silently discarded,
7519  *      NFT_DATA_VERDICT bumps the reference to chains in case of NFT_JUMP and
7520  *      NFT_GOTO verdicts. This function must be called on active data objects
7521  *      from the second phase of the commit protocol.
7522  */
7523 void nft_data_hold(const struct nft_data *data, enum nft_data_types type)
7524 {
7525         struct nft_chain *chain;
7526
7527         if (type == NFT_DATA_VERDICT) {
7528                 switch (data->verdict.code) {
7529                 case NFT_JUMP:
7530                 case NFT_GOTO:
7531                         chain = data->verdict.chain;
7532                         nft_use_inc_restore(&chain->use);
7533                         break;
7534                 }
7535         }
7536 }
7537
7538 static int nft_setelem_active_next(const struct net *net,
7539                                    const struct nft_set *set,
7540                                    struct nft_elem_priv *elem_priv)
7541 {
7542         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
7543         u8 genmask = nft_genmask_next(net);
7544
7545         return nft_set_elem_active(ext, genmask);
7546 }
7547
7548 static void nft_setelem_data_activate(const struct net *net,
7549                                       const struct nft_set *set,
7550                                       struct nft_elem_priv *elem_priv)
7551 {
7552         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
7553
7554         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
7555                 nft_data_hold(nft_set_ext_data(ext), set->dtype);
7556         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
7557                 nft_use_inc_restore(&(*nft_set_ext_obj(ext))->use);
7558 }
7559
7560 void nft_setelem_data_deactivate(const struct net *net,
7561                                  const struct nft_set *set,
7562                                  struct nft_elem_priv *elem_priv)
7563 {
7564         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
7565
7566         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
7567                 nft_data_release(nft_set_ext_data(ext), set->dtype);
7568         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
7569                 nft_use_dec(&(*nft_set_ext_obj(ext))->use);
7570 }
7571
7572 /* similar to nft_trans_elems_remove, but called from abort path to undo newsetelem.
7573  * No notifications and no ndeact changes.
7574  *
7575  * Returns true if set had been added to (i.e., elements need to be removed again).
7576  */
7577 static bool nft_trans_elems_new_abort(const struct nft_ctx *ctx,
7578                                       struct nft_trans_elem *te)
7579 {
7580         bool removed = false;
7581         int i;
7582
7583         for (i = 0; i < te->nelems; i++) {
7584                 if (te->elems[i].update) {
7585                         kfree(te->elems[i].update);
7586                         te->elems[i].update = NULL;
7587                         /* Update request, so do not release this element */
7588                         te->elems[i].priv = NULL;
7589                         continue;
7590                 }
7591
7592                 if (!te->set->ops->abort || nft_setelem_is_catchall(te->set, te->elems[i].priv))
7593                         nft_setelem_remove(ctx->net, te->set, te->elems[i].priv);
7594
7595                 if (!nft_setelem_is_catchall(te->set, te->elems[i].priv))
7596                         atomic_dec(&te->set->nelems);
7597
7598                 removed = true;
7599         }
7600
7601         return removed;
7602 }
7603
7604 /* Called from abort path to undo DELSETELEM/DESTROYSETELEM. */
7605 static void nft_trans_elems_destroy_abort(const struct nft_ctx *ctx,
7606                                           const struct nft_trans_elem *te)
7607 {
7608         int i;
7609
7610         for (i = 0; i < te->nelems; i++) {
7611                 if (!nft_setelem_active_next(ctx->net, te->set, te->elems[i].priv)) {
7612                         nft_setelem_data_activate(ctx->net, te->set, te->elems[i].priv);
7613                         nft_setelem_activate(ctx->net, te->set, te->elems[i].priv);
7614                 }
7615
7616                 if (!nft_setelem_is_catchall(te->set, te->elems[i].priv))
7617                         te->set->ndeact--;
7618         }
7619 }
7620
7621 static int nft_del_setelem(struct nft_ctx *ctx, struct nft_set *set,
7622                            const struct nlattr *attr)
7623 {
7624         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
7625         struct nft_set_ext_tmpl tmpl;
7626         struct nft_set_elem elem;
7627         struct nft_set_ext *ext;
7628         struct nft_trans *trans;
7629         u32 flags = 0;
7630         int err;
7631
7632         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
7633                                           nft_set_elem_policy, NULL);
7634         if (err < 0)
7635                 return err;
7636
7637         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
7638         if (err < 0)
7639                 return err;
7640
7641         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
7642                 return -EINVAL;
7643
7644         if (!nft_setelem_valid_key_end(set, nla, flags))
7645                 return -EINVAL;
7646
7647         nft_set_ext_prepare(&tmpl);
7648
7649         if (flags != 0) {
7650                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
7651                 if (err < 0)
7652                         return err;
7653         }
7654
7655         if (nla[NFTA_SET_ELEM_KEY]) {
7656                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
7657                                             nla[NFTA_SET_ELEM_KEY]);
7658                 if (err < 0)
7659                         return err;
7660
7661                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
7662                 if (err < 0)
7663                         goto fail_elem;
7664         }
7665
7666         if (nla[NFTA_SET_ELEM_KEY_END]) {
7667                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
7668                                             nla[NFTA_SET_ELEM_KEY_END]);
7669                 if (err < 0)
7670                         goto fail_elem;
7671
7672                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
7673                 if (err < 0)
7674                         goto fail_elem_key_end;
7675         }
7676
7677         err = -ENOMEM;
7678         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
7679                                       elem.key_end.val.data, NULL, 0, 0,
7680                                       GFP_KERNEL_ACCOUNT);
7681         if (IS_ERR(elem.priv)) {
7682                 err = PTR_ERR(elem.priv);
7683                 goto fail_elem_key_end;
7684         }
7685
7686         ext = nft_set_elem_ext(set, elem.priv);
7687         if (flags)
7688                 *nft_set_ext_flags(ext) = flags;
7689
7690         trans = nft_trans_elem_alloc(ctx, NFT_MSG_DELSETELEM, set);
7691         if (trans == NULL)
7692                 goto fail_trans;
7693
7694         err = nft_setelem_deactivate(ctx->net, set, &elem, flags);
7695         if (err < 0)
7696                 goto fail_ops;
7697
7698         nft_setelem_data_deactivate(ctx->net, set, elem.priv);
7699
7700         nft_trans_container_elem(trans)->elems[0].priv = elem.priv;
7701         nft_trans_commit_list_add_elem(ctx->net, trans, GFP_KERNEL);
7702         return 0;
7703
7704 fail_ops:
7705         kfree(trans);
7706 fail_trans:
7707         kfree(elem.priv);
7708 fail_elem_key_end:
7709         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
7710 fail_elem:
7711         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
7712         return err;
7713 }
7714
7715 static int nft_setelem_flush(const struct nft_ctx *ctx,
7716                              struct nft_set *set,
7717                              const struct nft_set_iter *iter,
7718                              struct nft_elem_priv *elem_priv)
7719 {
7720         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
7721         struct nft_trans *trans;
7722
7723         if (!nft_set_elem_active(ext, iter->genmask))
7724                 return 0;
7725
7726         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
7727                                     struct_size_t(struct nft_trans_elem, elems, 1),
7728                                     GFP_ATOMIC);
7729         if (!trans)
7730                 return -ENOMEM;
7731
7732         set->ops->flush(ctx->net, set, elem_priv);
7733         set->ndeact++;
7734
7735         nft_setelem_data_deactivate(ctx->net, set, elem_priv);
7736         nft_trans_elem_set(trans) = set;
7737         nft_trans_container_elem(trans)->nelems = 1;
7738         nft_trans_container_elem(trans)->elems[0].priv = elem_priv;
7739         nft_trans_commit_list_add_elem(ctx->net, trans, GFP_ATOMIC);
7740
7741         return 0;
7742 }
7743
7744 static int __nft_set_catchall_flush(const struct nft_ctx *ctx,
7745                                     struct nft_set *set,
7746                                     struct nft_elem_priv *elem_priv)
7747 {
7748         struct nft_trans *trans;
7749
7750         trans = nft_trans_elem_alloc(ctx, NFT_MSG_DELSETELEM, set);
7751         if (!trans)
7752                 return -ENOMEM;
7753
7754         nft_setelem_data_deactivate(ctx->net, set, elem_priv);
7755         nft_trans_container_elem(trans)->elems[0].priv = elem_priv;
7756         nft_trans_commit_list_add_elem(ctx->net, trans, GFP_KERNEL);
7757
7758         return 0;
7759 }
7760
7761 static int nft_set_catchall_flush(const struct nft_ctx *ctx,
7762                                   struct nft_set *set)
7763 {
7764         u8 genmask = nft_genmask_next(ctx->net);
7765         struct nft_set_elem_catchall *catchall;
7766         struct nft_set_ext *ext;
7767         int ret = 0;
7768
7769         list_for_each_entry_rcu(catchall, &set->catchall_list, list,
7770                                 lockdep_commit_lock_is_held(ctx->net)) {
7771                 ext = nft_set_elem_ext(set, catchall->elem);
7772                 if (!nft_set_elem_active(ext, genmask))
7773                         continue;
7774
7775                 ret = __nft_set_catchall_flush(ctx, set, catchall->elem);
7776                 if (ret < 0)
7777                         break;
7778                 nft_set_elem_change_active(ctx->net, set, ext);
7779         }
7780
7781         return ret;
7782 }
7783
7784 static int nft_set_flush(struct nft_ctx *ctx, struct nft_set *set, u8 genmask)
7785 {
7786         struct nft_set_iter iter = {
7787                 .genmask        = genmask,
7788                 .type           = NFT_ITER_UPDATE,
7789                 .fn             = nft_setelem_flush,
7790         };
7791
7792         set->ops->walk(ctx, set, &iter);
7793         if (!iter.err)
7794                 iter.err = nft_set_catchall_flush(ctx, set);
7795
7796         return iter.err;
7797 }
7798
7799 static int nf_tables_delsetelem(struct sk_buff *skb,
7800                                 const struct nfnl_info *info,
7801                                 const struct nlattr * const nla[])
7802 {
7803         struct netlink_ext_ack *extack = info->extack;
7804         u8 genmask = nft_genmask_next(info->net);
7805         u8 family = info->nfmsg->nfgen_family;
7806         struct net *net = info->net;
7807         const struct nlattr *attr;
7808         struct nft_table *table;
7809         struct nft_set *set;
7810         struct nft_ctx ctx;
7811         int rem, err = 0;
7812
7813         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
7814                                  genmask, NETLINK_CB(skb).portid);
7815         if (IS_ERR(table)) {
7816                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
7817                 return PTR_ERR(table);
7818         }
7819
7820         set = nft_set_lookup(net, table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
7821         if (IS_ERR(set)) {
7822                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_SET]);
7823                 return PTR_ERR(set);
7824         }
7825
7826         if (nft_set_is_anonymous(set))
7827                 return -EOPNOTSUPP;
7828
7829         if (!list_empty(&set->bindings) && (set->flags & NFT_SET_CONSTANT))
7830                 return -EBUSY;
7831
7832         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7833
7834         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
7835                 return nft_set_flush(&ctx, set, genmask);
7836
7837         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
7838                 err = nft_del_setelem(&ctx, set, attr);
7839                 if (err == -ENOENT &&
7840                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYSETELEM)
7841                         continue;
7842
7843                 if (err < 0) {
7844                         NL_SET_BAD_ATTR(extack, attr);
7845                         return err;
7846                 }
7847         }
7848
7849         return 0;
7850 }
7851
7852 /*
7853  * Stateful objects
7854  */
7855
7856 /**
7857  *      nft_register_obj- register nf_tables stateful object type
7858  *      @obj_type: object type
7859  *
7860  *      Registers the object type for use with nf_tables. Returns zero on
7861  *      success or a negative errno code otherwise.
7862  */
7863 int nft_register_obj(struct nft_object_type *obj_type)
7864 {
7865         if (obj_type->type == NFT_OBJECT_UNSPEC)
7866                 return -EINVAL;
7867
7868         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7869         list_add_rcu(&obj_type->list, &nf_tables_objects);
7870         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7871         return 0;
7872 }
7873 EXPORT_SYMBOL_GPL(nft_register_obj);
7874
7875 /**
7876  *      nft_unregister_obj - unregister nf_tables object type
7877  *      @obj_type: object type
7878  *
7879  *      Unregisters the object type for use with nf_tables.
7880  */
7881 void nft_unregister_obj(struct nft_object_type *obj_type)
7882 {
7883         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7884         list_del_rcu(&obj_type->list);
7885         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7886 }
7887 EXPORT_SYMBOL_GPL(nft_unregister_obj);
7888
7889 struct nft_object *nft_obj_lookup(const struct net *net,
7890                                   const struct nft_table *table,
7891                                   const struct nlattr *nla, u32 objtype,
7892                                   u8 genmask)
7893 {
7894         struct nft_object_hash_key k = { .table = table };
7895         char search[NFT_OBJ_MAXNAMELEN];
7896         struct rhlist_head *tmp, *list;
7897         struct nft_object *obj;
7898
7899         nla_strscpy(search, nla, sizeof(search));
7900         k.name = search;
7901
7902         WARN_ON_ONCE(!rcu_read_lock_held() &&
7903                      !lockdep_commit_lock_is_held(net));
7904
7905         rcu_read_lock();
7906         list = rhltable_lookup(&nft_objname_ht, &k, nft_objname_ht_params);
7907         if (!list)
7908                 goto out;
7909
7910         rhl_for_each_entry_rcu(obj, tmp, list, rhlhead) {
7911                 if (objtype == obj->ops->type->type &&
7912                     nft_active_genmask(obj, genmask)) {
7913                         rcu_read_unlock();
7914                         return obj;
7915                 }
7916         }
7917 out:
7918         rcu_read_unlock();
7919         return ERR_PTR(-ENOENT);
7920 }
7921 EXPORT_SYMBOL_GPL(nft_obj_lookup);
7922
7923 static struct nft_object *nft_obj_lookup_byhandle(const struct nft_table *table,
7924                                                   const struct nlattr *nla,
7925                                                   u32 objtype, u8 genmask)
7926 {
7927         struct nft_object *obj;
7928
7929         list_for_each_entry(obj, &table->objects, list) {
7930                 if (be64_to_cpu(nla_get_be64(nla)) == obj->handle &&
7931                     objtype == obj->ops->type->type &&
7932                     nft_active_genmask(obj, genmask))
7933                         return obj;
7934         }
7935         return ERR_PTR(-ENOENT);
7936 }
7937
7938 static const struct nla_policy nft_obj_policy[NFTA_OBJ_MAX + 1] = {
7939         [NFTA_OBJ_TABLE]        = { .type = NLA_STRING,
7940                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
7941         [NFTA_OBJ_NAME]         = { .type = NLA_STRING,
7942                                     .len = NFT_OBJ_MAXNAMELEN - 1 },
7943         [NFTA_OBJ_TYPE]         = { .type = NLA_U32 },
7944         [NFTA_OBJ_DATA]         = { .type = NLA_NESTED },
7945         [NFTA_OBJ_HANDLE]       = { .type = NLA_U64},
7946         [NFTA_OBJ_USERDATA]     = { .type = NLA_BINARY,
7947                                     .len = NFT_USERDATA_MAXLEN },
7948 };
7949
7950 static struct nft_object *nft_obj_init(const struct nft_ctx *ctx,
7951                                        const struct nft_object_type *type,
7952                                        const struct nlattr *attr)
7953 {
7954         struct nlattr **tb;
7955         const struct nft_object_ops *ops;
7956         struct nft_object *obj;
7957         int err = -ENOMEM;
7958
7959         tb = kmalloc_array(type->maxattr + 1, sizeof(*tb), GFP_KERNEL);
7960         if (!tb)
7961                 goto err1;
7962
7963         if (attr) {
7964                 err = nla_parse_nested_deprecated(tb, type->maxattr, attr,
7965                                                   type->policy, NULL);
7966                 if (err < 0)
7967                         goto err2;
7968         } else {
7969                 memset(tb, 0, sizeof(tb[0]) * (type->maxattr + 1));
7970         }
7971
7972         if (type->select_ops) {
7973                 ops = type->select_ops(ctx, (const struct nlattr * const *)tb);
7974                 if (IS_ERR(ops)) {
7975                         err = PTR_ERR(ops);
7976                         goto err2;
7977                 }
7978         } else {
7979                 ops = type->ops;
7980         }
7981
7982         err = -ENOMEM;
7983         obj = kzalloc(sizeof(*obj) + ops->size, GFP_KERNEL_ACCOUNT);
7984         if (!obj)
7985                 goto err2;
7986
7987         err = ops->init(ctx, (const struct nlattr * const *)tb, obj);
7988         if (err < 0)
7989                 goto err3;
7990
7991         obj->ops = ops;
7992
7993         kfree(tb);
7994         return obj;
7995 err3:
7996         kfree(obj);
7997 err2:
7998         kfree(tb);
7999 err1:
8000         return ERR_PTR(err);
8001 }
8002
8003 static int nft_object_dump(struct sk_buff *skb, unsigned int attr,
8004                            struct nft_object *obj, bool reset)
8005 {
8006         struct nlattr *nest;
8007
8008         nest = nla_nest_start_noflag(skb, attr);
8009         if (!nest)
8010                 goto nla_put_failure;
8011         if (obj->ops->dump(skb, obj, reset) < 0)
8012                 goto nla_put_failure;
8013         nla_nest_end(skb, nest);
8014         return 0;
8015
8016 nla_put_failure:
8017         return -1;
8018 }
8019
8020 static const struct nft_object_type *__nft_obj_type_get(u32 objtype, u8 family)
8021 {
8022         const struct nft_object_type *type;
8023
8024         list_for_each_entry_rcu(type, &nf_tables_objects, list) {
8025                 if (type->family != NFPROTO_UNSPEC &&
8026                     type->family != family)
8027                         continue;
8028
8029                 if (objtype == type->type)
8030                         return type;
8031         }
8032         return NULL;
8033 }
8034
8035 static const struct nft_object_type *
8036 nft_obj_type_get(struct net *net, u32 objtype, u8 family)
8037 {
8038         const struct nft_object_type *type;
8039
8040         rcu_read_lock();
8041         type = __nft_obj_type_get(objtype, family);
8042         if (type != NULL && try_module_get(type->owner)) {
8043                 rcu_read_unlock();
8044                 return type;
8045         }
8046         rcu_read_unlock();
8047
8048         lockdep_nfnl_nft_mutex_not_held();
8049 #ifdef CONFIG_MODULES
8050         if (type == NULL) {
8051                 if (nft_request_module(net, "nft-obj-%u", objtype) == -EAGAIN)
8052                         return ERR_PTR(-EAGAIN);
8053         }
8054 #endif
8055         return ERR_PTR(-ENOENT);
8056 }
8057
8058 static int nf_tables_updobj(const struct nft_ctx *ctx,
8059                             const struct nft_object_type *type,
8060                             const struct nlattr *attr,
8061                             struct nft_object *obj)
8062 {
8063         struct nft_object *newobj;
8064         struct nft_trans *trans;
8065         int err = -ENOMEM;
8066
8067         /* caller must have obtained type->owner reference. */
8068         trans = nft_trans_alloc(ctx, NFT_MSG_NEWOBJ,
8069                                 sizeof(struct nft_trans_obj));
8070         if (!trans)
8071                 goto err_trans;
8072
8073         newobj = nft_obj_init(ctx, type, attr);
8074         if (IS_ERR(newobj)) {
8075                 err = PTR_ERR(newobj);
8076                 goto err_free_trans;
8077         }
8078
8079         nft_trans_obj(trans) = obj;
8080         nft_trans_obj_update(trans) = true;
8081         nft_trans_obj_newobj(trans) = newobj;
8082         nft_trans_commit_list_add_tail(ctx->net, trans);
8083
8084         return 0;
8085
8086 err_free_trans:
8087         kfree(trans);
8088 err_trans:
8089         module_put(type->owner);
8090         return err;
8091 }
8092
8093 static int nf_tables_newobj(struct sk_buff *skb, const struct nfnl_info *info,
8094                             const struct nlattr * const nla[])
8095 {
8096         struct netlink_ext_ack *extack = info->extack;
8097         u8 genmask = nft_genmask_next(info->net);
8098         u8 family = info->nfmsg->nfgen_family;
8099         const struct nft_object_type *type;
8100         struct net *net = info->net;
8101         struct nft_table *table;
8102         struct nft_object *obj;
8103         struct nft_ctx ctx;
8104         u32 objtype;
8105         int err;
8106
8107         if (!nla[NFTA_OBJ_TYPE] ||
8108             !nla[NFTA_OBJ_NAME] ||
8109             !nla[NFTA_OBJ_DATA])
8110                 return -EINVAL;
8111
8112         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
8113                                  NETLINK_CB(skb).portid);
8114         if (IS_ERR(table)) {
8115                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
8116                 return PTR_ERR(table);
8117         }
8118
8119         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
8120         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
8121         if (IS_ERR(obj)) {
8122                 err = PTR_ERR(obj);
8123                 if (err != -ENOENT) {
8124                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
8125                         return err;
8126                 }
8127         } else {
8128                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
8129                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
8130                         return -EEXIST;
8131                 }
8132                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
8133                         return -EOPNOTSUPP;
8134
8135                 if (!obj->ops->update)
8136                         return 0;
8137
8138                 type = nft_obj_type_get(net, objtype, family);
8139                 if (WARN_ON_ONCE(IS_ERR(type)))
8140                         return PTR_ERR(type);
8141
8142                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8143
8144                 /* type->owner reference is put when transaction object is released. */
8145                 return nf_tables_updobj(&ctx, type, nla[NFTA_OBJ_DATA], obj);
8146         }
8147
8148         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8149
8150         if (!nft_use_inc(&table->use))
8151                 return -EMFILE;
8152
8153         type = nft_obj_type_get(net, objtype, family);
8154         if (IS_ERR(type)) {
8155                 err = PTR_ERR(type);
8156                 goto err_type;
8157         }
8158
8159         obj = nft_obj_init(&ctx, type, nla[NFTA_OBJ_DATA]);
8160         if (IS_ERR(obj)) {
8161                 err = PTR_ERR(obj);
8162                 goto err_init;
8163         }
8164         obj->key.table = table;
8165         obj->handle = nf_tables_alloc_handle(table);
8166
8167         obj->key.name = nla_strdup(nla[NFTA_OBJ_NAME], GFP_KERNEL_ACCOUNT);
8168         if (!obj->key.name) {
8169                 err = -ENOMEM;
8170                 goto err_strdup;
8171         }
8172
8173         if (nla[NFTA_OBJ_USERDATA]) {
8174                 obj->udata = nla_memdup(nla[NFTA_OBJ_USERDATA], GFP_KERNEL_ACCOUNT);
8175                 if (obj->udata == NULL)
8176                         goto err_userdata;
8177
8178                 obj->udlen = nla_len(nla[NFTA_OBJ_USERDATA]);
8179         }
8180
8181         err = nft_trans_obj_add(&ctx, NFT_MSG_NEWOBJ, obj);
8182         if (err < 0)
8183                 goto err_trans;
8184
8185         err = rhltable_insert(&nft_objname_ht, &obj->rhlhead,
8186                               nft_objname_ht_params);
8187         if (err < 0)
8188                 goto err_obj_ht;
8189
8190         list_add_tail_rcu(&obj->list, &table->objects);
8191
8192         return 0;
8193 err_obj_ht:
8194         /* queued in transaction log */
8195         INIT_LIST_HEAD(&obj->list);
8196         return err;
8197 err_trans:
8198         kfree(obj->udata);
8199 err_userdata:
8200         kfree(obj->key.name);
8201 err_strdup:
8202         if (obj->ops->destroy)
8203                 obj->ops->destroy(&ctx, obj);
8204         kfree(obj);
8205 err_init:
8206         module_put(type->owner);
8207 err_type:
8208         nft_use_dec_restore(&table->use);
8209
8210         return err;
8211 }
8212
8213 static int nf_tables_fill_obj_info(struct sk_buff *skb, struct net *net,
8214                                    u32 portid, u32 seq, int event, u32 flags,
8215                                    int family, const struct nft_table *table,
8216                                    struct nft_object *obj, bool reset)
8217 {
8218         struct nlmsghdr *nlh;
8219
8220         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
8221         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
8222                            NFNETLINK_V0, nft_base_seq(net));
8223         if (!nlh)
8224                 goto nla_put_failure;
8225
8226         if (nla_put_string(skb, NFTA_OBJ_TABLE, table->name) ||
8227             nla_put_string(skb, NFTA_OBJ_NAME, obj->key.name) ||
8228             nla_put_be64(skb, NFTA_OBJ_HANDLE, cpu_to_be64(obj->handle),
8229                          NFTA_OBJ_PAD))
8230                 goto nla_put_failure;
8231
8232         if (event == NFT_MSG_DELOBJ) {
8233                 nlmsg_end(skb, nlh);
8234                 return 0;
8235         }
8236
8237         if (nla_put_be32(skb, NFTA_OBJ_TYPE, htonl(obj->ops->type->type)) ||
8238             nla_put_be32(skb, NFTA_OBJ_USE, htonl(obj->use)) ||
8239             nft_object_dump(skb, NFTA_OBJ_DATA, obj, reset))
8240                 goto nla_put_failure;
8241
8242         if (obj->udata &&
8243             nla_put(skb, NFTA_OBJ_USERDATA, obj->udlen, obj->udata))
8244                 goto nla_put_failure;
8245
8246         nlmsg_end(skb, nlh);
8247         return 0;
8248
8249 nla_put_failure:
8250         nlmsg_trim(skb, nlh);
8251         return -1;
8252 }
8253
8254 static void audit_log_obj_reset(const struct nft_table *table,
8255                                 unsigned int base_seq, unsigned int nentries)
8256 {
8257         char *buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, base_seq);
8258
8259         audit_log_nfcfg(buf, table->family, nentries,
8260                         AUDIT_NFT_OP_OBJ_RESET, GFP_ATOMIC);
8261         kfree(buf);
8262 }
8263
8264 struct nft_obj_dump_ctx {
8265         unsigned int    s_idx;
8266         char            *table;
8267         u32             type;
8268         bool            reset;
8269 };
8270
8271 static int nf_tables_dump_obj(struct sk_buff *skb, struct netlink_callback *cb)
8272 {
8273         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
8274         struct nft_obj_dump_ctx *ctx = (void *)cb->ctx;
8275         struct net *net = sock_net(skb->sk);
8276         int family = nfmsg->nfgen_family;
8277         struct nftables_pernet *nft_net;
8278         const struct nft_table *table;
8279         unsigned int entries = 0;
8280         struct nft_object *obj;
8281         unsigned int idx = 0;
8282         int rc = 0;
8283
8284         rcu_read_lock();
8285         nft_net = nft_pernet(net);
8286         cb->seq = READ_ONCE(nft_net->base_seq);
8287
8288         list_for_each_entry_rcu(table, &nft_net->tables, list) {
8289                 if (family != NFPROTO_UNSPEC && family != table->family)
8290                         continue;
8291
8292                 entries = 0;
8293                 list_for_each_entry_rcu(obj, &table->objects, list) {
8294                         if (!nft_is_active(net, obj))
8295                                 goto cont;
8296                         if (idx < ctx->s_idx)
8297                                 goto cont;
8298                         if (ctx->table && strcmp(ctx->table, table->name))
8299                                 goto cont;
8300                         if (ctx->type != NFT_OBJECT_UNSPEC &&
8301                             obj->ops->type->type != ctx->type)
8302                                 goto cont;
8303
8304                         rc = nf_tables_fill_obj_info(skb, net,
8305                                                      NETLINK_CB(cb->skb).portid,
8306                                                      cb->nlh->nlmsg_seq,
8307                                                      NFT_MSG_NEWOBJ,
8308                                                      NLM_F_MULTI | NLM_F_APPEND,
8309                                                      table->family, table,
8310                                                      obj, ctx->reset);
8311                         if (rc < 0)
8312                                 break;
8313
8314                         entries++;
8315                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
8316 cont:
8317                         idx++;
8318                 }
8319                 if (ctx->reset && entries)
8320                         audit_log_obj_reset(table, nft_net->base_seq, entries);
8321                 if (rc < 0)
8322                         break;
8323         }
8324         rcu_read_unlock();
8325
8326         ctx->s_idx = idx;
8327         return skb->len;
8328 }
8329
8330 static int nf_tables_dumpreset_obj(struct sk_buff *skb,
8331                                    struct netlink_callback *cb)
8332 {
8333         struct nftables_pernet *nft_net = nft_pernet(sock_net(skb->sk));
8334         int ret;
8335
8336         mutex_lock(&nft_net->commit_mutex);
8337         ret = nf_tables_dump_obj(skb, cb);
8338         mutex_unlock(&nft_net->commit_mutex);
8339
8340         return ret;
8341 }
8342
8343 static int nf_tables_dump_obj_start(struct netlink_callback *cb)
8344 {
8345         struct nft_obj_dump_ctx *ctx = (void *)cb->ctx;
8346         const struct nlattr * const *nla = cb->data;
8347
8348         BUILD_BUG_ON(sizeof(*ctx) > sizeof(cb->ctx));
8349
8350         if (nla[NFTA_OBJ_TABLE]) {
8351                 ctx->table = nla_strdup(nla[NFTA_OBJ_TABLE], GFP_ATOMIC);
8352                 if (!ctx->table)
8353                         return -ENOMEM;
8354         }
8355
8356         if (nla[NFTA_OBJ_TYPE])
8357                 ctx->type = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
8358
8359         return 0;
8360 }
8361
8362 static int nf_tables_dumpreset_obj_start(struct netlink_callback *cb)
8363 {
8364         struct nft_obj_dump_ctx *ctx = (void *)cb->ctx;
8365
8366         ctx->reset = true;
8367
8368         return nf_tables_dump_obj_start(cb);
8369 }
8370
8371 static int nf_tables_dump_obj_done(struct netlink_callback *cb)
8372 {
8373         struct nft_obj_dump_ctx *ctx = (void *)cb->ctx;
8374
8375         kfree(ctx->table);
8376
8377         return 0;
8378 }
8379
8380 /* Caller must hold rcu read lock or transaction mutex */
8381 static struct sk_buff *
8382 nf_tables_getobj_single(u32 portid, const struct nfnl_info *info,
8383                         const struct nlattr * const nla[], bool reset)
8384 {
8385         struct netlink_ext_ack *extack = info->extack;
8386         u8 genmask = nft_genmask_cur(info->net);
8387         u8 family = info->nfmsg->nfgen_family;
8388         const struct nft_table *table;
8389         struct net *net = info->net;
8390         struct nft_object *obj;
8391         struct sk_buff *skb2;
8392         u32 objtype;
8393         int err;
8394
8395         if (!nla[NFTA_OBJ_NAME] ||
8396             !nla[NFTA_OBJ_TYPE])
8397                 return ERR_PTR(-EINVAL);
8398
8399         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask, 0);
8400         if (IS_ERR(table)) {
8401                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
8402                 return ERR_CAST(table);
8403         }
8404
8405         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
8406         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
8407         if (IS_ERR(obj)) {
8408                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
8409                 return ERR_CAST(obj);
8410         }
8411
8412         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
8413         if (!skb2)
8414                 return ERR_PTR(-ENOMEM);
8415
8416         err = nf_tables_fill_obj_info(skb2, net, portid,
8417                                       info->nlh->nlmsg_seq, NFT_MSG_NEWOBJ, 0,
8418                                       family, table, obj, reset);
8419         if (err < 0) {
8420                 kfree_skb(skb2);
8421                 return ERR_PTR(err);
8422         }
8423
8424         return skb2;
8425 }
8426
8427 static int nf_tables_getobj(struct sk_buff *skb, const struct nfnl_info *info,
8428                             const struct nlattr * const nla[])
8429 {
8430         u32 portid = NETLINK_CB(skb).portid;
8431         struct sk_buff *skb2;
8432
8433         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
8434                 struct netlink_dump_control c = {
8435                         .start = nf_tables_dump_obj_start,
8436                         .dump = nf_tables_dump_obj,
8437                         .done = nf_tables_dump_obj_done,
8438                         .module = THIS_MODULE,
8439                         .data = (void *)nla,
8440                 };
8441
8442                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
8443         }
8444
8445         skb2 = nf_tables_getobj_single(portid, info, nla, false);
8446         if (IS_ERR(skb2))
8447                 return PTR_ERR(skb2);
8448
8449         return nfnetlink_unicast(skb2, info->net, portid);
8450 }
8451
8452 static int nf_tables_getobj_reset(struct sk_buff *skb,
8453                                   const struct nfnl_info *info,
8454                                   const struct nlattr * const nla[])
8455 {
8456         struct nftables_pernet *nft_net = nft_pernet(info->net);
8457         u32 portid = NETLINK_CB(skb).portid;
8458         struct net *net = info->net;
8459         struct sk_buff *skb2;
8460         char *buf;
8461
8462         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
8463                 struct netlink_dump_control c = {
8464                         .start = nf_tables_dumpreset_obj_start,
8465                         .dump = nf_tables_dumpreset_obj,
8466                         .done = nf_tables_dump_obj_done,
8467                         .module = THIS_MODULE,
8468                         .data = (void *)nla,
8469                 };
8470
8471                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
8472         }
8473
8474         if (!try_module_get(THIS_MODULE))
8475                 return -EINVAL;
8476         rcu_read_unlock();
8477         mutex_lock(&nft_net->commit_mutex);
8478         skb2 = nf_tables_getobj_single(portid, info, nla, true);
8479         mutex_unlock(&nft_net->commit_mutex);
8480         rcu_read_lock();
8481         module_put(THIS_MODULE);
8482
8483         if (IS_ERR(skb2))
8484                 return PTR_ERR(skb2);
8485
8486         buf = kasprintf(GFP_ATOMIC, "%.*s:%u",
8487                         nla_len(nla[NFTA_OBJ_TABLE]),
8488                         (char *)nla_data(nla[NFTA_OBJ_TABLE]),
8489                         nft_net->base_seq);
8490         audit_log_nfcfg(buf, info->nfmsg->nfgen_family, 1,
8491                         AUDIT_NFT_OP_OBJ_RESET, GFP_ATOMIC);
8492         kfree(buf);
8493
8494         return nfnetlink_unicast(skb2, net, portid);
8495 }
8496
8497 static void nft_obj_destroy(const struct nft_ctx *ctx, struct nft_object *obj)
8498 {
8499         if (obj->ops->destroy)
8500                 obj->ops->destroy(ctx, obj);
8501
8502         module_put(obj->ops->type->owner);
8503         kfree(obj->key.name);
8504         kfree(obj->udata);
8505         kfree(obj);
8506 }
8507
8508 static int nf_tables_delobj(struct sk_buff *skb, const struct nfnl_info *info,
8509                             const struct nlattr * const nla[])
8510 {
8511         struct netlink_ext_ack *extack = info->extack;
8512         u8 genmask = nft_genmask_next(info->net);
8513         u8 family = info->nfmsg->nfgen_family;
8514         struct net *net = info->net;
8515         const struct nlattr *attr;
8516         struct nft_table *table;
8517         struct nft_object *obj;
8518         struct nft_ctx ctx;
8519         u32 objtype;
8520
8521         if (!nla[NFTA_OBJ_TYPE] ||
8522             (!nla[NFTA_OBJ_NAME] && !nla[NFTA_OBJ_HANDLE]))
8523                 return -EINVAL;
8524
8525         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
8526                                  NETLINK_CB(skb).portid);
8527         if (IS_ERR(table)) {
8528                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
8529                 return PTR_ERR(table);
8530         }
8531
8532         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
8533         if (nla[NFTA_OBJ_HANDLE]) {
8534                 attr = nla[NFTA_OBJ_HANDLE];
8535                 obj = nft_obj_lookup_byhandle(table, attr, objtype, genmask);
8536         } else {
8537                 attr = nla[NFTA_OBJ_NAME];
8538                 obj = nft_obj_lookup(net, table, attr, objtype, genmask);
8539         }
8540
8541         if (IS_ERR(obj)) {
8542                 if (PTR_ERR(obj) == -ENOENT &&
8543                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYOBJ)
8544                         return 0;
8545
8546                 NL_SET_BAD_ATTR(extack, attr);
8547                 return PTR_ERR(obj);
8548         }
8549         if (obj->use > 0) {
8550                 NL_SET_BAD_ATTR(extack, attr);
8551                 return -EBUSY;
8552         }
8553
8554         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8555
8556         return nft_delobj(&ctx, obj);
8557 }
8558
8559 static void
8560 __nft_obj_notify(struct net *net, const struct nft_table *table,
8561                  struct nft_object *obj, u32 portid, u32 seq, int event,
8562                  u16 flags, int family, int report, gfp_t gfp)
8563 {
8564         struct nftables_pernet *nft_net = nft_pernet(net);
8565         struct sk_buff *skb;
8566         int err;
8567
8568         if (!report &&
8569             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
8570                 return;
8571
8572         skb = nlmsg_new(NLMSG_GOODSIZE, gfp);
8573         if (skb == NULL)
8574                 goto err;
8575
8576         err = nf_tables_fill_obj_info(skb, net, portid, seq, event,
8577                                       flags & (NLM_F_CREATE | NLM_F_EXCL),
8578                                       family, table, obj, false);
8579         if (err < 0) {
8580                 kfree_skb(skb);
8581                 goto err;
8582         }
8583
8584         nft_notify_enqueue(skb, report, &nft_net->notify_list);
8585         return;
8586 err:
8587         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
8588 }
8589
8590 void nft_obj_notify(struct net *net, const struct nft_table *table,
8591                     struct nft_object *obj, u32 portid, u32 seq, int event,
8592                     u16 flags, int family, int report, gfp_t gfp)
8593 {
8594         struct nftables_pernet *nft_net = nft_pernet(net);
8595         char *buf = kasprintf(gfp, "%s:%u",
8596                               table->name, nft_net->base_seq);
8597
8598         audit_log_nfcfg(buf,
8599                         family,
8600                         obj->handle,
8601                         event == NFT_MSG_NEWOBJ ?
8602                                  AUDIT_NFT_OP_OBJ_REGISTER :
8603                                  AUDIT_NFT_OP_OBJ_UNREGISTER,
8604                         gfp);
8605         kfree(buf);
8606
8607         __nft_obj_notify(net, table, obj, portid, seq, event,
8608                          flags, family, report, gfp);
8609 }
8610 EXPORT_SYMBOL_GPL(nft_obj_notify);
8611
8612 static void nf_tables_obj_notify(const struct nft_ctx *ctx,
8613                                  struct nft_object *obj, int event)
8614 {
8615         __nft_obj_notify(ctx->net, ctx->table, obj, ctx->portid,
8616                          ctx->seq, event, ctx->flags, ctx->family,
8617                          ctx->report, GFP_KERNEL);
8618 }
8619
8620 /*
8621  * Flow tables
8622  */
8623 void nft_register_flowtable_type(struct nf_flowtable_type *type)
8624 {
8625         nfnl_lock(NFNL_SUBSYS_NFTABLES);
8626         list_add_tail_rcu(&type->list, &nf_tables_flowtables);
8627         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
8628 }
8629 EXPORT_SYMBOL_GPL(nft_register_flowtable_type);
8630
8631 void nft_unregister_flowtable_type(struct nf_flowtable_type *type)
8632 {
8633         nfnl_lock(NFNL_SUBSYS_NFTABLES);
8634         list_del_rcu(&type->list);
8635         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
8636 }
8637 EXPORT_SYMBOL_GPL(nft_unregister_flowtable_type);
8638
8639 static const struct nla_policy nft_flowtable_policy[NFTA_FLOWTABLE_MAX + 1] = {
8640         [NFTA_FLOWTABLE_TABLE]          = { .type = NLA_STRING,
8641                                             .len = NFT_NAME_MAXLEN - 1 },
8642         [NFTA_FLOWTABLE_NAME]           = { .type = NLA_STRING,
8643                                             .len = NFT_NAME_MAXLEN - 1 },
8644         [NFTA_FLOWTABLE_HOOK]           = { .type = NLA_NESTED },
8645         [NFTA_FLOWTABLE_HANDLE]         = { .type = NLA_U64 },
8646         [NFTA_FLOWTABLE_FLAGS]          = { .type = NLA_U32 },
8647 };
8648
8649 struct nft_flowtable *nft_flowtable_lookup(const struct net *net,
8650                                            const struct nft_table *table,
8651                                            const struct nlattr *nla, u8 genmask)
8652 {
8653         struct nft_flowtable *flowtable;
8654
8655         list_for_each_entry_rcu(flowtable, &table->flowtables, list,
8656                                 lockdep_commit_lock_is_held(net)) {
8657                 if (!nla_strcmp(nla, flowtable->name) &&
8658                     nft_active_genmask(flowtable, genmask))
8659                         return flowtable;
8660         }
8661         return ERR_PTR(-ENOENT);
8662 }
8663 EXPORT_SYMBOL_GPL(nft_flowtable_lookup);
8664
8665 void nf_tables_deactivate_flowtable(const struct nft_ctx *ctx,
8666                                     struct nft_flowtable *flowtable,
8667                                     enum nft_trans_phase phase)
8668 {
8669         switch (phase) {
8670         case NFT_TRANS_PREPARE_ERROR:
8671         case NFT_TRANS_PREPARE:
8672         case NFT_TRANS_ABORT:
8673         case NFT_TRANS_RELEASE:
8674                 nft_use_dec(&flowtable->use);
8675                 fallthrough;
8676         default:
8677                 return;
8678         }
8679 }
8680 EXPORT_SYMBOL_GPL(nf_tables_deactivate_flowtable);
8681
8682 static struct nft_flowtable *
8683 nft_flowtable_lookup_byhandle(const struct nft_table *table,
8684                               const struct nlattr *nla, u8 genmask)
8685 {
8686        struct nft_flowtable *flowtable;
8687
8688        list_for_each_entry(flowtable, &table->flowtables, list) {
8689                if (be64_to_cpu(nla_get_be64(nla)) == flowtable->handle &&
8690                    nft_active_genmask(flowtable, genmask))
8691                        return flowtable;
8692        }
8693        return ERR_PTR(-ENOENT);
8694 }
8695
8696 struct nft_flowtable_hook {
8697         u32                     num;
8698         int                     priority;
8699         struct list_head        list;
8700 };
8701
8702 static const struct nla_policy nft_flowtable_hook_policy[NFTA_FLOWTABLE_HOOK_MAX + 1] = {
8703         [NFTA_FLOWTABLE_HOOK_NUM]       = { .type = NLA_U32 },
8704         [NFTA_FLOWTABLE_HOOK_PRIORITY]  = { .type = NLA_U32 },
8705         [NFTA_FLOWTABLE_HOOK_DEVS]      = { .type = NLA_NESTED },
8706 };
8707
8708 static int nft_flowtable_parse_hook(const struct nft_ctx *ctx,
8709                                     const struct nlattr * const nla[],
8710                                     struct nft_flowtable_hook *flowtable_hook,
8711                                     struct nft_flowtable *flowtable,
8712                                     struct netlink_ext_ack *extack, bool add)
8713 {
8714         struct nlattr *tb[NFTA_FLOWTABLE_HOOK_MAX + 1];
8715         struct nft_hook *hook;
8716         int hooknum, priority;
8717         int err;
8718
8719         INIT_LIST_HEAD(&flowtable_hook->list);
8720
8721         err = nla_parse_nested_deprecated(tb, NFTA_FLOWTABLE_HOOK_MAX,
8722                                           nla[NFTA_FLOWTABLE_HOOK],
8723                                           nft_flowtable_hook_policy, NULL);
8724         if (err < 0)
8725                 return err;
8726
8727         if (add) {
8728                 if (!tb[NFTA_FLOWTABLE_HOOK_NUM] ||
8729                     !tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
8730                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8731                         return -ENOENT;
8732                 }
8733
8734                 hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
8735                 if (hooknum != NF_NETDEV_INGRESS)
8736                         return -EOPNOTSUPP;
8737
8738                 priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
8739
8740                 flowtable_hook->priority        = priority;
8741                 flowtable_hook->num             = hooknum;
8742         } else {
8743                 if (tb[NFTA_FLOWTABLE_HOOK_NUM]) {
8744                         hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
8745                         if (hooknum != flowtable->hooknum)
8746                                 return -EOPNOTSUPP;
8747                 }
8748
8749                 if (tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
8750                         priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
8751                         if (priority != flowtable->data.priority)
8752                                 return -EOPNOTSUPP;
8753                 }
8754
8755                 flowtable_hook->priority        = flowtable->data.priority;
8756                 flowtable_hook->num             = flowtable->hooknum;
8757         }
8758
8759         if (tb[NFTA_FLOWTABLE_HOOK_DEVS]) {
8760                 err = nf_tables_parse_netdev_hooks(ctx->net,
8761                                                    tb[NFTA_FLOWTABLE_HOOK_DEVS],
8762                                                    &flowtable_hook->list,
8763                                                    extack);
8764                 if (err < 0)
8765                         return err;
8766         }
8767
8768         list_for_each_entry(hook, &flowtable_hook->list, list) {
8769                 hook->ops.pf            = NFPROTO_NETDEV;
8770                 hook->ops.hooknum       = flowtable_hook->num;
8771                 hook->ops.priority      = flowtable_hook->priority;
8772                 hook->ops.priv          = &flowtable->data;
8773                 hook->ops.hook          = flowtable->data.type->hook;
8774         }
8775
8776         return err;
8777 }
8778
8779 /* call under rcu_read_lock */
8780 static const struct nf_flowtable_type *__nft_flowtable_type_get(u8 family)
8781 {
8782         const struct nf_flowtable_type *type;
8783
8784         list_for_each_entry_rcu(type, &nf_tables_flowtables, list) {
8785                 if (family == type->family)
8786                         return type;
8787         }
8788         return NULL;
8789 }
8790
8791 static const struct nf_flowtable_type *
8792 nft_flowtable_type_get(struct net *net, u8 family)
8793 {
8794         const struct nf_flowtable_type *type;
8795
8796         rcu_read_lock();
8797         type = __nft_flowtable_type_get(family);
8798         if (type != NULL && try_module_get(type->owner)) {
8799                 rcu_read_unlock();
8800                 return type;
8801         }
8802         rcu_read_unlock();
8803
8804         lockdep_nfnl_nft_mutex_not_held();
8805 #ifdef CONFIG_MODULES
8806         if (type == NULL) {
8807                 if (nft_request_module(net, "nf-flowtable-%u", family) == -EAGAIN)
8808                         return ERR_PTR(-EAGAIN);
8809         }
8810 #endif
8811         return ERR_PTR(-ENOENT);
8812 }
8813
8814 /* Only called from error and netdev event paths. */
8815 static void nft_unregister_flowtable_hook(struct net *net,
8816                                           struct nft_flowtable *flowtable,
8817                                           struct nft_hook *hook)
8818 {
8819         nf_unregister_net_hook(net, &hook->ops);
8820         flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
8821                                     FLOW_BLOCK_UNBIND);
8822 }
8823
8824 static void __nft_unregister_flowtable_net_hooks(struct net *net,
8825                                                  struct list_head *hook_list,
8826                                                  bool release_netdev)
8827 {
8828         struct nft_hook *hook, *next;
8829
8830         list_for_each_entry_safe(hook, next, hook_list, list) {
8831                 nf_unregister_net_hook(net, &hook->ops);
8832                 if (release_netdev) {
8833                         list_del(&hook->list);
8834                         kfree_rcu(hook, rcu);
8835                 }
8836         }
8837 }
8838
8839 static void nft_unregister_flowtable_net_hooks(struct net *net,
8840                                                struct list_head *hook_list)
8841 {
8842         __nft_unregister_flowtable_net_hooks(net, hook_list, false);
8843 }
8844
8845 static int nft_register_flowtable_net_hooks(struct net *net,
8846                                             struct nft_table *table,
8847                                             struct list_head *hook_list,
8848                                             struct nft_flowtable *flowtable)
8849 {
8850         struct nft_hook *hook, *hook2, *next;
8851         struct nft_flowtable *ft;
8852         int err, i = 0;
8853
8854         list_for_each_entry(hook, hook_list, list) {
8855                 list_for_each_entry(ft, &table->flowtables, list) {
8856                         if (!nft_is_active_next(net, ft))
8857                                 continue;
8858
8859                         list_for_each_entry(hook2, &ft->hook_list, list) {
8860                                 if (hook->ops.dev == hook2->ops.dev &&
8861                                     hook->ops.pf == hook2->ops.pf) {
8862                                         err = -EEXIST;
8863                                         goto err_unregister_net_hooks;
8864                                 }
8865                         }
8866                 }
8867
8868                 err = flowtable->data.type->setup(&flowtable->data,
8869                                                   hook->ops.dev,
8870                                                   FLOW_BLOCK_BIND);
8871                 if (err < 0)
8872                         goto err_unregister_net_hooks;
8873
8874                 err = nf_register_net_hook(net, &hook->ops);
8875                 if (err < 0) {
8876                         flowtable->data.type->setup(&flowtable->data,
8877                                                     hook->ops.dev,
8878                                                     FLOW_BLOCK_UNBIND);
8879                         goto err_unregister_net_hooks;
8880                 }
8881
8882                 i++;
8883         }
8884
8885         return 0;
8886
8887 err_unregister_net_hooks:
8888         list_for_each_entry_safe(hook, next, hook_list, list) {
8889                 if (i-- <= 0)
8890                         break;
8891
8892                 nft_unregister_flowtable_hook(net, flowtable, hook);
8893                 list_del_rcu(&hook->list);
8894                 kfree_rcu(hook, rcu);
8895         }
8896
8897         return err;
8898 }
8899
8900 static void nft_hooks_destroy(struct list_head *hook_list)
8901 {
8902         struct nft_hook *hook, *next;
8903
8904         list_for_each_entry_safe(hook, next, hook_list, list) {
8905                 list_del_rcu(&hook->list);
8906                 kfree_rcu(hook, rcu);
8907         }
8908 }
8909
8910 static int nft_flowtable_update(struct nft_ctx *ctx, const struct nlmsghdr *nlh,
8911                                 struct nft_flowtable *flowtable,
8912                                 struct netlink_ext_ack *extack)
8913 {
8914         const struct nlattr * const *nla = ctx->nla;
8915         struct nft_flowtable_hook flowtable_hook;
8916         struct nft_hook *hook, *next;
8917         struct nft_trans *trans;
8918         bool unregister = false;
8919         u32 flags;
8920         int err;
8921
8922         err = nft_flowtable_parse_hook(ctx, nla, &flowtable_hook, flowtable,
8923                                        extack, false);
8924         if (err < 0)
8925                 return err;
8926
8927         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
8928                 if (nft_hook_list_find(&flowtable->hook_list, hook)) {
8929                         list_del(&hook->list);
8930                         kfree(hook);
8931                 }
8932         }
8933
8934         if (nla[NFTA_FLOWTABLE_FLAGS]) {
8935                 flags = ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
8936                 if (flags & ~NFT_FLOWTABLE_MASK) {
8937                         err = -EOPNOTSUPP;
8938                         goto err_flowtable_update_hook;
8939                 }
8940                 if ((flowtable->data.flags & NFT_FLOWTABLE_HW_OFFLOAD) ^
8941                     (flags & NFT_FLOWTABLE_HW_OFFLOAD)) {
8942                         err = -EOPNOTSUPP;
8943                         goto err_flowtable_update_hook;
8944                 }
8945         } else {
8946                 flags = flowtable->data.flags;
8947         }
8948
8949         err = nft_register_flowtable_net_hooks(ctx->net, ctx->table,
8950                                                &flowtable_hook.list, flowtable);
8951         if (err < 0)
8952                 goto err_flowtable_update_hook;
8953
8954         trans = nft_trans_alloc(ctx, NFT_MSG_NEWFLOWTABLE,
8955                                 sizeof(struct nft_trans_flowtable));
8956         if (!trans) {
8957                 unregister = true;
8958                 err = -ENOMEM;
8959                 goto err_flowtable_update_hook;
8960         }
8961
8962         nft_trans_flowtable_flags(trans) = flags;
8963         nft_trans_flowtable(trans) = flowtable;
8964         nft_trans_flowtable_update(trans) = true;
8965         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
8966         list_splice(&flowtable_hook.list, &nft_trans_flowtable_hooks(trans));
8967
8968         nft_trans_commit_list_add_tail(ctx->net, trans);
8969
8970         return 0;
8971
8972 err_flowtable_update_hook:
8973         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
8974                 if (unregister)
8975                         nft_unregister_flowtable_hook(ctx->net, flowtable, hook);
8976                 list_del_rcu(&hook->list);
8977                 kfree_rcu(hook, rcu);
8978         }
8979
8980         return err;
8981
8982 }
8983
8984 static int nf_tables_newflowtable(struct sk_buff *skb,
8985                                   const struct nfnl_info *info,
8986                                   const struct nlattr * const nla[])
8987 {
8988         struct netlink_ext_ack *extack = info->extack;
8989         struct nft_flowtable_hook flowtable_hook;
8990         u8 genmask = nft_genmask_next(info->net);
8991         u8 family = info->nfmsg->nfgen_family;
8992         const struct nf_flowtable_type *type;
8993         struct nft_flowtable *flowtable;
8994         struct net *net = info->net;
8995         struct nft_table *table;
8996         struct nft_trans *trans;
8997         struct nft_ctx ctx;
8998         int err;
8999
9000         if (!nla[NFTA_FLOWTABLE_TABLE] ||
9001             !nla[NFTA_FLOWTABLE_NAME] ||
9002             !nla[NFTA_FLOWTABLE_HOOK])
9003                 return -EINVAL;
9004
9005         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
9006                                  genmask, NETLINK_CB(skb).portid);
9007         if (IS_ERR(table)) {
9008                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
9009                 return PTR_ERR(table);
9010         }
9011
9012         flowtable = nft_flowtable_lookup(net, table, nla[NFTA_FLOWTABLE_NAME],
9013                                          genmask);
9014         if (IS_ERR(flowtable)) {
9015                 err = PTR_ERR(flowtable);
9016                 if (err != -ENOENT) {
9017                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
9018                         return err;
9019                 }
9020         } else {
9021                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
9022                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
9023                         return -EEXIST;
9024                 }
9025
9026                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
9027
9028                 return nft_flowtable_update(&ctx, info->nlh, flowtable, extack);
9029         }
9030
9031         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
9032
9033         if (!nft_use_inc(&table->use))
9034                 return -EMFILE;
9035
9036         flowtable = kzalloc(sizeof(*flowtable), GFP_KERNEL_ACCOUNT);
9037         if (!flowtable) {
9038                 err = -ENOMEM;
9039                 goto flowtable_alloc;
9040         }
9041
9042         flowtable->table = table;
9043         flowtable->handle = nf_tables_alloc_handle(table);
9044         INIT_LIST_HEAD(&flowtable->hook_list);
9045
9046         flowtable->name = nla_strdup(nla[NFTA_FLOWTABLE_NAME], GFP_KERNEL_ACCOUNT);
9047         if (!flowtable->name) {
9048                 err = -ENOMEM;
9049                 goto err1;
9050         }
9051
9052         type = nft_flowtable_type_get(net, family);
9053         if (IS_ERR(type)) {
9054                 err = PTR_ERR(type);
9055                 goto err2;
9056         }
9057
9058         if (nla[NFTA_FLOWTABLE_FLAGS]) {
9059                 flowtable->data.flags =
9060                         ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
9061                 if (flowtable->data.flags & ~NFT_FLOWTABLE_MASK) {
9062                         err = -EOPNOTSUPP;
9063                         goto err3;
9064                 }
9065         }
9066
9067         write_pnet(&flowtable->data.net, net);
9068         flowtable->data.type = type;
9069         err = type->init(&flowtable->data);
9070         if (err < 0)
9071                 goto err3;
9072
9073         err = nft_flowtable_parse_hook(&ctx, nla, &flowtable_hook, flowtable,
9074                                        extack, true);
9075         if (err < 0)
9076                 goto err_flowtable_parse_hooks;
9077
9078         list_splice(&flowtable_hook.list, &flowtable->hook_list);
9079         flowtable->data.priority = flowtable_hook.priority;
9080         flowtable->hooknum = flowtable_hook.num;
9081
9082         trans = nft_trans_flowtable_add(&ctx, NFT_MSG_NEWFLOWTABLE, flowtable);
9083         if (IS_ERR(trans)) {
9084                 err = PTR_ERR(trans);
9085                 goto err_flowtable_trans;
9086         }
9087
9088         /* This must be LAST to ensure no packets are walking over this flowtable. */
9089         err = nft_register_flowtable_net_hooks(ctx.net, table,
9090                                                &flowtable->hook_list,
9091                                                flowtable);
9092         if (err < 0)
9093                 goto err_flowtable_hooks;
9094
9095         list_add_tail_rcu(&flowtable->list, &table->flowtables);
9096
9097         return 0;
9098
9099 err_flowtable_hooks:
9100         nft_trans_destroy(trans);
9101 err_flowtable_trans:
9102         nft_hooks_destroy(&flowtable->hook_list);
9103 err_flowtable_parse_hooks:
9104         flowtable->data.type->free(&flowtable->data);
9105 err3:
9106         module_put(type->owner);
9107 err2:
9108         kfree(flowtable->name);
9109 err1:
9110         kfree(flowtable);
9111 flowtable_alloc:
9112         nft_use_dec_restore(&table->use);
9113
9114         return err;
9115 }
9116
9117 static void nft_flowtable_hook_release(struct nft_flowtable_hook *flowtable_hook)
9118 {
9119         struct nft_hook *this, *next;
9120
9121         list_for_each_entry_safe(this, next, &flowtable_hook->list, list) {
9122                 list_del(&this->list);
9123                 kfree(this);
9124         }
9125 }
9126
9127 static int nft_delflowtable_hook(struct nft_ctx *ctx,
9128                                  struct nft_flowtable *flowtable,
9129                                  struct netlink_ext_ack *extack)
9130 {
9131         const struct nlattr * const *nla = ctx->nla;
9132         struct nft_flowtable_hook flowtable_hook;
9133         LIST_HEAD(flowtable_del_list);
9134         struct nft_hook *this, *hook;
9135         struct nft_trans *trans;
9136         int err;
9137
9138         err = nft_flowtable_parse_hook(ctx, nla, &flowtable_hook, flowtable,
9139                                        extack, false);
9140         if (err < 0)
9141                 return err;
9142
9143         list_for_each_entry(this, &flowtable_hook.list, list) {
9144                 hook = nft_hook_list_find(&flowtable->hook_list, this);
9145                 if (!hook) {
9146                         err = -ENOENT;
9147                         goto err_flowtable_del_hook;
9148                 }
9149                 list_move(&hook->list, &flowtable_del_list);
9150         }
9151
9152         trans = nft_trans_alloc(ctx, NFT_MSG_DELFLOWTABLE,
9153                                 sizeof(struct nft_trans_flowtable));
9154         if (!trans) {
9155                 err = -ENOMEM;
9156                 goto err_flowtable_del_hook;
9157         }
9158
9159         nft_trans_flowtable(trans) = flowtable;
9160         nft_trans_flowtable_update(trans) = true;
9161         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
9162         list_splice(&flowtable_del_list, &nft_trans_flowtable_hooks(trans));
9163         nft_flowtable_hook_release(&flowtable_hook);
9164
9165         nft_trans_commit_list_add_tail(ctx->net, trans);
9166
9167         return 0;
9168
9169 err_flowtable_del_hook:
9170         list_splice(&flowtable_del_list, &flowtable->hook_list);
9171         nft_flowtable_hook_release(&flowtable_hook);
9172
9173         return err;
9174 }
9175
9176 static int nf_tables_delflowtable(struct sk_buff *skb,
9177                                   const struct nfnl_info *info,
9178                                   const struct nlattr * const nla[])
9179 {
9180         struct netlink_ext_ack *extack = info->extack;
9181         u8 genmask = nft_genmask_next(info->net);
9182         u8 family = info->nfmsg->nfgen_family;
9183         struct nft_flowtable *flowtable;
9184         struct net *net = info->net;
9185         const struct nlattr *attr;
9186         struct nft_table *table;
9187         struct nft_ctx ctx;
9188
9189         if (!nla[NFTA_FLOWTABLE_TABLE] ||
9190             (!nla[NFTA_FLOWTABLE_NAME] &&
9191              !nla[NFTA_FLOWTABLE_HANDLE]))
9192                 return -EINVAL;
9193
9194         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
9195                                  genmask, NETLINK_CB(skb).portid);
9196         if (IS_ERR(table)) {
9197                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
9198                 return PTR_ERR(table);
9199         }
9200
9201         if (nla[NFTA_FLOWTABLE_HANDLE]) {
9202                 attr = nla[NFTA_FLOWTABLE_HANDLE];
9203                 flowtable = nft_flowtable_lookup_byhandle(table, attr, genmask);
9204         } else {
9205                 attr = nla[NFTA_FLOWTABLE_NAME];
9206                 flowtable = nft_flowtable_lookup(net, table, attr, genmask);
9207         }
9208
9209         if (IS_ERR(flowtable)) {
9210                 if (PTR_ERR(flowtable) == -ENOENT &&
9211                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYFLOWTABLE)
9212                         return 0;
9213
9214                 NL_SET_BAD_ATTR(extack, attr);
9215                 return PTR_ERR(flowtable);
9216         }
9217
9218         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
9219
9220         if (nla[NFTA_FLOWTABLE_HOOK])
9221                 return nft_delflowtable_hook(&ctx, flowtable, extack);
9222
9223         if (flowtable->use > 0) {
9224                 NL_SET_BAD_ATTR(extack, attr);
9225                 return -EBUSY;
9226         }
9227
9228         return nft_delflowtable(&ctx, flowtable);
9229 }
9230
9231 static int nf_tables_fill_flowtable_info(struct sk_buff *skb, struct net *net,
9232                                          u32 portid, u32 seq, int event,
9233                                          u32 flags, int family,
9234                                          struct nft_flowtable *flowtable,
9235                                          struct list_head *hook_list)
9236 {
9237         struct nlattr *nest, *nest_devs;
9238         struct nft_hook *hook;
9239         struct nlmsghdr *nlh;
9240
9241         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
9242         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
9243                            NFNETLINK_V0, nft_base_seq(net));
9244         if (!nlh)
9245                 goto nla_put_failure;
9246
9247         if (nla_put_string(skb, NFTA_FLOWTABLE_TABLE, flowtable->table->name) ||
9248             nla_put_string(skb, NFTA_FLOWTABLE_NAME, flowtable->name) ||
9249             nla_put_be64(skb, NFTA_FLOWTABLE_HANDLE, cpu_to_be64(flowtable->handle),
9250                          NFTA_FLOWTABLE_PAD))
9251                 goto nla_put_failure;
9252
9253         if (event == NFT_MSG_DELFLOWTABLE && !hook_list) {
9254                 nlmsg_end(skb, nlh);
9255                 return 0;
9256         }
9257
9258         if (nla_put_be32(skb, NFTA_FLOWTABLE_USE, htonl(flowtable->use)) ||
9259             nla_put_be32(skb, NFTA_FLOWTABLE_FLAGS, htonl(flowtable->data.flags)))
9260                 goto nla_put_failure;
9261
9262         nest = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK);
9263         if (!nest)
9264                 goto nla_put_failure;
9265         if (nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_NUM, htonl(flowtable->hooknum)) ||
9266             nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_PRIORITY, htonl(flowtable->data.priority)))
9267                 goto nla_put_failure;
9268
9269         nest_devs = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK_DEVS);
9270         if (!nest_devs)
9271                 goto nla_put_failure;
9272
9273         if (!hook_list)
9274                 hook_list = &flowtable->hook_list;
9275
9276         list_for_each_entry_rcu(hook, hook_list, list,
9277                                 lockdep_commit_lock_is_held(net)) {
9278                 if (nla_put_string(skb, NFTA_DEVICE_NAME, hook->ops.dev->name))
9279                         goto nla_put_failure;
9280         }
9281         nla_nest_end(skb, nest_devs);
9282         nla_nest_end(skb, nest);
9283
9284         nlmsg_end(skb, nlh);
9285         return 0;
9286
9287 nla_put_failure:
9288         nlmsg_trim(skb, nlh);
9289         return -1;
9290 }
9291
9292 struct nft_flowtable_filter {
9293         char            *table;
9294 };
9295
9296 static int nf_tables_dump_flowtable(struct sk_buff *skb,
9297                                     struct netlink_callback *cb)
9298 {
9299         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
9300         struct nft_flowtable_filter *filter = cb->data;
9301         unsigned int idx = 0, s_idx = cb->args[0];
9302         struct net *net = sock_net(skb->sk);
9303         int family = nfmsg->nfgen_family;
9304         struct nft_flowtable *flowtable;
9305         struct nftables_pernet *nft_net;
9306         const struct nft_table *table;
9307
9308         rcu_read_lock();
9309         nft_net = nft_pernet(net);
9310         cb->seq = READ_ONCE(nft_net->base_seq);
9311
9312         list_for_each_entry_rcu(table, &nft_net->tables, list) {
9313                 if (family != NFPROTO_UNSPEC && family != table->family)
9314                         continue;
9315
9316                 list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
9317                         if (!nft_is_active(net, flowtable))
9318                                 goto cont;
9319                         if (idx < s_idx)
9320                                 goto cont;
9321                         if (idx > s_idx)
9322                                 memset(&cb->args[1], 0,
9323                                        sizeof(cb->args) - sizeof(cb->args[0]));
9324                         if (filter && filter->table &&
9325                             strcmp(filter->table, table->name))
9326                                 goto cont;
9327
9328                         if (nf_tables_fill_flowtable_info(skb, net, NETLINK_CB(cb->skb).portid,
9329                                                           cb->nlh->nlmsg_seq,
9330                                                           NFT_MSG_NEWFLOWTABLE,
9331                                                           NLM_F_MULTI | NLM_F_APPEND,
9332                                                           table->family,
9333                                                           flowtable, NULL) < 0)
9334                                 goto done;
9335
9336                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
9337 cont:
9338                         idx++;
9339                 }
9340         }
9341 done:
9342         rcu_read_unlock();
9343
9344         cb->args[0] = idx;
9345         return skb->len;
9346 }
9347
9348 static int nf_tables_dump_flowtable_start(struct netlink_callback *cb)
9349 {
9350         const struct nlattr * const *nla = cb->data;
9351         struct nft_flowtable_filter *filter = NULL;
9352
9353         if (nla[NFTA_FLOWTABLE_TABLE]) {
9354                 filter = kzalloc(sizeof(*filter), GFP_ATOMIC);
9355                 if (!filter)
9356                         return -ENOMEM;
9357
9358                 filter->table = nla_strdup(nla[NFTA_FLOWTABLE_TABLE],
9359                                            GFP_ATOMIC);
9360                 if (!filter->table) {
9361                         kfree(filter);
9362                         return -ENOMEM;
9363                 }
9364         }
9365
9366         cb->data = filter;
9367         return 0;
9368 }
9369
9370 static int nf_tables_dump_flowtable_done(struct netlink_callback *cb)
9371 {
9372         struct nft_flowtable_filter *filter = cb->data;
9373
9374         if (!filter)
9375                 return 0;
9376
9377         kfree(filter->table);
9378         kfree(filter);
9379
9380         return 0;
9381 }
9382
9383 /* called with rcu_read_lock held */
9384 static int nf_tables_getflowtable(struct sk_buff *skb,
9385                                   const struct nfnl_info *info,
9386                                   const struct nlattr * const nla[])
9387 {
9388         struct netlink_ext_ack *extack = info->extack;
9389         u8 genmask = nft_genmask_cur(info->net);
9390         u8 family = info->nfmsg->nfgen_family;
9391         struct nft_flowtable *flowtable;
9392         const struct nft_table *table;
9393         struct net *net = info->net;
9394         struct sk_buff *skb2;
9395         int err;
9396
9397         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
9398                 struct netlink_dump_control c = {
9399                         .start = nf_tables_dump_flowtable_start,
9400                         .dump = nf_tables_dump_flowtable,
9401                         .done = nf_tables_dump_flowtable_done,
9402                         .module = THIS_MODULE,
9403                         .data = (void *)nla,
9404                 };
9405
9406                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
9407         }
9408
9409         if (!nla[NFTA_FLOWTABLE_NAME])
9410                 return -EINVAL;
9411
9412         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
9413                                  genmask, 0);
9414         if (IS_ERR(table)) {
9415                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
9416                 return PTR_ERR(table);
9417         }
9418
9419         flowtable = nft_flowtable_lookup(net, table, nla[NFTA_FLOWTABLE_NAME],
9420                                          genmask);
9421         if (IS_ERR(flowtable)) {
9422                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
9423                 return PTR_ERR(flowtable);
9424         }
9425
9426         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
9427         if (!skb2)
9428                 return -ENOMEM;
9429
9430         err = nf_tables_fill_flowtable_info(skb2, net, NETLINK_CB(skb).portid,
9431                                             info->nlh->nlmsg_seq,
9432                                             NFT_MSG_NEWFLOWTABLE, 0, family,
9433                                             flowtable, NULL);
9434         if (err < 0)
9435                 goto err_fill_flowtable_info;
9436
9437         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
9438
9439 err_fill_flowtable_info:
9440         kfree_skb(skb2);
9441         return err;
9442 }
9443
9444 static void nf_tables_flowtable_notify(struct nft_ctx *ctx,
9445                                        struct nft_flowtable *flowtable,
9446                                        struct list_head *hook_list, int event)
9447 {
9448         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
9449         struct sk_buff *skb;
9450         u16 flags = 0;
9451         int err;
9452
9453         if (!ctx->report &&
9454             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
9455                 return;
9456
9457         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
9458         if (skb == NULL)
9459                 goto err;
9460
9461         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
9462                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
9463
9464         err = nf_tables_fill_flowtable_info(skb, ctx->net, ctx->portid,
9465                                             ctx->seq, event, flags,
9466                                             ctx->family, flowtable, hook_list);
9467         if (err < 0) {
9468                 kfree_skb(skb);
9469                 goto err;
9470         }
9471
9472         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
9473         return;
9474 err:
9475         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
9476 }
9477
9478 static void nf_tables_flowtable_destroy(struct nft_flowtable *flowtable)
9479 {
9480         struct nft_hook *hook, *next;
9481
9482         flowtable->data.type->free(&flowtable->data);
9483         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
9484                 flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
9485                                             FLOW_BLOCK_UNBIND);
9486                 list_del_rcu(&hook->list);
9487                 kfree_rcu(hook, rcu);
9488         }
9489         kfree(flowtable->name);
9490         module_put(flowtable->data.type->owner);
9491         kfree(flowtable);
9492 }
9493
9494 static int nf_tables_fill_gen_info(struct sk_buff *skb, struct net *net,
9495                                    u32 portid, u32 seq)
9496 {
9497         struct nftables_pernet *nft_net = nft_pernet(net);
9498         struct nlmsghdr *nlh;
9499         char buf[TASK_COMM_LEN];
9500         int event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWGEN);
9501
9502         nlh = nfnl_msg_put(skb, portid, seq, event, 0, AF_UNSPEC,
9503                            NFNETLINK_V0, nft_base_seq(net));
9504         if (!nlh)
9505                 goto nla_put_failure;
9506
9507         if (nla_put_be32(skb, NFTA_GEN_ID, htonl(nft_net->base_seq)) ||
9508             nla_put_be32(skb, NFTA_GEN_PROC_PID, htonl(task_pid_nr(current))) ||
9509             nla_put_string(skb, NFTA_GEN_PROC_NAME, get_task_comm(buf, current)))
9510                 goto nla_put_failure;
9511
9512         nlmsg_end(skb, nlh);
9513         return 0;
9514
9515 nla_put_failure:
9516         nlmsg_trim(skb, nlh);
9517         return -EMSGSIZE;
9518 }
9519
9520 static void nft_flowtable_event(unsigned long event, struct net_device *dev,
9521                                 struct nft_flowtable *flowtable)
9522 {
9523         struct nft_hook *hook;
9524
9525         list_for_each_entry(hook, &flowtable->hook_list, list) {
9526                 if (hook->ops.dev != dev)
9527                         continue;
9528
9529                 /* flow_offload_netdev_event() cleans up entries for us. */
9530                 nft_unregister_flowtable_hook(dev_net(dev), flowtable, hook);
9531                 list_del_rcu(&hook->list);
9532                 kfree_rcu(hook, rcu);
9533                 break;
9534         }
9535 }
9536
9537 static int nf_tables_flowtable_event(struct notifier_block *this,
9538                                      unsigned long event, void *ptr)
9539 {
9540         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
9541         struct nft_flowtable *flowtable;
9542         struct nftables_pernet *nft_net;
9543         struct nft_table *table;
9544         struct net *net;
9545
9546         if (event != NETDEV_UNREGISTER)
9547                 return 0;
9548
9549         net = dev_net(dev);
9550         nft_net = nft_pernet(net);
9551         mutex_lock(&nft_net->commit_mutex);
9552         list_for_each_entry(table, &nft_net->tables, list) {
9553                 list_for_each_entry(flowtable, &table->flowtables, list) {
9554                         nft_flowtable_event(event, dev, flowtable);
9555                 }
9556         }
9557         mutex_unlock(&nft_net->commit_mutex);
9558
9559         return NOTIFY_DONE;
9560 }
9561
9562 static struct notifier_block nf_tables_flowtable_notifier = {
9563         .notifier_call  = nf_tables_flowtable_event,
9564 };
9565
9566 static void nf_tables_gen_notify(struct net *net, struct sk_buff *skb,
9567                                  int event)
9568 {
9569         struct nlmsghdr *nlh = nlmsg_hdr(skb);
9570         struct sk_buff *skb2;
9571         int err;
9572
9573         if (!nlmsg_report(nlh) &&
9574             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
9575                 return;
9576
9577         skb2 = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
9578         if (skb2 == NULL)
9579                 goto err;
9580
9581         err = nf_tables_fill_gen_info(skb2, net, NETLINK_CB(skb).portid,
9582                                       nlh->nlmsg_seq);
9583         if (err < 0) {
9584                 kfree_skb(skb2);
9585                 goto err;
9586         }
9587
9588         nfnetlink_send(skb2, net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
9589                        nlmsg_report(nlh), GFP_KERNEL);
9590         return;
9591 err:
9592         nfnetlink_set_err(net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
9593                           -ENOBUFS);
9594 }
9595
9596 static int nf_tables_getgen(struct sk_buff *skb, const struct nfnl_info *info,
9597                             const struct nlattr * const nla[])
9598 {
9599         struct sk_buff *skb2;
9600         int err;
9601
9602         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
9603         if (skb2 == NULL)
9604                 return -ENOMEM;
9605
9606         err = nf_tables_fill_gen_info(skb2, info->net, NETLINK_CB(skb).portid,
9607                                       info->nlh->nlmsg_seq);
9608         if (err < 0)
9609                 goto err_fill_gen_info;
9610
9611         return nfnetlink_unicast(skb2, info->net, NETLINK_CB(skb).portid);
9612
9613 err_fill_gen_info:
9614         kfree_skb(skb2);
9615         return err;
9616 }
9617
9618 static const struct nfnl_callback nf_tables_cb[NFT_MSG_MAX] = {
9619         [NFT_MSG_NEWTABLE] = {
9620                 .call           = nf_tables_newtable,
9621                 .type           = NFNL_CB_BATCH,
9622                 .attr_count     = NFTA_TABLE_MAX,
9623                 .policy         = nft_table_policy,
9624         },
9625         [NFT_MSG_GETTABLE] = {
9626                 .call           = nf_tables_gettable,
9627                 .type           = NFNL_CB_RCU,
9628                 .attr_count     = NFTA_TABLE_MAX,
9629                 .policy         = nft_table_policy,
9630         },
9631         [NFT_MSG_DELTABLE] = {
9632                 .call           = nf_tables_deltable,
9633                 .type           = NFNL_CB_BATCH,
9634                 .attr_count     = NFTA_TABLE_MAX,
9635                 .policy         = nft_table_policy,
9636         },
9637         [NFT_MSG_DESTROYTABLE] = {
9638                 .call           = nf_tables_deltable,
9639                 .type           = NFNL_CB_BATCH,
9640                 .attr_count     = NFTA_TABLE_MAX,
9641                 .policy         = nft_table_policy,
9642         },
9643         [NFT_MSG_NEWCHAIN] = {
9644                 .call           = nf_tables_newchain,
9645                 .type           = NFNL_CB_BATCH,
9646                 .attr_count     = NFTA_CHAIN_MAX,
9647                 .policy         = nft_chain_policy,
9648         },
9649         [NFT_MSG_GETCHAIN] = {
9650                 .call           = nf_tables_getchain,
9651                 .type           = NFNL_CB_RCU,
9652                 .attr_count     = NFTA_CHAIN_MAX,
9653                 .policy         = nft_chain_policy,
9654         },
9655         [NFT_MSG_DELCHAIN] = {
9656                 .call           = nf_tables_delchain,
9657                 .type           = NFNL_CB_BATCH,
9658                 .attr_count     = NFTA_CHAIN_MAX,
9659                 .policy         = nft_chain_policy,
9660         },
9661         [NFT_MSG_DESTROYCHAIN] = {
9662                 .call           = nf_tables_delchain,
9663                 .type           = NFNL_CB_BATCH,
9664                 .attr_count     = NFTA_CHAIN_MAX,
9665                 .policy         = nft_chain_policy,
9666         },
9667         [NFT_MSG_NEWRULE] = {
9668                 .call           = nf_tables_newrule,
9669                 .type           = NFNL_CB_BATCH,
9670                 .attr_count     = NFTA_RULE_MAX,
9671                 .policy         = nft_rule_policy,
9672         },
9673         [NFT_MSG_GETRULE] = {
9674                 .call           = nf_tables_getrule,
9675                 .type           = NFNL_CB_RCU,
9676                 .attr_count     = NFTA_RULE_MAX,
9677                 .policy         = nft_rule_policy,
9678         },
9679         [NFT_MSG_GETRULE_RESET] = {
9680                 .call           = nf_tables_getrule_reset,
9681                 .type           = NFNL_CB_RCU,
9682                 .attr_count     = NFTA_RULE_MAX,
9683                 .policy         = nft_rule_policy,
9684         },
9685         [NFT_MSG_DELRULE] = {
9686                 .call           = nf_tables_delrule,
9687                 .type           = NFNL_CB_BATCH,
9688                 .attr_count     = NFTA_RULE_MAX,
9689                 .policy         = nft_rule_policy,
9690         },
9691         [NFT_MSG_DESTROYRULE] = {
9692                 .call           = nf_tables_delrule,
9693                 .type           = NFNL_CB_BATCH,
9694                 .attr_count     = NFTA_RULE_MAX,
9695                 .policy         = nft_rule_policy,
9696         },
9697         [NFT_MSG_NEWSET] = {
9698                 .call           = nf_tables_newset,
9699                 .type           = NFNL_CB_BATCH,
9700                 .attr_count     = NFTA_SET_MAX,
9701                 .policy         = nft_set_policy,
9702         },
9703         [NFT_MSG_GETSET] = {
9704                 .call           = nf_tables_getset,
9705                 .type           = NFNL_CB_RCU,
9706                 .attr_count     = NFTA_SET_MAX,
9707                 .policy         = nft_set_policy,
9708         },
9709         [NFT_MSG_DELSET] = {
9710                 .call           = nf_tables_delset,
9711                 .type           = NFNL_CB_BATCH,
9712                 .attr_count     = NFTA_SET_MAX,
9713                 .policy         = nft_set_policy,
9714         },
9715         [NFT_MSG_DESTROYSET] = {
9716                 .call           = nf_tables_delset,
9717                 .type           = NFNL_CB_BATCH,
9718                 .attr_count     = NFTA_SET_MAX,
9719                 .policy         = nft_set_policy,
9720         },
9721         [NFT_MSG_NEWSETELEM] = {
9722                 .call           = nf_tables_newsetelem,
9723                 .type           = NFNL_CB_BATCH,
9724                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9725                 .policy         = nft_set_elem_list_policy,
9726         },
9727         [NFT_MSG_GETSETELEM] = {
9728                 .call           = nf_tables_getsetelem,
9729                 .type           = NFNL_CB_RCU,
9730                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9731                 .policy         = nft_set_elem_list_policy,
9732         },
9733         [NFT_MSG_GETSETELEM_RESET] = {
9734                 .call           = nf_tables_getsetelem_reset,
9735                 .type           = NFNL_CB_RCU,
9736                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9737                 .policy         = nft_set_elem_list_policy,
9738         },
9739         [NFT_MSG_DELSETELEM] = {
9740                 .call           = nf_tables_delsetelem,
9741                 .type           = NFNL_CB_BATCH,
9742                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9743                 .policy         = nft_set_elem_list_policy,
9744         },
9745         [NFT_MSG_DESTROYSETELEM] = {
9746                 .call           = nf_tables_delsetelem,
9747                 .type           = NFNL_CB_BATCH,
9748                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9749                 .policy         = nft_set_elem_list_policy,
9750         },
9751         [NFT_MSG_GETGEN] = {
9752                 .call           = nf_tables_getgen,
9753                 .type           = NFNL_CB_RCU,
9754         },
9755         [NFT_MSG_NEWOBJ] = {
9756                 .call           = nf_tables_newobj,
9757                 .type           = NFNL_CB_BATCH,
9758                 .attr_count     = NFTA_OBJ_MAX,
9759                 .policy         = nft_obj_policy,
9760         },
9761         [NFT_MSG_GETOBJ] = {
9762                 .call           = nf_tables_getobj,
9763                 .type           = NFNL_CB_RCU,
9764                 .attr_count     = NFTA_OBJ_MAX,
9765                 .policy         = nft_obj_policy,
9766         },
9767         [NFT_MSG_DELOBJ] = {
9768                 .call           = nf_tables_delobj,
9769                 .type           = NFNL_CB_BATCH,
9770                 .attr_count     = NFTA_OBJ_MAX,
9771                 .policy         = nft_obj_policy,
9772         },
9773         [NFT_MSG_DESTROYOBJ] = {
9774                 .call           = nf_tables_delobj,
9775                 .type           = NFNL_CB_BATCH,
9776                 .attr_count     = NFTA_OBJ_MAX,
9777                 .policy         = nft_obj_policy,
9778         },
9779         [NFT_MSG_GETOBJ_RESET] = {
9780                 .call           = nf_tables_getobj_reset,
9781                 .type           = NFNL_CB_RCU,
9782                 .attr_count     = NFTA_OBJ_MAX,
9783                 .policy         = nft_obj_policy,
9784         },
9785         [NFT_MSG_NEWFLOWTABLE] = {
9786                 .call           = nf_tables_newflowtable,
9787                 .type           = NFNL_CB_BATCH,
9788                 .attr_count     = NFTA_FLOWTABLE_MAX,
9789                 .policy         = nft_flowtable_policy,
9790         },
9791         [NFT_MSG_GETFLOWTABLE] = {
9792                 .call           = nf_tables_getflowtable,
9793                 .type           = NFNL_CB_RCU,
9794                 .attr_count     = NFTA_FLOWTABLE_MAX,
9795                 .policy         = nft_flowtable_policy,
9796         },
9797         [NFT_MSG_DELFLOWTABLE] = {
9798                 .call           = nf_tables_delflowtable,
9799                 .type           = NFNL_CB_BATCH,
9800                 .attr_count     = NFTA_FLOWTABLE_MAX,
9801                 .policy         = nft_flowtable_policy,
9802         },
9803         [NFT_MSG_DESTROYFLOWTABLE] = {
9804                 .call           = nf_tables_delflowtable,
9805                 .type           = NFNL_CB_BATCH,
9806                 .attr_count     = NFTA_FLOWTABLE_MAX,
9807                 .policy         = nft_flowtable_policy,
9808         },
9809 };
9810
9811 static int nf_tables_validate(struct net *net)
9812 {
9813         struct nftables_pernet *nft_net = nft_pernet(net);
9814         struct nft_table *table;
9815
9816         list_for_each_entry(table, &nft_net->tables, list) {
9817                 switch (table->validate_state) {
9818                 case NFT_VALIDATE_SKIP:
9819                         continue;
9820                 case NFT_VALIDATE_NEED:
9821                         nft_validate_state_update(table, NFT_VALIDATE_DO);
9822                         fallthrough;
9823                 case NFT_VALIDATE_DO:
9824                         if (nft_table_validate(net, table) < 0)
9825                                 return -EAGAIN;
9826
9827                         nft_validate_state_update(table, NFT_VALIDATE_SKIP);
9828                         break;
9829                 }
9830         }
9831
9832         return 0;
9833 }
9834
9835 /* a drop policy has to be deferred until all rules have been activated,
9836  * otherwise a large ruleset that contains a drop-policy base chain will
9837  * cause all packets to get dropped until the full transaction has been
9838  * processed.
9839  *
9840  * We defer the drop policy until the transaction has been finalized.
9841  */
9842 static void nft_chain_commit_drop_policy(struct nft_trans_chain *trans)
9843 {
9844         struct nft_base_chain *basechain;
9845
9846         if (trans->policy != NF_DROP)
9847                 return;
9848
9849         if (!nft_is_base_chain(trans->chain))
9850                 return;
9851
9852         basechain = nft_base_chain(trans->chain);
9853         basechain->policy = NF_DROP;
9854 }
9855
9856 static void nft_chain_commit_update(struct nft_trans_chain *trans)
9857 {
9858         struct nft_table *table = trans->nft_trans_binding.nft_trans.table;
9859         struct nft_base_chain *basechain;
9860
9861         if (trans->name) {
9862                 rhltable_remove(&table->chains_ht,
9863                                 &trans->chain->rhlhead,
9864                                 nft_chain_ht_params);
9865                 swap(trans->chain->name, trans->name);
9866                 rhltable_insert_key(&table->chains_ht,
9867                                     trans->chain->name,
9868                                     &trans->chain->rhlhead,
9869                                     nft_chain_ht_params);
9870         }
9871
9872         if (!nft_is_base_chain(trans->chain))
9873                 return;
9874
9875         nft_chain_stats_replace(trans);
9876
9877         basechain = nft_base_chain(trans->chain);
9878
9879         switch (trans->policy) {
9880         case NF_DROP:
9881         case NF_ACCEPT:
9882                 basechain->policy = trans->policy;
9883                 break;
9884         }
9885 }
9886
9887 static void nft_obj_commit_update(const struct nft_ctx *ctx,
9888                                   struct nft_trans *trans)
9889 {
9890         struct nft_object *newobj;
9891         struct nft_object *obj;
9892
9893         obj = nft_trans_obj(trans);
9894         newobj = nft_trans_obj_newobj(trans);
9895
9896         if (WARN_ON_ONCE(!obj->ops->update))
9897                 return;
9898
9899         obj->ops->update(obj, newobj);
9900         nft_obj_destroy(ctx, newobj);
9901 }
9902
9903 static void nft_commit_release(struct nft_trans *trans)
9904 {
9905         struct nft_ctx ctx = {
9906                 .net = trans->net,
9907         };
9908
9909         nft_ctx_update(&ctx, trans);
9910
9911         switch (trans->msg_type) {
9912         case NFT_MSG_DELTABLE:
9913         case NFT_MSG_DESTROYTABLE:
9914                 nf_tables_table_destroy(trans->table);
9915                 break;
9916         case NFT_MSG_NEWCHAIN:
9917                 free_percpu(nft_trans_chain_stats(trans));
9918                 kfree(nft_trans_chain_name(trans));
9919                 break;
9920         case NFT_MSG_DELCHAIN:
9921         case NFT_MSG_DESTROYCHAIN:
9922                 if (nft_trans_chain_update(trans))
9923                         nft_hooks_destroy(&nft_trans_chain_hooks(trans));
9924                 else
9925                         nf_tables_chain_destroy(nft_trans_chain(trans));
9926                 break;
9927         case NFT_MSG_DELRULE:
9928         case NFT_MSG_DESTROYRULE:
9929                 nf_tables_rule_destroy(&ctx, nft_trans_rule(trans));
9930                 break;
9931         case NFT_MSG_DELSET:
9932         case NFT_MSG_DESTROYSET:
9933                 nft_set_destroy(&ctx, nft_trans_set(trans));
9934                 break;
9935         case NFT_MSG_DELSETELEM:
9936         case NFT_MSG_DESTROYSETELEM:
9937                 nft_trans_elems_destroy(&ctx, nft_trans_container_elem(trans));
9938                 break;
9939         case NFT_MSG_DELOBJ:
9940         case NFT_MSG_DESTROYOBJ:
9941                 nft_obj_destroy(&ctx, nft_trans_obj(trans));
9942                 break;
9943         case NFT_MSG_DELFLOWTABLE:
9944         case NFT_MSG_DESTROYFLOWTABLE:
9945                 if (nft_trans_flowtable_update(trans))
9946                         nft_hooks_destroy(&nft_trans_flowtable_hooks(trans));
9947                 else
9948                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
9949                 break;
9950         }
9951
9952         if (trans->put_net)
9953                 put_net(trans->net);
9954
9955         kfree(trans);
9956 }
9957
9958 static void nf_tables_trans_destroy_work(struct work_struct *w)
9959 {
9960         struct nft_trans *trans, *next;
9961         LIST_HEAD(head);
9962
9963         spin_lock(&nf_tables_destroy_list_lock);
9964         list_splice_init(&nf_tables_destroy_list, &head);
9965         spin_unlock(&nf_tables_destroy_list_lock);
9966
9967         if (list_empty(&head))
9968                 return;
9969
9970         synchronize_rcu();
9971
9972         list_for_each_entry_safe(trans, next, &head, list) {
9973                 nft_trans_list_del(trans);
9974                 nft_commit_release(trans);
9975         }
9976 }
9977
9978 void nf_tables_trans_destroy_flush_work(void)
9979 {
9980         flush_work(&trans_destroy_work);
9981 }
9982 EXPORT_SYMBOL_GPL(nf_tables_trans_destroy_flush_work);
9983
9984 static bool nft_expr_reduce(struct nft_regs_track *track,
9985                             const struct nft_expr *expr)
9986 {
9987         return false;
9988 }
9989
9990 static int nf_tables_commit_chain_prepare(struct net *net, struct nft_chain *chain)
9991 {
9992         const struct nft_expr *expr, *last;
9993         struct nft_regs_track track = {};
9994         unsigned int size, data_size;
9995         void *data, *data_boundary;
9996         struct nft_rule_dp *prule;
9997         struct nft_rule *rule;
9998
9999         /* already handled or inactive chain? */
10000         if (chain->blob_next || !nft_is_active_next(net, chain))
10001                 return 0;
10002
10003         data_size = 0;
10004         list_for_each_entry(rule, &chain->rules, list) {
10005                 if (nft_is_active_next(net, rule)) {
10006                         data_size += sizeof(*prule) + rule->dlen;
10007                         if (data_size > INT_MAX)
10008                                 return -ENOMEM;
10009                 }
10010         }
10011
10012         chain->blob_next = nf_tables_chain_alloc_rules(chain, data_size);
10013         if (!chain->blob_next)
10014                 return -ENOMEM;
10015
10016         data = (void *)chain->blob_next->data;
10017         data_boundary = data + data_size;
10018         size = 0;
10019
10020         list_for_each_entry(rule, &chain->rules, list) {
10021                 if (!nft_is_active_next(net, rule))
10022                         continue;
10023
10024                 prule = (struct nft_rule_dp *)data;
10025                 data += offsetof(struct nft_rule_dp, data);
10026                 if (WARN_ON_ONCE(data > data_boundary))
10027                         return -ENOMEM;
10028
10029                 size = 0;
10030                 track.last = nft_expr_last(rule);
10031                 nft_rule_for_each_expr(expr, last, rule) {
10032                         track.cur = expr;
10033
10034                         if (nft_expr_reduce(&track, expr)) {
10035                                 expr = track.cur;
10036                                 continue;
10037                         }
10038
10039                         if (WARN_ON_ONCE(data + size + expr->ops->size > data_boundary))
10040                                 return -ENOMEM;
10041
10042                         memcpy(data + size, expr, expr->ops->size);
10043                         size += expr->ops->size;
10044                 }
10045                 if (WARN_ON_ONCE(size >= 1 << 12))
10046                         return -ENOMEM;
10047
10048                 prule->handle = rule->handle;
10049                 prule->dlen = size;
10050                 prule->is_last = 0;
10051
10052                 data += size;
10053                 size = 0;
10054                 chain->blob_next->size += (unsigned long)(data - (void *)prule);
10055         }
10056
10057         if (WARN_ON_ONCE(data > data_boundary))
10058                 return -ENOMEM;
10059
10060         prule = (struct nft_rule_dp *)data;
10061         nft_last_rule(chain, prule);
10062
10063         return 0;
10064 }
10065
10066 static void nf_tables_commit_chain_prepare_cancel(struct net *net)
10067 {
10068         struct nftables_pernet *nft_net = nft_pernet(net);
10069         struct nft_trans *trans, *next;
10070
10071         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
10072                 if (trans->msg_type == NFT_MSG_NEWRULE ||
10073                     trans->msg_type == NFT_MSG_DELRULE) {
10074                         struct nft_chain *chain = nft_trans_rule_chain(trans);
10075
10076                         kvfree(chain->blob_next);
10077                         chain->blob_next = NULL;
10078                 }
10079         }
10080 }
10081
10082 static void __nf_tables_commit_chain_free_rules(struct rcu_head *h)
10083 {
10084         struct nft_rule_dp_last *l = container_of(h, struct nft_rule_dp_last, h);
10085
10086         kvfree(l->blob);
10087 }
10088
10089 static void nf_tables_commit_chain_free_rules_old(struct nft_rule_blob *blob)
10090 {
10091         struct nft_rule_dp_last *last;
10092
10093         /* last rule trailer is after end marker */
10094         last = (void *)blob + sizeof(*blob) + blob->size;
10095         last->blob = blob;
10096
10097         call_rcu(&last->h, __nf_tables_commit_chain_free_rules);
10098 }
10099
10100 static void nf_tables_commit_chain(struct net *net, struct nft_chain *chain)
10101 {
10102         struct nft_rule_blob *g0, *g1;
10103         bool next_genbit;
10104
10105         next_genbit = nft_gencursor_next(net);
10106
10107         g0 = rcu_dereference_protected(chain->blob_gen_0,
10108                                        lockdep_commit_lock_is_held(net));
10109         g1 = rcu_dereference_protected(chain->blob_gen_1,
10110                                        lockdep_commit_lock_is_held(net));
10111
10112         /* No changes to this chain? */
10113         if (chain->blob_next == NULL) {
10114                 /* chain had no change in last or next generation */
10115                 if (g0 == g1)
10116                         return;
10117                 /*
10118                  * chain had no change in this generation; make sure next
10119                  * one uses same rules as current generation.
10120                  */
10121                 if (next_genbit) {
10122                         rcu_assign_pointer(chain->blob_gen_1, g0);
10123                         nf_tables_commit_chain_free_rules_old(g1);
10124                 } else {
10125                         rcu_assign_pointer(chain->blob_gen_0, g1);
10126                         nf_tables_commit_chain_free_rules_old(g0);
10127                 }
10128
10129                 return;
10130         }
10131
10132         if (next_genbit)
10133                 rcu_assign_pointer(chain->blob_gen_1, chain->blob_next);
10134         else
10135                 rcu_assign_pointer(chain->blob_gen_0, chain->blob_next);
10136
10137         chain->blob_next = NULL;
10138
10139         if (g0 == g1)
10140                 return;
10141
10142         if (next_genbit)
10143                 nf_tables_commit_chain_free_rules_old(g1);
10144         else
10145                 nf_tables_commit_chain_free_rules_old(g0);
10146 }
10147
10148 static void nft_obj_del(struct nft_object *obj)
10149 {
10150         rhltable_remove(&nft_objname_ht, &obj->rhlhead, nft_objname_ht_params);
10151         list_del_rcu(&obj->list);
10152 }
10153
10154 void nft_chain_del(struct nft_chain *chain)
10155 {
10156         struct nft_table *table = chain->table;
10157
10158         WARN_ON_ONCE(rhltable_remove(&table->chains_ht, &chain->rhlhead,
10159                                      nft_chain_ht_params));
10160         list_del_rcu(&chain->list);
10161 }
10162
10163 static void nft_trans_gc_setelem_remove(struct nft_ctx *ctx,
10164                                         struct nft_trans_gc *trans)
10165 {
10166         struct nft_elem_priv **priv = trans->priv;
10167         unsigned int i;
10168
10169         for (i = 0; i < trans->count; i++) {
10170                 nft_setelem_data_deactivate(ctx->net, trans->set, priv[i]);
10171                 nft_setelem_remove(ctx->net, trans->set, priv[i]);
10172         }
10173 }
10174
10175 void nft_trans_gc_destroy(struct nft_trans_gc *trans)
10176 {
10177         nft_set_put(trans->set);
10178         put_net(trans->net);
10179         kfree(trans);
10180 }
10181
10182 static void nft_trans_gc_trans_free(struct rcu_head *rcu)
10183 {
10184         struct nft_elem_priv *elem_priv;
10185         struct nft_trans_gc *trans;
10186         struct nft_ctx ctx = {};
10187         unsigned int i;
10188
10189         trans = container_of(rcu, struct nft_trans_gc, rcu);
10190         ctx.net = read_pnet(&trans->set->net);
10191
10192         for (i = 0; i < trans->count; i++) {
10193                 elem_priv = trans->priv[i];
10194                 if (!nft_setelem_is_catchall(trans->set, elem_priv))
10195                         atomic_dec(&trans->set->nelems);
10196
10197                 nf_tables_set_elem_destroy(&ctx, trans->set, elem_priv);
10198         }
10199
10200         nft_trans_gc_destroy(trans);
10201 }
10202
10203 static bool nft_trans_gc_work_done(struct nft_trans_gc *trans)
10204 {
10205         struct nftables_pernet *nft_net;
10206         struct nft_ctx ctx = {};
10207
10208         nft_net = nft_pernet(trans->net);
10209
10210         mutex_lock(&nft_net->commit_mutex);
10211
10212         /* Check for race with transaction, otherwise this batch refers to
10213          * stale objects that might not be there anymore. Skip transaction if
10214          * set has been destroyed from control plane transaction in case gc
10215          * worker loses race.
10216          */
10217         if (READ_ONCE(nft_net->gc_seq) != trans->seq || trans->set->dead) {
10218                 mutex_unlock(&nft_net->commit_mutex);
10219                 return false;
10220         }
10221
10222         ctx.net = trans->net;
10223         ctx.table = trans->set->table;
10224
10225         nft_trans_gc_setelem_remove(&ctx, trans);
10226         mutex_unlock(&nft_net->commit_mutex);
10227
10228         return true;
10229 }
10230
10231 static void nft_trans_gc_work(struct work_struct *work)
10232 {
10233         struct nft_trans_gc *trans, *next;
10234         LIST_HEAD(trans_gc_list);
10235
10236         spin_lock(&nf_tables_gc_list_lock);
10237         list_splice_init(&nf_tables_gc_list, &trans_gc_list);
10238         spin_unlock(&nf_tables_gc_list_lock);
10239
10240         list_for_each_entry_safe(trans, next, &trans_gc_list, list) {
10241                 list_del(&trans->list);
10242                 if (!nft_trans_gc_work_done(trans)) {
10243                         nft_trans_gc_destroy(trans);
10244                         continue;
10245                 }
10246                 call_rcu(&trans->rcu, nft_trans_gc_trans_free);
10247         }
10248 }
10249
10250 struct nft_trans_gc *nft_trans_gc_alloc(struct nft_set *set,
10251                                         unsigned int gc_seq, gfp_t gfp)
10252 {
10253         struct net *net = read_pnet(&set->net);
10254         struct nft_trans_gc *trans;
10255
10256         trans = kzalloc(sizeof(*trans), gfp);
10257         if (!trans)
10258                 return NULL;
10259
10260         trans->net = maybe_get_net(net);
10261         if (!trans->net) {
10262                 kfree(trans);
10263                 return NULL;
10264         }
10265
10266         refcount_inc(&set->refs);
10267         trans->set = set;
10268         trans->seq = gc_seq;
10269
10270         return trans;
10271 }
10272
10273 void nft_trans_gc_elem_add(struct nft_trans_gc *trans, void *priv)
10274 {
10275         trans->priv[trans->count++] = priv;
10276 }
10277
10278 static void nft_trans_gc_queue_work(struct nft_trans_gc *trans)
10279 {
10280         spin_lock(&nf_tables_gc_list_lock);
10281         list_add_tail(&trans->list, &nf_tables_gc_list);
10282         spin_unlock(&nf_tables_gc_list_lock);
10283
10284         schedule_work(&trans_gc_work);
10285 }
10286
10287 static int nft_trans_gc_space(struct nft_trans_gc *trans)
10288 {
10289         return NFT_TRANS_GC_BATCHCOUNT - trans->count;
10290 }
10291
10292 struct nft_trans_gc *nft_trans_gc_queue_async(struct nft_trans_gc *gc,
10293                                               unsigned int gc_seq, gfp_t gfp)
10294 {
10295         struct nft_set *set;
10296
10297         if (nft_trans_gc_space(gc))
10298                 return gc;
10299
10300         set = gc->set;
10301         nft_trans_gc_queue_work(gc);
10302
10303         return nft_trans_gc_alloc(set, gc_seq, gfp);
10304 }
10305
10306 void nft_trans_gc_queue_async_done(struct nft_trans_gc *trans)
10307 {
10308         if (trans->count == 0) {
10309                 nft_trans_gc_destroy(trans);
10310                 return;
10311         }
10312
10313         nft_trans_gc_queue_work(trans);
10314 }
10315
10316 struct nft_trans_gc *nft_trans_gc_queue_sync(struct nft_trans_gc *gc, gfp_t gfp)
10317 {
10318         struct nft_set *set;
10319
10320         if (WARN_ON_ONCE(!lockdep_commit_lock_is_held(gc->net)))
10321                 return NULL;
10322
10323         if (nft_trans_gc_space(gc))
10324                 return gc;
10325
10326         set = gc->set;
10327         call_rcu(&gc->rcu, nft_trans_gc_trans_free);
10328
10329         return nft_trans_gc_alloc(set, 0, gfp);
10330 }
10331
10332 void nft_trans_gc_queue_sync_done(struct nft_trans_gc *trans)
10333 {
10334         WARN_ON_ONCE(!lockdep_commit_lock_is_held(trans->net));
10335
10336         if (trans->count == 0) {
10337                 nft_trans_gc_destroy(trans);
10338                 return;
10339         }
10340
10341         call_rcu(&trans->rcu, nft_trans_gc_trans_free);
10342 }
10343
10344 struct nft_trans_gc *nft_trans_gc_catchall_async(struct nft_trans_gc *gc,
10345                                                  unsigned int gc_seq)
10346 {
10347         struct nft_set_elem_catchall *catchall;
10348         const struct nft_set *set = gc->set;
10349         struct nft_set_ext *ext;
10350
10351         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
10352                 ext = nft_set_elem_ext(set, catchall->elem);
10353
10354                 if (!nft_set_elem_expired(ext))
10355                         continue;
10356                 if (nft_set_elem_is_dead(ext))
10357                         goto dead_elem;
10358
10359                 nft_set_elem_dead(ext);
10360 dead_elem:
10361                 gc = nft_trans_gc_queue_async(gc, gc_seq, GFP_ATOMIC);
10362                 if (!gc)
10363                         return NULL;
10364
10365                 nft_trans_gc_elem_add(gc, catchall->elem);
10366         }
10367
10368         return gc;
10369 }
10370
10371 struct nft_trans_gc *nft_trans_gc_catchall_sync(struct nft_trans_gc *gc)
10372 {
10373         struct nft_set_elem_catchall *catchall, *next;
10374         u64 tstamp = nft_net_tstamp(gc->net);
10375         const struct nft_set *set = gc->set;
10376         struct nft_elem_priv *elem_priv;
10377         struct nft_set_ext *ext;
10378
10379         WARN_ON_ONCE(!lockdep_commit_lock_is_held(gc->net));
10380
10381         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
10382                 ext = nft_set_elem_ext(set, catchall->elem);
10383
10384                 if (!__nft_set_elem_expired(ext, tstamp))
10385                         continue;
10386
10387                 gc = nft_trans_gc_queue_sync(gc, GFP_KERNEL);
10388                 if (!gc)
10389                         return NULL;
10390
10391                 elem_priv = catchall->elem;
10392                 nft_setelem_data_deactivate(gc->net, gc->set, elem_priv);
10393                 nft_setelem_catchall_destroy(catchall);
10394                 nft_trans_gc_elem_add(gc, elem_priv);
10395         }
10396
10397         return gc;
10398 }
10399
10400 static void nf_tables_module_autoload_cleanup(struct net *net)
10401 {
10402         struct nftables_pernet *nft_net = nft_pernet(net);
10403         struct nft_module_request *req, *next;
10404
10405         WARN_ON_ONCE(!list_empty(&nft_net->commit_list));
10406         list_for_each_entry_safe(req, next, &nft_net->module_list, list) {
10407                 WARN_ON_ONCE(!req->done);
10408                 list_del(&req->list);
10409                 kfree(req);
10410         }
10411 }
10412
10413 static void nf_tables_commit_release(struct net *net)
10414 {
10415         struct nftables_pernet *nft_net = nft_pernet(net);
10416         struct nft_trans *trans;
10417
10418         /* all side effects have to be made visible.
10419          * For example, if a chain named 'foo' has been deleted, a
10420          * new transaction must not find it anymore.
10421          *
10422          * Memory reclaim happens asynchronously from work queue
10423          * to prevent expensive synchronize_rcu() in commit phase.
10424          */
10425         if (list_empty(&nft_net->commit_list)) {
10426                 nf_tables_module_autoload_cleanup(net);
10427                 mutex_unlock(&nft_net->commit_mutex);
10428                 return;
10429         }
10430
10431         trans = list_last_entry(&nft_net->commit_list,
10432                                 struct nft_trans, list);
10433         get_net(trans->net);
10434         WARN_ON_ONCE(trans->put_net);
10435
10436         trans->put_net = true;
10437         spin_lock(&nf_tables_destroy_list_lock);
10438         list_splice_tail_init(&nft_net->commit_list, &nf_tables_destroy_list);
10439         spin_unlock(&nf_tables_destroy_list_lock);
10440
10441         nf_tables_module_autoload_cleanup(net);
10442         schedule_work(&trans_destroy_work);
10443
10444         mutex_unlock(&nft_net->commit_mutex);
10445 }
10446
10447 static void nft_commit_notify(struct net *net, u32 portid)
10448 {
10449         struct nftables_pernet *nft_net = nft_pernet(net);
10450         struct sk_buff *batch_skb = NULL, *nskb, *skb;
10451         unsigned char *data;
10452         int len;
10453
10454         list_for_each_entry_safe(skb, nskb, &nft_net->notify_list, list) {
10455                 if (!batch_skb) {
10456 new_batch:
10457                         batch_skb = skb;
10458                         len = NLMSG_GOODSIZE - skb->len;
10459                         list_del(&skb->list);
10460                         continue;
10461                 }
10462                 len -= skb->len;
10463                 if (len > 0 && NFT_CB(skb).report == NFT_CB(batch_skb).report) {
10464                         data = skb_put(batch_skb, skb->len);
10465                         memcpy(data, skb->data, skb->len);
10466                         list_del(&skb->list);
10467                         kfree_skb(skb);
10468                         continue;
10469                 }
10470                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
10471                                NFT_CB(batch_skb).report, GFP_KERNEL);
10472                 goto new_batch;
10473         }
10474
10475         if (batch_skb) {
10476                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
10477                                NFT_CB(batch_skb).report, GFP_KERNEL);
10478         }
10479
10480         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
10481 }
10482
10483 static int nf_tables_commit_audit_alloc(struct list_head *adl,
10484                                         struct nft_table *table)
10485 {
10486         struct nft_audit_data *adp;
10487
10488         list_for_each_entry(adp, adl, list) {
10489                 if (adp->table == table)
10490                         return 0;
10491         }
10492         adp = kzalloc(sizeof(*adp), GFP_KERNEL);
10493         if (!adp)
10494                 return -ENOMEM;
10495         adp->table = table;
10496         list_add(&adp->list, adl);
10497         return 0;
10498 }
10499
10500 static void nf_tables_commit_audit_free(struct list_head *adl)
10501 {
10502         struct nft_audit_data *adp, *adn;
10503
10504         list_for_each_entry_safe(adp, adn, adl, list) {
10505                 list_del(&adp->list);
10506                 kfree(adp);
10507         }
10508 }
10509
10510 /* nft audit emits the number of elements that get added/removed/updated,
10511  * so NEW/DELSETELEM needs to increment based on the total elem count.
10512  */
10513 static unsigned int nf_tables_commit_audit_entrycount(const struct nft_trans *trans)
10514 {
10515         switch (trans->msg_type) {
10516         case NFT_MSG_NEWSETELEM:
10517         case NFT_MSG_DELSETELEM:
10518                 return nft_trans_container_elem(trans)->nelems;
10519         }
10520
10521         return 1;
10522 }
10523
10524 static void nf_tables_commit_audit_collect(struct list_head *adl,
10525                                            const struct nft_trans *trans, u32 op)
10526 {
10527         const struct nft_table *table = trans->table;
10528         struct nft_audit_data *adp;
10529
10530         list_for_each_entry(adp, adl, list) {
10531                 if (adp->table == table)
10532                         goto found;
10533         }
10534         WARN_ONCE(1, "table=%s not expected in commit list", table->name);
10535         return;
10536 found:
10537         adp->entries += nf_tables_commit_audit_entrycount(trans);
10538         if (!adp->op || adp->op > op)
10539                 adp->op = op;
10540 }
10541
10542 #define AUNFTABLENAMELEN (NFT_TABLE_MAXNAMELEN + 22)
10543
10544 static void nf_tables_commit_audit_log(struct list_head *adl, u32 generation)
10545 {
10546         struct nft_audit_data *adp, *adn;
10547         char aubuf[AUNFTABLENAMELEN];
10548
10549         list_for_each_entry_safe(adp, adn, adl, list) {
10550                 snprintf(aubuf, AUNFTABLENAMELEN, "%s:%u", adp->table->name,
10551                          generation);
10552                 audit_log_nfcfg(aubuf, adp->table->family, adp->entries,
10553                                 nft2audit_op[adp->op], GFP_KERNEL);
10554                 list_del(&adp->list);
10555                 kfree(adp);
10556         }
10557 }
10558
10559 static void nft_set_commit_update(struct list_head *set_update_list)
10560 {
10561         struct nft_set *set, *next;
10562
10563         list_for_each_entry_safe(set, next, set_update_list, pending_update) {
10564                 list_del_init(&set->pending_update);
10565
10566                 if (!set->ops->commit || set->dead)
10567                         continue;
10568
10569                 set->ops->commit(set);
10570         }
10571 }
10572
10573 static unsigned int nft_gc_seq_begin(struct nftables_pernet *nft_net)
10574 {
10575         unsigned int gc_seq;
10576
10577         /* Bump gc counter, it becomes odd, this is the busy mark. */
10578         gc_seq = READ_ONCE(nft_net->gc_seq);
10579         WRITE_ONCE(nft_net->gc_seq, ++gc_seq);
10580
10581         return gc_seq;
10582 }
10583
10584 static void nft_gc_seq_end(struct nftables_pernet *nft_net, unsigned int gc_seq)
10585 {
10586         WRITE_ONCE(nft_net->gc_seq, ++gc_seq);
10587 }
10588
10589 static int nf_tables_commit(struct net *net, struct sk_buff *skb)
10590 {
10591         struct nftables_pernet *nft_net = nft_pernet(net);
10592         const struct nlmsghdr *nlh = nlmsg_hdr(skb);
10593         struct nft_trans_binding *trans_binding;
10594         struct nft_trans *trans, *next;
10595         unsigned int base_seq, gc_seq;
10596         LIST_HEAD(set_update_list);
10597         struct nft_trans_elem *te;
10598         struct nft_chain *chain;
10599         struct nft_table *table;
10600         struct nft_ctx ctx;
10601         LIST_HEAD(adl);
10602         int err;
10603
10604         if (list_empty(&nft_net->commit_list)) {
10605                 mutex_unlock(&nft_net->commit_mutex);
10606                 return 0;
10607         }
10608
10609         nft_ctx_init(&ctx, net, skb, nlh, NFPROTO_UNSPEC, NULL, NULL, NULL);
10610
10611         list_for_each_entry(trans_binding, &nft_net->binding_list, binding_list) {
10612                 trans = &trans_binding->nft_trans;
10613                 switch (trans->msg_type) {
10614                 case NFT_MSG_NEWSET:
10615                         if (!nft_trans_set_update(trans) &&
10616                             nft_set_is_anonymous(nft_trans_set(trans)) &&
10617                             !nft_trans_set_bound(trans)) {
10618                                 pr_warn_once("nftables ruleset with unbound set\n");
10619                                 return -EINVAL;
10620                         }
10621                         break;
10622                 case NFT_MSG_NEWCHAIN:
10623                         if (!nft_trans_chain_update(trans) &&
10624                             nft_chain_binding(nft_trans_chain(trans)) &&
10625                             !nft_trans_chain_bound(trans)) {
10626                                 pr_warn_once("nftables ruleset with unbound chain\n");
10627                                 return -EINVAL;
10628                         }
10629                         break;
10630                 default:
10631                         WARN_ONCE(1, "Unhandled bind type %d", trans->msg_type);
10632                         break;
10633                 }
10634         }
10635
10636         /* 0. Validate ruleset, otherwise roll back for error reporting. */
10637         if (nf_tables_validate(net) < 0) {
10638                 nft_net->validate_state = NFT_VALIDATE_DO;
10639                 return -EAGAIN;
10640         }
10641
10642         err = nft_flow_rule_offload_commit(net);
10643         if (err < 0)
10644                 return err;
10645
10646         /* 1.  Allocate space for next generation rules_gen_X[] */
10647         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
10648                 struct nft_table *table = trans->table;
10649                 int ret;
10650
10651                 ret = nf_tables_commit_audit_alloc(&adl, table);
10652                 if (ret) {
10653                         nf_tables_commit_chain_prepare_cancel(net);
10654                         nf_tables_commit_audit_free(&adl);
10655                         return ret;
10656                 }
10657                 if (trans->msg_type == NFT_MSG_NEWRULE ||
10658                     trans->msg_type == NFT_MSG_DELRULE) {
10659                         chain = nft_trans_rule_chain(trans);
10660
10661                         ret = nf_tables_commit_chain_prepare(net, chain);
10662                         if (ret < 0) {
10663                                 nf_tables_commit_chain_prepare_cancel(net);
10664                                 nf_tables_commit_audit_free(&adl);
10665                                 return ret;
10666                         }
10667                 }
10668         }
10669
10670         /* step 2.  Make rules_gen_X visible to packet path */
10671         list_for_each_entry(table, &nft_net->tables, list) {
10672                 list_for_each_entry(chain, &table->chains, list)
10673                         nf_tables_commit_chain(net, chain);
10674         }
10675
10676         /*
10677          * Bump generation counter, invalidate any dump in progress.
10678          * Cannot fail after this point.
10679          */
10680         base_seq = READ_ONCE(nft_net->base_seq);
10681         while (++base_seq == 0)
10682                 ;
10683
10684         WRITE_ONCE(nft_net->base_seq, base_seq);
10685
10686         gc_seq = nft_gc_seq_begin(nft_net);
10687
10688         /* step 3. Start new generation, rules_gen_X now in use. */
10689         net->nft.gencursor = nft_gencursor_next(net);
10690
10691         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
10692                 struct nft_table *table = trans->table;
10693
10694                 nft_ctx_update(&ctx, trans);
10695
10696                 nf_tables_commit_audit_collect(&adl, trans, trans->msg_type);
10697                 switch (trans->msg_type) {
10698                 case NFT_MSG_NEWTABLE:
10699                         if (nft_trans_table_update(trans)) {
10700                                 if (!(table->flags & __NFT_TABLE_F_UPDATE)) {
10701                                         nft_trans_destroy(trans);
10702                                         break;
10703                                 }
10704                                 if (table->flags & NFT_TABLE_F_DORMANT)
10705                                         nf_tables_table_disable(net, table);
10706
10707                                 table->flags &= ~__NFT_TABLE_F_UPDATE;
10708                         } else {
10709                                 nft_clear(net, table);
10710                         }
10711                         nf_tables_table_notify(&ctx, NFT_MSG_NEWTABLE);
10712                         nft_trans_destroy(trans);
10713                         break;
10714                 case NFT_MSG_DELTABLE:
10715                 case NFT_MSG_DESTROYTABLE:
10716                         list_del_rcu(&table->list);
10717                         nf_tables_table_notify(&ctx, trans->msg_type);
10718                         break;
10719                 case NFT_MSG_NEWCHAIN:
10720                         if (nft_trans_chain_update(trans)) {
10721                                 nft_chain_commit_update(nft_trans_container_chain(trans));
10722                                 nf_tables_chain_notify(&ctx, NFT_MSG_NEWCHAIN,
10723                                                        &nft_trans_chain_hooks(trans));
10724                                 list_splice(&nft_trans_chain_hooks(trans),
10725                                             &nft_trans_basechain(trans)->hook_list);
10726                                 /* trans destroyed after rcu grace period */
10727                         } else {
10728                                 nft_chain_commit_drop_policy(nft_trans_container_chain(trans));
10729                                 nft_clear(net, nft_trans_chain(trans));
10730                                 nf_tables_chain_notify(&ctx, NFT_MSG_NEWCHAIN, NULL);
10731                                 nft_trans_destroy(trans);
10732                         }
10733                         break;
10734                 case NFT_MSG_DELCHAIN:
10735                 case NFT_MSG_DESTROYCHAIN:
10736                         if (nft_trans_chain_update(trans)) {
10737                                 nf_tables_chain_notify(&ctx, NFT_MSG_DELCHAIN,
10738                                                        &nft_trans_chain_hooks(trans));
10739                                 if (!(table->flags & NFT_TABLE_F_DORMANT)) {
10740                                         nft_netdev_unregister_hooks(net,
10741                                                                     &nft_trans_chain_hooks(trans),
10742                                                                     true);
10743                                 }
10744                         } else {
10745                                 nft_chain_del(nft_trans_chain(trans));
10746                                 nf_tables_chain_notify(&ctx, NFT_MSG_DELCHAIN,
10747                                                        NULL);
10748                                 nf_tables_unregister_hook(ctx.net, ctx.table,
10749                                                           nft_trans_chain(trans));
10750                         }
10751                         break;
10752                 case NFT_MSG_NEWRULE:
10753                         nft_clear(net, nft_trans_rule(trans));
10754                         nf_tables_rule_notify(&ctx, nft_trans_rule(trans),
10755                                               NFT_MSG_NEWRULE);
10756                         if (nft_trans_rule_chain(trans)->flags & NFT_CHAIN_HW_OFFLOAD)
10757                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10758
10759                         nft_trans_destroy(trans);
10760                         break;
10761                 case NFT_MSG_DELRULE:
10762                 case NFT_MSG_DESTROYRULE:
10763                         list_del_rcu(&nft_trans_rule(trans)->list);
10764                         nf_tables_rule_notify(&ctx, nft_trans_rule(trans),
10765                                               trans->msg_type);
10766                         nft_rule_expr_deactivate(&ctx, nft_trans_rule(trans),
10767                                                  NFT_TRANS_COMMIT);
10768
10769                         if (nft_trans_rule_chain(trans)->flags & NFT_CHAIN_HW_OFFLOAD)
10770                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10771                         break;
10772                 case NFT_MSG_NEWSET:
10773                         list_del(&nft_trans_container_set(trans)->list_trans_newset);
10774                         if (nft_trans_set_update(trans)) {
10775                                 struct nft_set *set = nft_trans_set(trans);
10776
10777                                 WRITE_ONCE(set->timeout, nft_trans_set_timeout(trans));
10778                                 WRITE_ONCE(set->gc_int, nft_trans_set_gc_int(trans));
10779
10780                                 if (nft_trans_set_size(trans))
10781                                         WRITE_ONCE(set->size, nft_trans_set_size(trans));
10782                         } else {
10783                                 nft_clear(net, nft_trans_set(trans));
10784                                 /* This avoids hitting -EBUSY when deleting the table
10785                                  * from the transaction.
10786                                  */
10787                                 if (nft_set_is_anonymous(nft_trans_set(trans)) &&
10788                                     !list_empty(&nft_trans_set(trans)->bindings))
10789                                         nft_use_dec(&table->use);
10790                         }
10791                         nf_tables_set_notify(&ctx, nft_trans_set(trans),
10792                                              NFT_MSG_NEWSET, GFP_KERNEL);
10793                         nft_trans_destroy(trans);
10794                         break;
10795                 case NFT_MSG_DELSET:
10796                 case NFT_MSG_DESTROYSET:
10797                         nft_trans_set(trans)->dead = 1;
10798                         list_del_rcu(&nft_trans_set(trans)->list);
10799                         nf_tables_set_notify(&ctx, nft_trans_set(trans),
10800                                              trans->msg_type, GFP_KERNEL);
10801                         break;
10802                 case NFT_MSG_NEWSETELEM:
10803                         te = nft_trans_container_elem(trans);
10804
10805                         nft_trans_elems_add(&ctx, te);
10806
10807                         if (te->set->ops->commit &&
10808                             list_empty(&te->set->pending_update)) {
10809                                 list_add_tail(&te->set->pending_update,
10810                                               &set_update_list);
10811                         }
10812                         nft_trans_destroy(trans);
10813                         break;
10814                 case NFT_MSG_DELSETELEM:
10815                 case NFT_MSG_DESTROYSETELEM:
10816                         te = nft_trans_container_elem(trans);
10817
10818                         nft_trans_elems_remove(&ctx, te);
10819
10820                         if (te->set->ops->commit &&
10821                             list_empty(&te->set->pending_update)) {
10822                                 list_add_tail(&te->set->pending_update,
10823                                               &set_update_list);
10824                         }
10825                         break;
10826                 case NFT_MSG_NEWOBJ:
10827                         if (nft_trans_obj_update(trans)) {
10828                                 nft_obj_commit_update(&ctx, trans);
10829                                 nf_tables_obj_notify(&ctx,
10830                                                      nft_trans_obj(trans),
10831                                                      NFT_MSG_NEWOBJ);
10832                         } else {
10833                                 nft_clear(net, nft_trans_obj(trans));
10834                                 nf_tables_obj_notify(&ctx,
10835                                                      nft_trans_obj(trans),
10836                                                      NFT_MSG_NEWOBJ);
10837                                 nft_trans_destroy(trans);
10838                         }
10839                         break;
10840                 case NFT_MSG_DELOBJ:
10841                 case NFT_MSG_DESTROYOBJ:
10842                         nft_obj_del(nft_trans_obj(trans));
10843                         nf_tables_obj_notify(&ctx, nft_trans_obj(trans),
10844                                              trans->msg_type);
10845                         break;
10846                 case NFT_MSG_NEWFLOWTABLE:
10847                         if (nft_trans_flowtable_update(trans)) {
10848                                 nft_trans_flowtable(trans)->data.flags =
10849                                         nft_trans_flowtable_flags(trans);
10850                                 nf_tables_flowtable_notify(&ctx,
10851                                                            nft_trans_flowtable(trans),
10852                                                            &nft_trans_flowtable_hooks(trans),
10853                                                            NFT_MSG_NEWFLOWTABLE);
10854                                 list_splice(&nft_trans_flowtable_hooks(trans),
10855                                             &nft_trans_flowtable(trans)->hook_list);
10856                         } else {
10857                                 nft_clear(net, nft_trans_flowtable(trans));
10858                                 nf_tables_flowtable_notify(&ctx,
10859                                                            nft_trans_flowtable(trans),
10860                                                            NULL,
10861                                                            NFT_MSG_NEWFLOWTABLE);
10862                         }
10863                         nft_trans_destroy(trans);
10864                         break;
10865                 case NFT_MSG_DELFLOWTABLE:
10866                 case NFT_MSG_DESTROYFLOWTABLE:
10867                         if (nft_trans_flowtable_update(trans)) {
10868                                 nf_tables_flowtable_notify(&ctx,
10869                                                            nft_trans_flowtable(trans),
10870                                                            &nft_trans_flowtable_hooks(trans),
10871                                                            trans->msg_type);
10872                                 nft_unregister_flowtable_net_hooks(net,
10873                                                                    &nft_trans_flowtable_hooks(trans));
10874                         } else {
10875                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
10876                                 nf_tables_flowtable_notify(&ctx,
10877                                                            nft_trans_flowtable(trans),
10878                                                            NULL,
10879                                                            trans->msg_type);
10880                                 nft_unregister_flowtable_net_hooks(net,
10881                                                 &nft_trans_flowtable(trans)->hook_list);
10882                         }
10883                         break;
10884                 }
10885         }
10886
10887         nft_set_commit_update(&set_update_list);
10888
10889         nft_commit_notify(net, NETLINK_CB(skb).portid);
10890         nf_tables_gen_notify(net, skb, NFT_MSG_NEWGEN);
10891         nf_tables_commit_audit_log(&adl, nft_net->base_seq);
10892
10893         nft_gc_seq_end(nft_net, gc_seq);
10894         nft_net->validate_state = NFT_VALIDATE_SKIP;
10895         nf_tables_commit_release(net);
10896
10897         return 0;
10898 }
10899
10900 static void nf_tables_module_autoload(struct net *net)
10901 {
10902         struct nftables_pernet *nft_net = nft_pernet(net);
10903         struct nft_module_request *req, *next;
10904         LIST_HEAD(module_list);
10905
10906         list_splice_init(&nft_net->module_list, &module_list);
10907         mutex_unlock(&nft_net->commit_mutex);
10908         list_for_each_entry_safe(req, next, &module_list, list) {
10909                 request_module("%s", req->module);
10910                 req->done = true;
10911         }
10912         mutex_lock(&nft_net->commit_mutex);
10913         list_splice(&module_list, &nft_net->module_list);
10914 }
10915
10916 static void nf_tables_abort_release(struct nft_trans *trans)
10917 {
10918         struct nft_ctx ctx = { };
10919
10920         nft_ctx_update(&ctx, trans);
10921
10922         switch (trans->msg_type) {
10923         case NFT_MSG_NEWTABLE:
10924                 nf_tables_table_destroy(trans->table);
10925                 break;
10926         case NFT_MSG_NEWCHAIN:
10927                 if (nft_trans_chain_update(trans))
10928                         nft_hooks_destroy(&nft_trans_chain_hooks(trans));
10929                 else
10930                         nf_tables_chain_destroy(nft_trans_chain(trans));
10931                 break;
10932         case NFT_MSG_NEWRULE:
10933                 nf_tables_rule_destroy(&ctx, nft_trans_rule(trans));
10934                 break;
10935         case NFT_MSG_NEWSET:
10936                 nft_set_destroy(&ctx, nft_trans_set(trans));
10937                 break;
10938         case NFT_MSG_NEWSETELEM:
10939                 nft_trans_set_elem_destroy(&ctx, nft_trans_container_elem(trans));
10940                 break;
10941         case NFT_MSG_NEWOBJ:
10942                 nft_obj_destroy(&ctx, nft_trans_obj(trans));
10943                 break;
10944         case NFT_MSG_NEWFLOWTABLE:
10945                 if (nft_trans_flowtable_update(trans))
10946                         nft_hooks_destroy(&nft_trans_flowtable_hooks(trans));
10947                 else
10948                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
10949                 break;
10950         }
10951         kfree(trans);
10952 }
10953
10954 static void nft_set_abort_update(struct list_head *set_update_list)
10955 {
10956         struct nft_set *set, *next;
10957
10958         list_for_each_entry_safe(set, next, set_update_list, pending_update) {
10959                 list_del_init(&set->pending_update);
10960
10961                 if (!set->ops->abort)
10962                         continue;
10963
10964                 set->ops->abort(set);
10965         }
10966 }
10967
10968 static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action)
10969 {
10970         struct nftables_pernet *nft_net = nft_pernet(net);
10971         struct nft_trans *trans, *next;
10972         LIST_HEAD(set_update_list);
10973         struct nft_trans_elem *te;
10974         struct nft_ctx ctx = {
10975                 .net = net,
10976         };
10977         int err = 0;
10978
10979         if (action == NFNL_ABORT_VALIDATE &&
10980             nf_tables_validate(net) < 0)
10981                 err = -EAGAIN;
10982
10983         list_for_each_entry_safe_reverse(trans, next, &nft_net->commit_list,
10984                                          list) {
10985                 struct nft_table *table = trans->table;
10986
10987                 nft_ctx_update(&ctx, trans);
10988
10989                 switch (trans->msg_type) {
10990                 case NFT_MSG_NEWTABLE:
10991                         if (nft_trans_table_update(trans)) {
10992                                 if (!(table->flags & __NFT_TABLE_F_UPDATE)) {
10993                                         nft_trans_destroy(trans);
10994                                         break;
10995                                 }
10996                                 if (table->flags & __NFT_TABLE_F_WAS_DORMANT) {
10997                                         nf_tables_table_disable(net, table);
10998                                         table->flags |= NFT_TABLE_F_DORMANT;
10999                                 } else if (table->flags & __NFT_TABLE_F_WAS_AWAKEN) {
11000                                         table->flags &= ~NFT_TABLE_F_DORMANT;
11001                                 }
11002                                 if (table->flags & __NFT_TABLE_F_WAS_ORPHAN) {
11003                                         table->flags &= ~NFT_TABLE_F_OWNER;
11004                                         table->nlpid = 0;
11005                                 }
11006                                 table->flags &= ~__NFT_TABLE_F_UPDATE;
11007                                 nft_trans_destroy(trans);
11008                         } else {
11009                                 list_del_rcu(&table->list);
11010                         }
11011                         break;
11012                 case NFT_MSG_DELTABLE:
11013                 case NFT_MSG_DESTROYTABLE:
11014                         nft_clear(trans->net, table);
11015                         nft_trans_destroy(trans);
11016                         break;
11017                 case NFT_MSG_NEWCHAIN:
11018                         if (nft_trans_chain_update(trans)) {
11019                                 if (!(table->flags & NFT_TABLE_F_DORMANT)) {
11020                                         nft_netdev_unregister_hooks(net,
11021                                                                     &nft_trans_chain_hooks(trans),
11022                                                                     true);
11023                                 }
11024                                 free_percpu(nft_trans_chain_stats(trans));
11025                                 kfree(nft_trans_chain_name(trans));
11026                                 nft_trans_destroy(trans);
11027                         } else {
11028                                 if (nft_trans_chain_bound(trans)) {
11029                                         nft_trans_destroy(trans);
11030                                         break;
11031                                 }
11032                                 nft_use_dec_restore(&table->use);
11033                                 nft_chain_del(nft_trans_chain(trans));
11034                                 nf_tables_unregister_hook(trans->net, table,
11035                                                           nft_trans_chain(trans));
11036                         }
11037                         break;
11038                 case NFT_MSG_DELCHAIN:
11039                 case NFT_MSG_DESTROYCHAIN:
11040                         if (nft_trans_chain_update(trans)) {
11041                                 list_splice(&nft_trans_chain_hooks(trans),
11042                                             &nft_trans_basechain(trans)->hook_list);
11043                         } else {
11044                                 nft_use_inc_restore(&table->use);
11045                                 nft_clear(trans->net, nft_trans_chain(trans));
11046                         }
11047                         nft_trans_destroy(trans);
11048                         break;
11049                 case NFT_MSG_NEWRULE:
11050                         if (nft_trans_rule_bound(trans)) {
11051                                 nft_trans_destroy(trans);
11052                                 break;
11053                         }
11054                         nft_use_dec_restore(&nft_trans_rule_chain(trans)->use);
11055                         list_del_rcu(&nft_trans_rule(trans)->list);
11056                         nft_rule_expr_deactivate(&ctx,
11057                                                  nft_trans_rule(trans),
11058                                                  NFT_TRANS_ABORT);
11059                         if (nft_trans_rule_chain(trans)->flags & NFT_CHAIN_HW_OFFLOAD)
11060                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
11061                         break;
11062                 case NFT_MSG_DELRULE:
11063                 case NFT_MSG_DESTROYRULE:
11064                         nft_use_inc_restore(&nft_trans_rule_chain(trans)->use);
11065                         nft_clear(trans->net, nft_trans_rule(trans));
11066                         nft_rule_expr_activate(&ctx, nft_trans_rule(trans));
11067                         if (nft_trans_rule_chain(trans)->flags & NFT_CHAIN_HW_OFFLOAD)
11068                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
11069
11070                         nft_trans_destroy(trans);
11071                         break;
11072                 case NFT_MSG_NEWSET:
11073                         list_del(&nft_trans_container_set(trans)->list_trans_newset);
11074                         if (nft_trans_set_update(trans)) {
11075                                 nft_trans_destroy(trans);
11076                                 break;
11077                         }
11078                         nft_use_dec_restore(&table->use);
11079                         if (nft_trans_set_bound(trans)) {
11080                                 nft_trans_destroy(trans);
11081                                 break;
11082                         }
11083                         nft_trans_set(trans)->dead = 1;
11084                         list_del_rcu(&nft_trans_set(trans)->list);
11085                         break;
11086                 case NFT_MSG_DELSET:
11087                 case NFT_MSG_DESTROYSET:
11088                         nft_use_inc_restore(&table->use);
11089                         nft_clear(trans->net, nft_trans_set(trans));
11090                         if (nft_trans_set(trans)->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
11091                                 nft_map_activate(&ctx, nft_trans_set(trans));
11092
11093                         nft_trans_destroy(trans);
11094                         break;
11095                 case NFT_MSG_NEWSETELEM:
11096                         if (nft_trans_elem_set_bound(trans)) {
11097                                 nft_trans_destroy(trans);
11098                                 break;
11099                         }
11100                         te = nft_trans_container_elem(trans);
11101                         if (!nft_trans_elems_new_abort(&ctx, te)) {
11102                                 nft_trans_destroy(trans);
11103                                 break;
11104                         }
11105
11106                         if (te->set->ops->abort &&
11107                             list_empty(&te->set->pending_update)) {
11108                                 list_add_tail(&te->set->pending_update,
11109                                               &set_update_list);
11110                         }
11111                         break;
11112                 case NFT_MSG_DELSETELEM:
11113                 case NFT_MSG_DESTROYSETELEM:
11114                         te = nft_trans_container_elem(trans);
11115
11116                         nft_trans_elems_destroy_abort(&ctx, te);
11117
11118                         if (te->set->ops->abort &&
11119                             list_empty(&te->set->pending_update)) {
11120                                 list_add_tail(&te->set->pending_update,
11121                                               &set_update_list);
11122                         }
11123                         nft_trans_destroy(trans);
11124                         break;
11125                 case NFT_MSG_NEWOBJ:
11126                         if (nft_trans_obj_update(trans)) {
11127                                 nft_obj_destroy(&ctx, nft_trans_obj_newobj(trans));
11128                                 nft_trans_destroy(trans);
11129                         } else {
11130                                 nft_use_dec_restore(&table->use);
11131                                 nft_obj_del(nft_trans_obj(trans));
11132                         }
11133                         break;
11134                 case NFT_MSG_DELOBJ:
11135                 case NFT_MSG_DESTROYOBJ:
11136                         nft_use_inc_restore(&table->use);
11137                         nft_clear(trans->net, nft_trans_obj(trans));
11138                         nft_trans_destroy(trans);
11139                         break;
11140                 case NFT_MSG_NEWFLOWTABLE:
11141                         if (nft_trans_flowtable_update(trans)) {
11142                                 nft_unregister_flowtable_net_hooks(net,
11143                                                 &nft_trans_flowtable_hooks(trans));
11144                         } else {
11145                                 nft_use_dec_restore(&table->use);
11146                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
11147                                 nft_unregister_flowtable_net_hooks(net,
11148                                                 &nft_trans_flowtable(trans)->hook_list);
11149                         }
11150                         break;
11151                 case NFT_MSG_DELFLOWTABLE:
11152                 case NFT_MSG_DESTROYFLOWTABLE:
11153                         if (nft_trans_flowtable_update(trans)) {
11154                                 list_splice(&nft_trans_flowtable_hooks(trans),
11155                                             &nft_trans_flowtable(trans)->hook_list);
11156                         } else {
11157                                 nft_use_inc_restore(&table->use);
11158                                 nft_clear(trans->net, nft_trans_flowtable(trans));
11159                         }
11160                         nft_trans_destroy(trans);
11161                         break;
11162                 }
11163         }
11164
11165         WARN_ON_ONCE(!list_empty(&nft_net->commit_set_list));
11166
11167         nft_set_abort_update(&set_update_list);
11168
11169         synchronize_rcu();
11170
11171         list_for_each_entry_safe_reverse(trans, next,
11172                                          &nft_net->commit_list, list) {
11173                 nft_trans_list_del(trans);
11174                 nf_tables_abort_release(trans);
11175         }
11176
11177         return err;
11178 }
11179
11180 static int nf_tables_abort(struct net *net, struct sk_buff *skb,
11181                            enum nfnl_abort_action action)
11182 {
11183         struct nftables_pernet *nft_net = nft_pernet(net);
11184         unsigned int gc_seq;
11185         int ret;
11186
11187         gc_seq = nft_gc_seq_begin(nft_net);
11188         ret = __nf_tables_abort(net, action);
11189         nft_gc_seq_end(nft_net, gc_seq);
11190
11191         WARN_ON_ONCE(!list_empty(&nft_net->commit_list));
11192
11193         /* module autoload needs to happen after GC sequence update because it
11194          * temporarily releases and grabs mutex again.
11195          */
11196         if (action == NFNL_ABORT_AUTOLOAD)
11197                 nf_tables_module_autoload(net);
11198         else
11199                 nf_tables_module_autoload_cleanup(net);
11200
11201         mutex_unlock(&nft_net->commit_mutex);
11202
11203         return ret;
11204 }
11205
11206 static bool nf_tables_valid_genid(struct net *net, u32 genid)
11207 {
11208         struct nftables_pernet *nft_net = nft_pernet(net);
11209         bool genid_ok;
11210
11211         mutex_lock(&nft_net->commit_mutex);
11212         nft_net->tstamp = get_jiffies_64();
11213
11214         genid_ok = genid == 0 || nft_net->base_seq == genid;
11215         if (!genid_ok)
11216                 mutex_unlock(&nft_net->commit_mutex);
11217
11218         /* else, commit mutex has to be released by commit or abort function */
11219         return genid_ok;
11220 }
11221
11222 static const struct nfnetlink_subsystem nf_tables_subsys = {
11223         .name           = "nf_tables",
11224         .subsys_id      = NFNL_SUBSYS_NFTABLES,
11225         .cb_count       = NFT_MSG_MAX,
11226         .cb             = nf_tables_cb,
11227         .commit         = nf_tables_commit,
11228         .abort          = nf_tables_abort,
11229         .valid_genid    = nf_tables_valid_genid,
11230         .owner          = THIS_MODULE,
11231 };
11232
11233 int nft_chain_validate_dependency(const struct nft_chain *chain,
11234                                   enum nft_chain_types type)
11235 {
11236         const struct nft_base_chain *basechain;
11237
11238         if (nft_is_base_chain(chain)) {
11239                 basechain = nft_base_chain(chain);
11240                 if (basechain->type->type != type)
11241                         return -EOPNOTSUPP;
11242         }
11243         return 0;
11244 }
11245 EXPORT_SYMBOL_GPL(nft_chain_validate_dependency);
11246
11247 int nft_chain_validate_hooks(const struct nft_chain *chain,
11248                              unsigned int hook_flags)
11249 {
11250         struct nft_base_chain *basechain;
11251
11252         if (nft_is_base_chain(chain)) {
11253                 basechain = nft_base_chain(chain);
11254
11255                 if ((1 << basechain->ops.hooknum) & hook_flags)
11256                         return 0;
11257
11258                 return -EOPNOTSUPP;
11259         }
11260
11261         return 0;
11262 }
11263 EXPORT_SYMBOL_GPL(nft_chain_validate_hooks);
11264
11265 /**
11266  *      nft_parse_u32_check - fetch u32 attribute and check for maximum value
11267  *
11268  *      @attr: netlink attribute to fetch value from
11269  *      @max: maximum value to be stored in dest
11270  *      @dest: pointer to the variable
11271  *
11272  *      Parse, check and store a given u32 netlink attribute into variable.
11273  *      This function returns -ERANGE if the value goes over maximum value.
11274  *      Otherwise a 0 is returned and the attribute value is stored in the
11275  *      destination variable.
11276  */
11277 int nft_parse_u32_check(const struct nlattr *attr, int max, u32 *dest)
11278 {
11279         u32 val;
11280
11281         val = ntohl(nla_get_be32(attr));
11282         if (val > max)
11283                 return -ERANGE;
11284
11285         *dest = val;
11286         return 0;
11287 }
11288 EXPORT_SYMBOL_GPL(nft_parse_u32_check);
11289
11290 static int nft_parse_register(const struct nlattr *attr, u32 *preg)
11291 {
11292         unsigned int reg;
11293
11294         reg = ntohl(nla_get_be32(attr));
11295         switch (reg) {
11296         case NFT_REG_VERDICT...NFT_REG_4:
11297                 *preg = reg * NFT_REG_SIZE / NFT_REG32_SIZE;
11298                 break;
11299         case NFT_REG32_00...NFT_REG32_15:
11300                 *preg = reg + NFT_REG_SIZE / NFT_REG32_SIZE - NFT_REG32_00;
11301                 break;
11302         default:
11303                 return -ERANGE;
11304         }
11305
11306         return 0;
11307 }
11308
11309 /**
11310  *      nft_dump_register - dump a register value to a netlink attribute
11311  *
11312  *      @skb: socket buffer
11313  *      @attr: attribute number
11314  *      @reg: register number
11315  *
11316  *      Construct a netlink attribute containing the register number. For
11317  *      compatibility reasons, register numbers being a multiple of 4 are
11318  *      translated to the corresponding 128 bit register numbers.
11319  */
11320 int nft_dump_register(struct sk_buff *skb, unsigned int attr, unsigned int reg)
11321 {
11322         if (reg % (NFT_REG_SIZE / NFT_REG32_SIZE) == 0)
11323                 reg = reg / (NFT_REG_SIZE / NFT_REG32_SIZE);
11324         else
11325                 reg = reg - NFT_REG_SIZE / NFT_REG32_SIZE + NFT_REG32_00;
11326
11327         return nla_put_be32(skb, attr, htonl(reg));
11328 }
11329 EXPORT_SYMBOL_GPL(nft_dump_register);
11330
11331 static int nft_validate_register_load(enum nft_registers reg, unsigned int len)
11332 {
11333         if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
11334                 return -EINVAL;
11335         if (len == 0)
11336                 return -EINVAL;
11337         if (reg * NFT_REG32_SIZE + len > sizeof_field(struct nft_regs, data))
11338                 return -ERANGE;
11339
11340         return 0;
11341 }
11342
11343 int nft_parse_register_load(const struct nft_ctx *ctx,
11344                             const struct nlattr *attr, u8 *sreg, u32 len)
11345 {
11346         int err, invalid_reg;
11347         u32 reg, next_register;
11348
11349         err = nft_parse_register(attr, &reg);
11350         if (err < 0)
11351                 return err;
11352
11353         err = nft_validate_register_load(reg, len);
11354         if (err < 0)
11355                 return err;
11356
11357         next_register = DIV_ROUND_UP(len, NFT_REG32_SIZE) + reg;
11358
11359         /* Can't happen: nft_validate_register_load() should have failed */
11360         if (WARN_ON_ONCE(next_register > NFT_REG32_NUM))
11361                 return -EINVAL;
11362
11363         /* find first register that did not see an earlier store. */
11364         invalid_reg = find_next_zero_bit(ctx->reg_inited, NFT_REG32_NUM, reg);
11365
11366         /* invalid register within the range that we're loading from? */
11367         if (invalid_reg < next_register)
11368                 return -ENODATA;
11369
11370         *sreg = reg;
11371         return 0;
11372 }
11373 EXPORT_SYMBOL_GPL(nft_parse_register_load);
11374
11375 static void nft_saw_register_store(const struct nft_ctx *__ctx,
11376                                    int reg, unsigned int len)
11377 {
11378         unsigned int registers = DIV_ROUND_UP(len, NFT_REG32_SIZE);
11379         struct nft_ctx *ctx = (struct nft_ctx *)__ctx;
11380
11381         if (WARN_ON_ONCE(len == 0 || reg < 0))
11382                 return;
11383
11384         bitmap_set(ctx->reg_inited, reg, registers);
11385 }
11386
11387 static int nft_validate_register_store(const struct nft_ctx *ctx,
11388                                        enum nft_registers reg,
11389                                        const struct nft_data *data,
11390                                        enum nft_data_types type,
11391                                        unsigned int len)
11392 {
11393         int err;
11394
11395         switch (reg) {
11396         case NFT_REG_VERDICT:
11397                 if (type != NFT_DATA_VERDICT)
11398                         return -EINVAL;
11399
11400                 if (data != NULL &&
11401                     (data->verdict.code == NFT_GOTO ||
11402                      data->verdict.code == NFT_JUMP)) {
11403                         err = nft_chain_validate(ctx, data->verdict.chain);
11404                         if (err < 0)
11405                                 return err;
11406                 }
11407
11408                 break;
11409         default:
11410                 if (type != NFT_DATA_VALUE)
11411                         return -EINVAL;
11412
11413                 if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
11414                         return -EINVAL;
11415                 if (len == 0)
11416                         return -EINVAL;
11417                 if (reg * NFT_REG32_SIZE + len >
11418                     sizeof_field(struct nft_regs, data))
11419                         return -ERANGE;
11420
11421                 break;
11422         }
11423
11424         nft_saw_register_store(ctx, reg, len);
11425         return 0;
11426 }
11427
11428 int nft_parse_register_store(const struct nft_ctx *ctx,
11429                              const struct nlattr *attr, u8 *dreg,
11430                              const struct nft_data *data,
11431                              enum nft_data_types type, unsigned int len)
11432 {
11433         int err;
11434         u32 reg;
11435
11436         err = nft_parse_register(attr, &reg);
11437         if (err < 0)
11438                 return err;
11439
11440         err = nft_validate_register_store(ctx, reg, data, type, len);
11441         if (err < 0)
11442                 return err;
11443
11444         *dreg = reg;
11445         return 0;
11446 }
11447 EXPORT_SYMBOL_GPL(nft_parse_register_store);
11448
11449 static const struct nla_policy nft_verdict_policy[NFTA_VERDICT_MAX + 1] = {
11450         [NFTA_VERDICT_CODE]     = { .type = NLA_U32 },
11451         [NFTA_VERDICT_CHAIN]    = { .type = NLA_STRING,
11452                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
11453         [NFTA_VERDICT_CHAIN_ID] = { .type = NLA_U32 },
11454 };
11455
11456 static int nft_verdict_init(const struct nft_ctx *ctx, struct nft_data *data,
11457                             struct nft_data_desc *desc, const struct nlattr *nla)
11458 {
11459         u8 genmask = nft_genmask_next(ctx->net);
11460         struct nlattr *tb[NFTA_VERDICT_MAX + 1];
11461         struct nft_chain *chain;
11462         int err;
11463
11464         err = nla_parse_nested_deprecated(tb, NFTA_VERDICT_MAX, nla,
11465                                           nft_verdict_policy, NULL);
11466         if (err < 0)
11467                 return err;
11468
11469         if (!tb[NFTA_VERDICT_CODE])
11470                 return -EINVAL;
11471
11472         /* zero padding hole for memcmp */
11473         memset(data, 0, sizeof(*data));
11474         data->verdict.code = ntohl(nla_get_be32(tb[NFTA_VERDICT_CODE]));
11475
11476         switch (data->verdict.code) {
11477         case NF_ACCEPT:
11478         case NF_DROP:
11479         case NF_QUEUE:
11480                 break;
11481         case NFT_CONTINUE:
11482         case NFT_BREAK:
11483         case NFT_RETURN:
11484                 break;
11485         case NFT_JUMP:
11486         case NFT_GOTO:
11487                 if (tb[NFTA_VERDICT_CHAIN]) {
11488                         chain = nft_chain_lookup(ctx->net, ctx->table,
11489                                                  tb[NFTA_VERDICT_CHAIN],
11490                                                  genmask);
11491                 } else if (tb[NFTA_VERDICT_CHAIN_ID]) {
11492                         chain = nft_chain_lookup_byid(ctx->net, ctx->table,
11493                                                       tb[NFTA_VERDICT_CHAIN_ID],
11494                                                       genmask);
11495                         if (IS_ERR(chain))
11496                                 return PTR_ERR(chain);
11497                 } else {
11498                         return -EINVAL;
11499                 }
11500
11501                 if (IS_ERR(chain))
11502                         return PTR_ERR(chain);
11503                 if (nft_is_base_chain(chain))
11504                         return -EOPNOTSUPP;
11505                 if (nft_chain_is_bound(chain))
11506                         return -EINVAL;
11507                 if (desc->flags & NFT_DATA_DESC_SETELEM &&
11508                     chain->flags & NFT_CHAIN_BINDING)
11509                         return -EINVAL;
11510                 if (!nft_use_inc(&chain->use))
11511                         return -EMFILE;
11512
11513                 data->verdict.chain = chain;
11514                 break;
11515         default:
11516                 return -EINVAL;
11517         }
11518
11519         desc->len = sizeof(data->verdict);
11520
11521         return 0;
11522 }
11523
11524 static void nft_verdict_uninit(const struct nft_data *data)
11525 {
11526         struct nft_chain *chain;
11527
11528         switch (data->verdict.code) {
11529         case NFT_JUMP:
11530         case NFT_GOTO:
11531                 chain = data->verdict.chain;
11532                 nft_use_dec(&chain->use);
11533                 break;
11534         }
11535 }
11536
11537 int nft_verdict_dump(struct sk_buff *skb, int type, const struct nft_verdict *v)
11538 {
11539         struct nlattr *nest;
11540
11541         nest = nla_nest_start_noflag(skb, type);
11542         if (!nest)
11543                 goto nla_put_failure;
11544
11545         if (nla_put_be32(skb, NFTA_VERDICT_CODE, htonl(v->code)))
11546                 goto nla_put_failure;
11547
11548         switch (v->code) {
11549         case NFT_JUMP:
11550         case NFT_GOTO:
11551                 if (nla_put_string(skb, NFTA_VERDICT_CHAIN,
11552                                    v->chain->name))
11553                         goto nla_put_failure;
11554         }
11555         nla_nest_end(skb, nest);
11556         return 0;
11557
11558 nla_put_failure:
11559         return -1;
11560 }
11561
11562 static int nft_value_init(const struct nft_ctx *ctx,
11563                           struct nft_data *data, struct nft_data_desc *desc,
11564                           const struct nlattr *nla)
11565 {
11566         unsigned int len;
11567
11568         len = nla_len(nla);
11569         if (len == 0)
11570                 return -EINVAL;
11571         if (len > desc->size)
11572                 return -EOVERFLOW;
11573         if (desc->len) {
11574                 if (len != desc->len)
11575                         return -EINVAL;
11576         } else {
11577                 desc->len = len;
11578         }
11579
11580         nla_memcpy(data->data, nla, len);
11581
11582         return 0;
11583 }
11584
11585 static int nft_value_dump(struct sk_buff *skb, const struct nft_data *data,
11586                           unsigned int len)
11587 {
11588         return nla_put(skb, NFTA_DATA_VALUE, len, data->data);
11589 }
11590
11591 static const struct nla_policy nft_data_policy[NFTA_DATA_MAX + 1] = {
11592         [NFTA_DATA_VALUE]       = { .type = NLA_BINARY },
11593         [NFTA_DATA_VERDICT]     = { .type = NLA_NESTED },
11594 };
11595
11596 /**
11597  *      nft_data_init - parse nf_tables data netlink attributes
11598  *
11599  *      @ctx: context of the expression using the data
11600  *      @data: destination struct nft_data
11601  *      @desc: data description
11602  *      @nla: netlink attribute containing data
11603  *
11604  *      Parse the netlink data attributes and initialize a struct nft_data.
11605  *      The type and length of data are returned in the data description.
11606  *
11607  *      The caller can indicate that it only wants to accept data of type
11608  *      NFT_DATA_VALUE by passing NULL for the ctx argument.
11609  */
11610 int nft_data_init(const struct nft_ctx *ctx, struct nft_data *data,
11611                   struct nft_data_desc *desc, const struct nlattr *nla)
11612 {
11613         struct nlattr *tb[NFTA_DATA_MAX + 1];
11614         int err;
11615
11616         if (WARN_ON_ONCE(!desc->size))
11617                 return -EINVAL;
11618
11619         err = nla_parse_nested_deprecated(tb, NFTA_DATA_MAX, nla,
11620                                           nft_data_policy, NULL);
11621         if (err < 0)
11622                 return err;
11623
11624         if (tb[NFTA_DATA_VALUE]) {
11625                 if (desc->type != NFT_DATA_VALUE)
11626                         return -EINVAL;
11627
11628                 err = nft_value_init(ctx, data, desc, tb[NFTA_DATA_VALUE]);
11629         } else if (tb[NFTA_DATA_VERDICT] && ctx != NULL) {
11630                 if (desc->type != NFT_DATA_VERDICT)
11631                         return -EINVAL;
11632
11633                 err = nft_verdict_init(ctx, data, desc, tb[NFTA_DATA_VERDICT]);
11634         } else {
11635                 err = -EINVAL;
11636         }
11637
11638         return err;
11639 }
11640 EXPORT_SYMBOL_GPL(nft_data_init);
11641
11642 /**
11643  *      nft_data_release - release a nft_data item
11644  *
11645  *      @data: struct nft_data to release
11646  *      @type: type of data
11647  *
11648  *      Release a nft_data item. NFT_DATA_VALUE types can be silently discarded,
11649  *      all others need to be released by calling this function.
11650  */
11651 void nft_data_release(const struct nft_data *data, enum nft_data_types type)
11652 {
11653         if (type < NFT_DATA_VERDICT)
11654                 return;
11655         switch (type) {
11656         case NFT_DATA_VERDICT:
11657                 return nft_verdict_uninit(data);
11658         default:
11659                 WARN_ON(1);
11660         }
11661 }
11662 EXPORT_SYMBOL_GPL(nft_data_release);
11663
11664 int nft_data_dump(struct sk_buff *skb, int attr, const struct nft_data *data,
11665                   enum nft_data_types type, unsigned int len)
11666 {
11667         struct nlattr *nest;
11668         int err;
11669
11670         nest = nla_nest_start_noflag(skb, attr);
11671         if (nest == NULL)
11672                 return -1;
11673
11674         switch (type) {
11675         case NFT_DATA_VALUE:
11676                 err = nft_value_dump(skb, data, len);
11677                 break;
11678         case NFT_DATA_VERDICT:
11679                 err = nft_verdict_dump(skb, NFTA_DATA_VERDICT, &data->verdict);
11680                 break;
11681         default:
11682                 err = -EINVAL;
11683                 WARN_ON(1);
11684         }
11685
11686         nla_nest_end(skb, nest);
11687         return err;
11688 }
11689 EXPORT_SYMBOL_GPL(nft_data_dump);
11690
11691 static void __nft_release_basechain_now(struct nft_ctx *ctx)
11692 {
11693         struct nft_rule *rule, *nr;
11694
11695         list_for_each_entry_safe(rule, nr, &ctx->chain->rules, list) {
11696                 list_del(&rule->list);
11697                 nf_tables_rule_release(ctx, rule);
11698         }
11699         nf_tables_chain_destroy(ctx->chain);
11700 }
11701
11702 int __nft_release_basechain(struct nft_ctx *ctx)
11703 {
11704         struct nft_rule *rule;
11705
11706         if (WARN_ON_ONCE(!nft_is_base_chain(ctx->chain)))
11707                 return 0;
11708
11709         nf_tables_unregister_hook(ctx->net, ctx->chain->table, ctx->chain);
11710         list_for_each_entry(rule, &ctx->chain->rules, list)
11711                 nft_use_dec(&ctx->chain->use);
11712
11713         nft_chain_del(ctx->chain);
11714         nft_use_dec(&ctx->table->use);
11715
11716         if (!maybe_get_net(ctx->net)) {
11717                 __nft_release_basechain_now(ctx);
11718                 return 0;
11719         }
11720
11721         /* wait for ruleset dumps to complete.  Owning chain is no longer in
11722          * lists, so new dumps can't find any of these rules anymore.
11723          */
11724         synchronize_rcu();
11725
11726         __nft_release_basechain_now(ctx);
11727         put_net(ctx->net);
11728         return 0;
11729 }
11730 EXPORT_SYMBOL_GPL(__nft_release_basechain);
11731
11732 static void __nft_release_hook(struct net *net, struct nft_table *table)
11733 {
11734         struct nft_flowtable *flowtable;
11735         struct nft_chain *chain;
11736
11737         list_for_each_entry(chain, &table->chains, list)
11738                 __nf_tables_unregister_hook(net, table, chain, true);
11739         list_for_each_entry(flowtable, &table->flowtables, list)
11740                 __nft_unregister_flowtable_net_hooks(net, &flowtable->hook_list,
11741                                                      true);
11742 }
11743
11744 static void __nft_release_hooks(struct net *net)
11745 {
11746         struct nftables_pernet *nft_net = nft_pernet(net);
11747         struct nft_table *table;
11748
11749         list_for_each_entry(table, &nft_net->tables, list) {
11750                 if (nft_table_has_owner(table))
11751                         continue;
11752
11753                 __nft_release_hook(net, table);
11754         }
11755 }
11756
11757 static void __nft_release_table(struct net *net, struct nft_table *table)
11758 {
11759         struct nft_flowtable *flowtable, *nf;
11760         struct nft_chain *chain, *nc;
11761         struct nft_object *obj, *ne;
11762         struct nft_rule *rule, *nr;
11763         struct nft_set *set, *ns;
11764         struct nft_ctx ctx = {
11765                 .net    = net,
11766                 .family = NFPROTO_NETDEV,
11767         };
11768
11769         ctx.family = table->family;
11770         ctx.table = table;
11771         list_for_each_entry(chain, &table->chains, list) {
11772                 if (nft_chain_binding(chain))
11773                         continue;
11774
11775                 ctx.chain = chain;
11776                 list_for_each_entry_safe(rule, nr, &chain->rules, list) {
11777                         list_del(&rule->list);
11778                         nft_use_dec(&chain->use);
11779                         nf_tables_rule_release(&ctx, rule);
11780                 }
11781         }
11782         list_for_each_entry_safe(flowtable, nf, &table->flowtables, list) {
11783                 list_del(&flowtable->list);
11784                 nft_use_dec(&table->use);
11785                 nf_tables_flowtable_destroy(flowtable);
11786         }
11787         list_for_each_entry_safe(set, ns, &table->sets, list) {
11788                 list_del(&set->list);
11789                 nft_use_dec(&table->use);
11790                 if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
11791                         nft_map_deactivate(&ctx, set);
11792
11793                 nft_set_destroy(&ctx, set);
11794         }
11795         list_for_each_entry_safe(obj, ne, &table->objects, list) {
11796                 nft_obj_del(obj);
11797                 nft_use_dec(&table->use);
11798                 nft_obj_destroy(&ctx, obj);
11799         }
11800         list_for_each_entry_safe(chain, nc, &table->chains, list) {
11801                 nft_chain_del(chain);
11802                 nft_use_dec(&table->use);
11803                 nf_tables_chain_destroy(chain);
11804         }
11805         nf_tables_table_destroy(table);
11806 }
11807
11808 static void __nft_release_tables(struct net *net)
11809 {
11810         struct nftables_pernet *nft_net = nft_pernet(net);
11811         struct nft_table *table, *nt;
11812
11813         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
11814                 if (nft_table_has_owner(table))
11815                         continue;
11816
11817                 list_del(&table->list);
11818
11819                 __nft_release_table(net, table);
11820         }
11821 }
11822
11823 static int nft_rcv_nl_event(struct notifier_block *this, unsigned long event,
11824                             void *ptr)
11825 {
11826         struct nft_table *table, *to_delete[8];
11827         struct nftables_pernet *nft_net;
11828         struct netlink_notify *n = ptr;
11829         struct net *net = n->net;
11830         unsigned int deleted;
11831         bool restart = false;
11832         unsigned int gc_seq;
11833
11834         if (event != NETLINK_URELEASE || n->protocol != NETLINK_NETFILTER)
11835                 return NOTIFY_DONE;
11836
11837         nft_net = nft_pernet(net);
11838         deleted = 0;
11839         mutex_lock(&nft_net->commit_mutex);
11840
11841         gc_seq = nft_gc_seq_begin(nft_net);
11842
11843         nf_tables_trans_destroy_flush_work();
11844 again:
11845         list_for_each_entry(table, &nft_net->tables, list) {
11846                 if (nft_table_has_owner(table) &&
11847                     n->portid == table->nlpid) {
11848                         if (table->flags & NFT_TABLE_F_PERSIST) {
11849                                 table->flags &= ~NFT_TABLE_F_OWNER;
11850                                 continue;
11851                         }
11852                         __nft_release_hook(net, table);
11853                         list_del_rcu(&table->list);
11854                         to_delete[deleted++] = table;
11855                         if (deleted >= ARRAY_SIZE(to_delete))
11856                                 break;
11857                 }
11858         }
11859         if (deleted) {
11860                 restart = deleted >= ARRAY_SIZE(to_delete);
11861                 synchronize_rcu();
11862                 while (deleted)
11863                         __nft_release_table(net, to_delete[--deleted]);
11864
11865                 if (restart)
11866                         goto again;
11867         }
11868         nft_gc_seq_end(nft_net, gc_seq);
11869
11870         mutex_unlock(&nft_net->commit_mutex);
11871
11872         return NOTIFY_DONE;
11873 }
11874
11875 static struct notifier_block nft_nl_notifier = {
11876         .notifier_call  = nft_rcv_nl_event,
11877 };
11878
11879 static int __net_init nf_tables_init_net(struct net *net)
11880 {
11881         struct nftables_pernet *nft_net = nft_pernet(net);
11882
11883         INIT_LIST_HEAD(&nft_net->tables);
11884         INIT_LIST_HEAD(&nft_net->commit_list);
11885         INIT_LIST_HEAD(&nft_net->commit_set_list);
11886         INIT_LIST_HEAD(&nft_net->binding_list);
11887         INIT_LIST_HEAD(&nft_net->module_list);
11888         INIT_LIST_HEAD(&nft_net->notify_list);
11889         mutex_init(&nft_net->commit_mutex);
11890         nft_net->base_seq = 1;
11891         nft_net->gc_seq = 0;
11892         nft_net->validate_state = NFT_VALIDATE_SKIP;
11893
11894         return 0;
11895 }
11896
11897 static void __net_exit nf_tables_pre_exit_net(struct net *net)
11898 {
11899         struct nftables_pernet *nft_net = nft_pernet(net);
11900
11901         mutex_lock(&nft_net->commit_mutex);
11902         __nft_release_hooks(net);
11903         mutex_unlock(&nft_net->commit_mutex);
11904 }
11905
11906 static void __net_exit nf_tables_exit_net(struct net *net)
11907 {
11908         struct nftables_pernet *nft_net = nft_pernet(net);
11909         unsigned int gc_seq;
11910
11911         mutex_lock(&nft_net->commit_mutex);
11912
11913         gc_seq = nft_gc_seq_begin(nft_net);
11914
11915         WARN_ON_ONCE(!list_empty(&nft_net->commit_list));
11916         WARN_ON_ONCE(!list_empty(&nft_net->commit_set_list));
11917
11918         if (!list_empty(&nft_net->module_list))
11919                 nf_tables_module_autoload_cleanup(net);
11920
11921         __nft_release_tables(net);
11922
11923         nft_gc_seq_end(nft_net, gc_seq);
11924
11925         mutex_unlock(&nft_net->commit_mutex);
11926         WARN_ON_ONCE(!list_empty(&nft_net->tables));
11927         WARN_ON_ONCE(!list_empty(&nft_net->module_list));
11928         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
11929 }
11930
11931 static void nf_tables_exit_batch(struct list_head *net_exit_list)
11932 {
11933         flush_work(&trans_gc_work);
11934 }
11935
11936 static struct pernet_operations nf_tables_net_ops = {
11937         .init           = nf_tables_init_net,
11938         .pre_exit       = nf_tables_pre_exit_net,
11939         .exit           = nf_tables_exit_net,
11940         .exit_batch     = nf_tables_exit_batch,
11941         .id             = &nf_tables_net_id,
11942         .size           = sizeof(struct nftables_pernet),
11943 };
11944
11945 static int __init nf_tables_module_init(void)
11946 {
11947         int err;
11948
11949         BUILD_BUG_ON(offsetof(struct nft_trans_table, nft_trans) != 0);
11950         BUILD_BUG_ON(offsetof(struct nft_trans_chain, nft_trans_binding.nft_trans) != 0);
11951         BUILD_BUG_ON(offsetof(struct nft_trans_rule, nft_trans) != 0);
11952         BUILD_BUG_ON(offsetof(struct nft_trans_set, nft_trans_binding.nft_trans) != 0);
11953         BUILD_BUG_ON(offsetof(struct nft_trans_elem, nft_trans) != 0);
11954         BUILD_BUG_ON(offsetof(struct nft_trans_obj, nft_trans) != 0);
11955         BUILD_BUG_ON(offsetof(struct nft_trans_flowtable, nft_trans) != 0);
11956
11957         err = register_pernet_subsys(&nf_tables_net_ops);
11958         if (err < 0)
11959                 return err;
11960
11961         err = nft_chain_filter_init();
11962         if (err < 0)
11963                 goto err_chain_filter;
11964
11965         err = nf_tables_core_module_init();
11966         if (err < 0)
11967                 goto err_core_module;
11968
11969         err = register_netdevice_notifier(&nf_tables_flowtable_notifier);
11970         if (err < 0)
11971                 goto err_netdev_notifier;
11972
11973         err = rhltable_init(&nft_objname_ht, &nft_objname_ht_params);
11974         if (err < 0)
11975                 goto err_rht_objname;
11976
11977         err = nft_offload_init();
11978         if (err < 0)
11979                 goto err_offload;
11980
11981         err = netlink_register_notifier(&nft_nl_notifier);
11982         if (err < 0)
11983                 goto err_netlink_notifier;
11984
11985         /* must be last */
11986         err = nfnetlink_subsys_register(&nf_tables_subsys);
11987         if (err < 0)
11988                 goto err_nfnl_subsys;
11989
11990         nft_chain_route_init();
11991
11992         return err;
11993
11994 err_nfnl_subsys:
11995         netlink_unregister_notifier(&nft_nl_notifier);
11996 err_netlink_notifier:
11997         nft_offload_exit();
11998 err_offload:
11999         rhltable_destroy(&nft_objname_ht);
12000 err_rht_objname:
12001         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
12002 err_netdev_notifier:
12003         nf_tables_core_module_exit();
12004 err_core_module:
12005         nft_chain_filter_fini();
12006 err_chain_filter:
12007         unregister_pernet_subsys(&nf_tables_net_ops);
12008         return err;
12009 }
12010
12011 static void __exit nf_tables_module_exit(void)
12012 {
12013         nfnetlink_subsys_unregister(&nf_tables_subsys);
12014         netlink_unregister_notifier(&nft_nl_notifier);
12015         nft_offload_exit();
12016         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
12017         nft_chain_filter_fini();
12018         nft_chain_route_fini();
12019         nf_tables_trans_destroy_flush_work();
12020         unregister_pernet_subsys(&nf_tables_net_ops);
12021         cancel_work_sync(&trans_gc_work);
12022         cancel_work_sync(&trans_destroy_work);
12023         rcu_barrier();
12024         rhltable_destroy(&nft_objname_ht);
12025         nf_tables_core_module_exit();
12026 }
12027
12028 module_init(nf_tables_module_init);
12029 module_exit(nf_tables_module_exit);
12030
12031 MODULE_LICENSE("GPL");
12032 MODULE_AUTHOR("Patrick McHardy <[email protected]>");
12033 MODULE_DESCRIPTION("Framework for packet filtering and classification");
12034 MODULE_ALIAS_NFNL_SUBSYS(NFNL_SUBSYS_NFTABLES);