net/netfilter/nf_tables_api.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2007-2009 Patrick McHardy <[email protected]>
   4  *
   5  * Development of this code funded by Astaro AG (http://www.astaro.com/)
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/init.h>
  10 #include <linux/list.h>
  11 #include <linux/skbuff.h>
  12 #include <linux/netlink.h>
  13 #include <linux/vmalloc.h>
  14 #include <linux/rhashtable.h>
  15 #include <linux/audit.h>
  16 #include <linux/netfilter.h>
  17 #include <linux/netfilter/nfnetlink.h>
  18 #include <linux/netfilter/nf_tables.h>
  19 #include <net/netfilter/nf_flow_table.h>
  20 #include <net/netfilter/nf_tables_core.h>
  21 #include <net/netfilter/nf_tables.h>
  22 #include <net/netfilter/nf_tables_offload.h>
  23 #include <net/net_namespace.h>
  24 #include <net/sock.h>
  25
  26 #define NFT_MODULE_AUTOLOAD_LIMIT (MODULE_NAME_LEN - sizeof("nft-expr-255-"))
  27 #define NFT_SET_MAX_ANONLEN 16
  28
  29 unsigned int nf_tables_net_id __read_mostly;
  30
  31 static LIST_HEAD(nf_tables_expressions);
  32 static LIST_HEAD(nf_tables_objects);
  33 static LIST_HEAD(nf_tables_flowtables);
  34 static LIST_HEAD(nf_tables_destroy_list);
  35 static LIST_HEAD(nf_tables_gc_list);
  36 static DEFINE_SPINLOCK(nf_tables_destroy_list_lock);
  37 static DEFINE_SPINLOCK(nf_tables_gc_list_lock);
  38
  39 enum {
  40         NFT_VALIDATE_SKIP       = 0,
  41         NFT_VALIDATE_NEED,
  42         NFT_VALIDATE_DO,
  43 };
  44
  45 static struct rhltable nft_objname_ht;
  46
  47 static u32 nft_chain_hash(const void *data, u32 len, u32 seed);
  48 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed);
  49 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *, const void *);
  50
  51 static u32 nft_objname_hash(const void *data, u32 len, u32 seed);
  52 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed);
  53 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *, const void *);
  54
  55 static const struct rhashtable_params nft_chain_ht_params = {
  56         .head_offset            = offsetof(struct nft_chain, rhlhead),
  57         .key_offset             = offsetof(struct nft_chain, name),
  58         .hashfn                 = nft_chain_hash,
  59         .obj_hashfn             = nft_chain_hash_obj,
  60         .obj_cmpfn              = nft_chain_hash_cmp,
  61         .automatic_shrinking    = true,
  62 };
  63
  64 static const struct rhashtable_params nft_objname_ht_params = {
  65         .head_offset            = offsetof(struct nft_object, rhlhead),
  66         .key_offset             = offsetof(struct nft_object, key),
  67         .hashfn                 = nft_objname_hash,
  68         .obj_hashfn             = nft_objname_hash_obj,
  69         .obj_cmpfn              = nft_objname_hash_cmp,
  70         .automatic_shrinking    = true,
  71 };
  72
  73 struct nft_audit_data {
  74         struct nft_table *table;
  75         int entries;
  76         int op;
  77         struct list_head list;
  78 };
  79
  80 static const u8 nft2audit_op[NFT_MSG_MAX] = { // enum nf_tables_msg_types
  81         [NFT_MSG_NEWTABLE]      = AUDIT_NFT_OP_TABLE_REGISTER,
  82         [NFT_MSG_GETTABLE]      = AUDIT_NFT_OP_INVALID,
  83         [NFT_MSG_DELTABLE]      = AUDIT_NFT_OP_TABLE_UNREGISTER,
  84         [NFT_MSG_NEWCHAIN]      = AUDIT_NFT_OP_CHAIN_REGISTER,
  85         [NFT_MSG_GETCHAIN]      = AUDIT_NFT_OP_INVALID,
  86         [NFT_MSG_DELCHAIN]      = AUDIT_NFT_OP_CHAIN_UNREGISTER,
  87         [NFT_MSG_NEWRULE]       = AUDIT_NFT_OP_RULE_REGISTER,
  88         [NFT_MSG_GETRULE]       = AUDIT_NFT_OP_INVALID,
  89         [NFT_MSG_DELRULE]       = AUDIT_NFT_OP_RULE_UNREGISTER,
  90         [NFT_MSG_NEWSET]        = AUDIT_NFT_OP_SET_REGISTER,
  91         [NFT_MSG_GETSET]        = AUDIT_NFT_OP_INVALID,
  92         [NFT_MSG_DELSET]        = AUDIT_NFT_OP_SET_UNREGISTER,
  93         [NFT_MSG_NEWSETELEM]    = AUDIT_NFT_OP_SETELEM_REGISTER,
  94         [NFT_MSG_GETSETELEM]    = AUDIT_NFT_OP_INVALID,
  95         [NFT_MSG_DELSETELEM]    = AUDIT_NFT_OP_SETELEM_UNREGISTER,
  96         [NFT_MSG_NEWGEN]        = AUDIT_NFT_OP_GEN_REGISTER,
  97         [NFT_MSG_GETGEN]        = AUDIT_NFT_OP_INVALID,
  98         [NFT_MSG_TRACE]         = AUDIT_NFT_OP_INVALID,
  99         [NFT_MSG_NEWOBJ]        = AUDIT_NFT_OP_OBJ_REGISTER,
 100         [NFT_MSG_GETOBJ]        = AUDIT_NFT_OP_INVALID,
 101         [NFT_MSG_DELOBJ]        = AUDIT_NFT_OP_OBJ_UNREGISTER,
 102         [NFT_MSG_GETOBJ_RESET]  = AUDIT_NFT_OP_OBJ_RESET,
 103         [NFT_MSG_NEWFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_REGISTER,
 104         [NFT_MSG_GETFLOWTABLE]  = AUDIT_NFT_OP_INVALID,
 105         [NFT_MSG_DELFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_UNREGISTER,
 106         [NFT_MSG_GETSETELEM_RESET] = AUDIT_NFT_OP_SETELEM_RESET,
 107 };
 108
 109 static void nft_validate_state_update(struct nft_table *table, u8 new_validate_state)
 110 {
 111         switch (table->validate_state) {
 112         case NFT_VALIDATE_SKIP:
 113                 WARN_ON_ONCE(new_validate_state == NFT_VALIDATE_DO);
 114                 break;
 115         case NFT_VALIDATE_NEED:
 116                 break;
 117         case NFT_VALIDATE_DO:
 118                 if (new_validate_state == NFT_VALIDATE_NEED)
 119                         return;
 120         }
 121
 122         table->validate_state = new_validate_state;
 123 }
 124 static void nf_tables_trans_destroy_work(struct work_struct *w);
 125 static DECLARE_WORK(trans_destroy_work, nf_tables_trans_destroy_work);
 126
 127 static void nft_trans_gc_work(struct work_struct *work);
 128 static DECLARE_WORK(trans_gc_work, nft_trans_gc_work);
 129
 130 static void nft_ctx_init(struct nft_ctx *ctx,
 131                          struct net *net,
 132                          const struct sk_buff *skb,
 133                          const struct nlmsghdr *nlh,
 134                          u8 family,
 135                          struct nft_table *table,
 136                          struct nft_chain *chain,
 137                          const struct nlattr * const *nla)
 138 {
 139         ctx->net        = net;
 140         ctx->family     = family;
 141         ctx->level      = 0;
 142         ctx->table      = table;
 143         ctx->chain      = chain;
 144         ctx->nla        = nla;
 145         ctx->portid     = NETLINK_CB(skb).portid;
 146         ctx->report     = nlmsg_report(nlh);
 147         ctx->flags      = nlh->nlmsg_flags;
 148         ctx->seq        = nlh->nlmsg_seq;
 149 }
 150
 151 static struct nft_trans *nft_trans_alloc_gfp(const struct nft_ctx *ctx,
 152                                              int msg_type, u32 size, gfp_t gfp)
 153 {
 154         struct nft_trans *trans;
 155
 156         trans = kzalloc(sizeof(struct nft_trans) + size, gfp);
 157         if (trans == NULL)
 158                 return NULL;
 159
 160         INIT_LIST_HEAD(&trans->list);
 161         INIT_LIST_HEAD(&trans->binding_list);
 162         trans->msg_type = msg_type;
 163         trans->ctx      = *ctx;
 164
 165         return trans;
 166 }
 167
 168 static struct nft_trans *nft_trans_alloc(const struct nft_ctx *ctx,
 169                                          int msg_type, u32 size)
 170 {
 171         return nft_trans_alloc_gfp(ctx, msg_type, size, GFP_KERNEL);
 172 }
 173
 174 static void nft_trans_list_del(struct nft_trans *trans)
 175 {
 176         list_del(&trans->list);
 177         list_del(&trans->binding_list);
 178 }
 179
 180 static void nft_trans_destroy(struct nft_trans *trans)
 181 {
 182         nft_trans_list_del(trans);
 183         kfree(trans);
 184 }
 185
 186 static void __nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set,
 187                                  bool bind)
 188 {
 189         struct nftables_pernet *nft_net;
 190         struct net *net = ctx->net;
 191         struct nft_trans *trans;
 192
 193         if (!nft_set_is_anonymous(set))
 194                 return;
 195
 196         nft_net = nft_pernet(net);
 197         list_for_each_entry_reverse(trans, &nft_net->commit_list, list) {
 198                 switch (trans->msg_type) {
 199                 case NFT_MSG_NEWSET:
 200                         if (nft_trans_set(trans) == set)
 201                                 nft_trans_set_bound(trans) = bind;
 202                         break;
 203                 case NFT_MSG_NEWSETELEM:
 204                         if (nft_trans_elem_set(trans) == set)
 205                                 nft_trans_elem_set_bound(trans) = bind;
 206                         break;
 207                 }
 208         }
 209 }
 210
 211 static void nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set)
 212 {
 213         return __nft_set_trans_bind(ctx, set, true);
 214 }
 215
 216 static void nft_set_trans_unbind(const struct nft_ctx *ctx, struct nft_set *set)
 217 {
 218         return __nft_set_trans_bind(ctx, set, false);
 219 }
 220
 221 static void __nft_chain_trans_bind(const struct nft_ctx *ctx,
 222                                    struct nft_chain *chain, bool bind)
 223 {
 224         struct nftables_pernet *nft_net;
 225         struct net *net = ctx->net;
 226         struct nft_trans *trans;
 227
 228         if (!nft_chain_binding(chain))
 229                 return;
 230
 231         nft_net = nft_pernet(net);
 232         list_for_each_entry_reverse(trans, &nft_net->commit_list, list) {
 233                 switch (trans->msg_type) {
 234                 case NFT_MSG_NEWCHAIN:
 235                         if (nft_trans_chain(trans) == chain)
 236                                 nft_trans_chain_bound(trans) = bind;
 237                         break;
 238                 case NFT_MSG_NEWRULE:
 239                         if (trans->ctx.chain == chain)
 240                                 nft_trans_rule_bound(trans) = bind;
 241                         break;
 242                 }
 243         }
 244 }
 245
 246 static void nft_chain_trans_bind(const struct nft_ctx *ctx,
 247                                  struct nft_chain *chain)
 248 {
 249         __nft_chain_trans_bind(ctx, chain, true);
 250 }
 251
 252 int nf_tables_bind_chain(const struct nft_ctx *ctx, struct nft_chain *chain)
 253 {
 254         if (!nft_chain_binding(chain))
 255                 return 0;
 256
 257         if (nft_chain_binding(ctx->chain))
 258                 return -EOPNOTSUPP;
 259
 260         if (chain->bound)
 261                 return -EBUSY;
 262
 263         if (!nft_use_inc(&chain->use))
 264                 return -EMFILE;
 265
 266         chain->bound = true;
 267         nft_chain_trans_bind(ctx, chain);
 268
 269         return 0;
 270 }
 271
 272 void nf_tables_unbind_chain(const struct nft_ctx *ctx, struct nft_chain *chain)
 273 {
 274         __nft_chain_trans_bind(ctx, chain, false);
 275 }
 276
 277 static int nft_netdev_register_hooks(struct net *net,
 278                                      struct list_head *hook_list)
 279 {
 280         struct nft_hook *hook;
 281         int err, j;
 282
 283         j = 0;
 284         list_for_each_entry(hook, hook_list, list) {
 285                 err = nf_register_net_hook(net, &hook->ops);
 286                 if (err < 0)
 287                         goto err_register;
 288
 289                 j++;
 290         }
 291         return 0;
 292
 293 err_register:
 294         list_for_each_entry(hook, hook_list, list) {
 295                 if (j-- <= 0)
 296                         break;
 297
 298                 nf_unregister_net_hook(net, &hook->ops);
 299         }
 300         return err;
 301 }
 302
 303 static void nft_netdev_unregister_hooks(struct net *net,
 304                                         struct list_head *hook_list,
 305                                         bool release_netdev)
 306 {
 307         struct nft_hook *hook, *next;
 308
 309         list_for_each_entry_safe(hook, next, hook_list, list) {
 310                 nf_unregister_net_hook(net, &hook->ops);
 311                 if (release_netdev) {
 312                         list_del(&hook->list);
 313                         kfree_rcu(hook, rcu);
 314                 }
 315         }
 316 }
 317
 318 static int nf_tables_register_hook(struct net *net,
 319                                    const struct nft_table *table,
 320                                    struct nft_chain *chain)
 321 {
 322         struct nft_base_chain *basechain;
 323         const struct nf_hook_ops *ops;
 324
 325         if (table->flags & NFT_TABLE_F_DORMANT ||
 326             !nft_is_base_chain(chain))
 327                 return 0;
 328
 329         basechain = nft_base_chain(chain);
 330         ops = &basechain->ops;
 331
 332         if (basechain->type->ops_register)
 333                 return basechain->type->ops_register(net, ops);
 334
 335         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 336                 return nft_netdev_register_hooks(net, &basechain->hook_list);
 337
 338         return nf_register_net_hook(net, &basechain->ops);
 339 }
 340
 341 static void __nf_tables_unregister_hook(struct net *net,
 342                                         const struct nft_table *table,
 343                                         struct nft_chain *chain,
 344                                         bool release_netdev)
 345 {
 346         struct nft_base_chain *basechain;
 347         const struct nf_hook_ops *ops;
 348
 349         if (table->flags & NFT_TABLE_F_DORMANT ||
 350             !nft_is_base_chain(chain))
 351                 return;
 352         basechain = nft_base_chain(chain);
 353         ops = &basechain->ops;
 354
 355         if (basechain->type->ops_unregister)
 356                 return basechain->type->ops_unregister(net, ops);
 357
 358         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 359                 nft_netdev_unregister_hooks(net, &basechain->hook_list,
 360                                             release_netdev);
 361         else
 362                 nf_unregister_net_hook(net, &basechain->ops);
 363 }
 364
 365 static void nf_tables_unregister_hook(struct net *net,
 366                                       const struct nft_table *table,
 367                                       struct nft_chain *chain)
 368 {
 369         return __nf_tables_unregister_hook(net, table, chain, false);
 370 }
 371
 372 static void nft_trans_commit_list_add_tail(struct net *net, struct nft_trans *trans)
 373 {
 374         struct nftables_pernet *nft_net = nft_pernet(net);
 375
 376         switch (trans->msg_type) {
 377         case NFT_MSG_NEWSET:
 378                 if (!nft_trans_set_update(trans) &&
 379                     nft_set_is_anonymous(nft_trans_set(trans)))
 380                         list_add_tail(&trans->binding_list, &nft_net->binding_list);
 381                 break;
 382         case NFT_MSG_NEWCHAIN:
 383                 if (!nft_trans_chain_update(trans) &&
 384                     nft_chain_binding(nft_trans_chain(trans)))
 385                         list_add_tail(&trans->binding_list, &nft_net->binding_list);
 386                 break;
 387         }
 388
 389         list_add_tail(&trans->list, &nft_net->commit_list);
 390 }
 391
 392 static int nft_trans_table_add(struct nft_ctx *ctx, int msg_type)
 393 {
 394         struct nft_trans *trans;
 395
 396         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_table));
 397         if (trans == NULL)
 398                 return -ENOMEM;
 399
 400         if (msg_type == NFT_MSG_NEWTABLE)
 401                 nft_activate_next(ctx->net, ctx->table);
 402
 403         nft_trans_commit_list_add_tail(ctx->net, trans);
 404         return 0;
 405 }
 406
 407 static int nft_deltable(struct nft_ctx *ctx)
 408 {
 409         int err;
 410
 411         err = nft_trans_table_add(ctx, NFT_MSG_DELTABLE);
 412         if (err < 0)
 413                 return err;
 414
 415         nft_deactivate_next(ctx->net, ctx->table);
 416         return err;
 417 }
 418
 419 static struct nft_trans *nft_trans_chain_add(struct nft_ctx *ctx, int msg_type)
 420 {
 421         struct nft_trans *trans;
 422
 423         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_chain));
 424         if (trans == NULL)
 425                 return ERR_PTR(-ENOMEM);
 426
 427         if (msg_type == NFT_MSG_NEWCHAIN) {
 428                 nft_activate_next(ctx->net, ctx->chain);
 429
 430                 if (ctx->nla[NFTA_CHAIN_ID]) {
 431                         nft_trans_chain_id(trans) =
 432                                 ntohl(nla_get_be32(ctx->nla[NFTA_CHAIN_ID]));
 433                 }
 434         }
 435         nft_trans_chain(trans) = ctx->chain;
 436         nft_trans_commit_list_add_tail(ctx->net, trans);
 437
 438         return trans;
 439 }
 440
 441 static int nft_delchain(struct nft_ctx *ctx)
 442 {
 443         struct nft_trans *trans;
 444
 445         trans = nft_trans_chain_add(ctx, NFT_MSG_DELCHAIN);
 446         if (IS_ERR(trans))
 447                 return PTR_ERR(trans);
 448
 449         nft_use_dec(&ctx->table->use);
 450         nft_deactivate_next(ctx->net, ctx->chain);
 451
 452         return 0;
 453 }
 454
 455 void nft_rule_expr_activate(const struct nft_ctx *ctx, struct nft_rule *rule)
 456 {
 457         struct nft_expr *expr;
 458
 459         expr = nft_expr_first(rule);
 460         while (nft_expr_more(rule, expr)) {
 461                 if (expr->ops->activate)
 462                         expr->ops->activate(ctx, expr);
 463
 464                 expr = nft_expr_next(expr);
 465         }
 466 }
 467
 468 void nft_rule_expr_deactivate(const struct nft_ctx *ctx, struct nft_rule *rule,
 469                               enum nft_trans_phase phase)
 470 {
 471         struct nft_expr *expr;
 472
 473         expr = nft_expr_first(rule);
 474         while (nft_expr_more(rule, expr)) {
 475                 if (expr->ops->deactivate)
 476                         expr->ops->deactivate(ctx, expr, phase);
 477
 478                 expr = nft_expr_next(expr);
 479         }
 480 }
 481
 482 static int
 483 nf_tables_delrule_deactivate(struct nft_ctx *ctx, struct nft_rule *rule)
 484 {
 485         /* You cannot delete the same rule twice */
 486         if (nft_is_active_next(ctx->net, rule)) {
 487                 nft_deactivate_next(ctx->net, rule);
 488                 nft_use_dec(&ctx->chain->use);
 489                 return 0;
 490         }
 491         return -ENOENT;
 492 }
 493
 494 static struct nft_trans *nft_trans_rule_add(struct nft_ctx *ctx, int msg_type,
 495                                             struct nft_rule *rule)
 496 {
 497         struct nft_trans *trans;
 498
 499         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_rule));
 500         if (trans == NULL)
 501                 return NULL;
 502
 503         if (msg_type == NFT_MSG_NEWRULE && ctx->nla[NFTA_RULE_ID] != NULL) {
 504                 nft_trans_rule_id(trans) =
 505                         ntohl(nla_get_be32(ctx->nla[NFTA_RULE_ID]));
 506         }
 507         nft_trans_rule(trans) = rule;
 508         nft_trans_commit_list_add_tail(ctx->net, trans);
 509
 510         return trans;
 511 }
 512
 513 static int nft_delrule(struct nft_ctx *ctx, struct nft_rule *rule)
 514 {
 515         struct nft_flow_rule *flow;
 516         struct nft_trans *trans;
 517         int err;
 518
 519         trans = nft_trans_rule_add(ctx, NFT_MSG_DELRULE, rule);
 520         if (trans == NULL)
 521                 return -ENOMEM;
 522
 523         if (ctx->chain->flags & NFT_CHAIN_HW_OFFLOAD) {
 524                 flow = nft_flow_rule_create(ctx->net, rule);
 525                 if (IS_ERR(flow)) {
 526                         nft_trans_destroy(trans);
 527                         return PTR_ERR(flow);
 528                 }
 529
 530                 nft_trans_flow_rule(trans) = flow;
 531         }
 532
 533         err = nf_tables_delrule_deactivate(ctx, rule);
 534         if (err < 0) {
 535                 nft_trans_destroy(trans);
 536                 return err;
 537         }
 538         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_PREPARE);
 539
 540         return 0;
 541 }
 542
 543 static int nft_delrule_by_chain(struct nft_ctx *ctx)
 544 {
 545         struct nft_rule *rule;
 546         int err;
 547
 548         list_for_each_entry(rule, &ctx->chain->rules, list) {
 549                 if (!nft_is_active_next(ctx->net, rule))
 550                         continue;
 551
 552                 err = nft_delrule(ctx, rule);
 553                 if (err < 0)
 554                         return err;
 555         }
 556         return 0;
 557 }
 558
 559 static int __nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
 560                                struct nft_set *set,
 561                                const struct nft_set_desc *desc)
 562 {
 563         struct nft_trans *trans;
 564
 565         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_set));
 566         if (trans == NULL)
 567                 return -ENOMEM;
 568
 569         if (msg_type == NFT_MSG_NEWSET && ctx->nla[NFTA_SET_ID] && !desc) {
 570                 nft_trans_set_id(trans) =
 571                         ntohl(nla_get_be32(ctx->nla[NFTA_SET_ID]));
 572                 nft_activate_next(ctx->net, set);
 573         }
 574         nft_trans_set(trans) = set;
 575         if (desc) {
 576                 nft_trans_set_update(trans) = true;
 577                 nft_trans_set_gc_int(trans) = desc->gc_int;
 578                 nft_trans_set_timeout(trans) = desc->timeout;
 579                 nft_trans_set_size(trans) = desc->size;
 580         }
 581         nft_trans_commit_list_add_tail(ctx->net, trans);
 582
 583         return 0;
 584 }
 585
 586 static int nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
 587                              struct nft_set *set)
 588 {
 589         return __nft_trans_set_add(ctx, msg_type, set, NULL);
 590 }
 591
 592 static int nft_mapelem_deactivate(const struct nft_ctx *ctx,
 593                                   struct nft_set *set,
 594                                   const struct nft_set_iter *iter,
 595                                   struct nft_elem_priv *elem_priv)
 596 {
 597         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
 598
 599         if (!nft_set_elem_active(ext, iter->genmask))
 600                 return 0;
 601
 602         nft_set_elem_change_active(ctx->net, set, ext);
 603         nft_setelem_data_deactivate(ctx->net, set, elem_priv);
 604
 605         return 0;
 606 }
 607
 608 struct nft_set_elem_catchall {
 609         struct list_head        list;
 610         struct rcu_head         rcu;
 611         struct nft_elem_priv    *elem;
 612 };
 613
 614 static void nft_map_catchall_deactivate(const struct nft_ctx *ctx,
 615                                         struct nft_set *set)
 616 {
 617         u8 genmask = nft_genmask_next(ctx->net);
 618         struct nft_set_elem_catchall *catchall;
 619         struct nft_set_ext *ext;
 620
 621         list_for_each_entry(catchall, &set->catchall_list, list) {
 622                 ext = nft_set_elem_ext(set, catchall->elem);
 623                 if (!nft_set_elem_active(ext, genmask))
 624                         continue;
 625
 626                 nft_set_elem_change_active(ctx->net, set, ext);
 627                 nft_setelem_data_deactivate(ctx->net, set, catchall->elem);
 628                 break;
 629         }
 630 }
 631
 632 static void nft_map_deactivate(const struct nft_ctx *ctx, struct nft_set *set)
 633 {
 634         struct nft_set_iter iter = {
 635                 .genmask        = nft_genmask_next(ctx->net),
 636                 .type           = NFT_ITER_UPDATE,
 637                 .fn             = nft_mapelem_deactivate,
 638         };
 639
 640         set->ops->walk(ctx, set, &iter);
 641         WARN_ON_ONCE(iter.err);
 642
 643         nft_map_catchall_deactivate(ctx, set);
 644 }
 645
 646 static int nft_delset(const struct nft_ctx *ctx, struct nft_set *set)
 647 {
 648         int err;
 649
 650         err = nft_trans_set_add(ctx, NFT_MSG_DELSET, set);
 651         if (err < 0)
 652                 return err;
 653
 654         if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
 655                 nft_map_deactivate(ctx, set);
 656
 657         nft_deactivate_next(ctx->net, set);
 658         nft_use_dec(&ctx->table->use);
 659
 660         return err;
 661 }
 662
 663 static int nft_trans_obj_add(struct nft_ctx *ctx, int msg_type,
 664                              struct nft_object *obj)
 665 {
 666         struct nft_trans *trans;
 667
 668         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_obj));
 669         if (trans == NULL)
 670                 return -ENOMEM;
 671
 672         if (msg_type == NFT_MSG_NEWOBJ)
 673                 nft_activate_next(ctx->net, obj);
 674
 675         nft_trans_obj(trans) = obj;
 676         nft_trans_commit_list_add_tail(ctx->net, trans);
 677
 678         return 0;
 679 }
 680
 681 static int nft_delobj(struct nft_ctx *ctx, struct nft_object *obj)
 682 {
 683         int err;
 684
 685         err = nft_trans_obj_add(ctx, NFT_MSG_DELOBJ, obj);
 686         if (err < 0)
 687                 return err;
 688
 689         nft_deactivate_next(ctx->net, obj);
 690         nft_use_dec(&ctx->table->use);
 691
 692         return err;
 693 }
 694
 695 static struct nft_trans *
 696 nft_trans_flowtable_add(struct nft_ctx *ctx, int msg_type,
 697                         struct nft_flowtable *flowtable)
 698 {
 699         struct nft_trans *trans;
 700
 701         trans = nft_trans_alloc(ctx, msg_type,
 702                                 sizeof(struct nft_trans_flowtable));
 703         if (trans == NULL)
 704                 return ERR_PTR(-ENOMEM);
 705
 706         if (msg_type == NFT_MSG_NEWFLOWTABLE)
 707                 nft_activate_next(ctx->net, flowtable);
 708
 709         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
 710         nft_trans_flowtable(trans) = flowtable;
 711         nft_trans_commit_list_add_tail(ctx->net, trans);
 712
 713         return trans;
 714 }
 715
 716 static int nft_delflowtable(struct nft_ctx *ctx,
 717                             struct nft_flowtable *flowtable)
 718 {
 719         struct nft_trans *trans;
 720
 721         trans = nft_trans_flowtable_add(ctx, NFT_MSG_DELFLOWTABLE, flowtable);
 722         if (IS_ERR(trans))
 723                 return PTR_ERR(trans);
 724
 725         nft_deactivate_next(ctx->net, flowtable);
 726         nft_use_dec(&ctx->table->use);
 727
 728         return 0;
 729 }
 730
 731 static void __nft_reg_track_clobber(struct nft_regs_track *track, u8 dreg)
 732 {
 733         int i;
 734
 735         for (i = track->regs[dreg].num_reg; i > 0; i--)
 736                 __nft_reg_track_cancel(track, dreg - i);
 737 }
 738
 739 static void __nft_reg_track_update(struct nft_regs_track *track,
 740                                    const struct nft_expr *expr,
 741                                    u8 dreg, u8 num_reg)
 742 {
 743         track->regs[dreg].selector = expr;
 744         track->regs[dreg].bitwise = NULL;
 745         track->regs[dreg].num_reg = num_reg;
 746 }
 747
 748 void nft_reg_track_update(struct nft_regs_track *track,
 749                           const struct nft_expr *expr, u8 dreg, u8 len)
 750 {
 751         unsigned int regcount;
 752         int i;
 753
 754         __nft_reg_track_clobber(track, dreg);
 755
 756         regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
 757         for (i = 0; i < regcount; i++, dreg++)
 758                 __nft_reg_track_update(track, expr, dreg, i);
 759 }
 760 EXPORT_SYMBOL_GPL(nft_reg_track_update);
 761
 762 void nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg, u8 len)
 763 {
 764         unsigned int regcount;
 765         int i;
 766
 767         __nft_reg_track_clobber(track, dreg);
 768
 769         regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
 770         for (i = 0; i < regcount; i++, dreg++)
 771                 __nft_reg_track_cancel(track, dreg);
 772 }
 773 EXPORT_SYMBOL_GPL(nft_reg_track_cancel);
 774
 775 void __nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg)
 776 {
 777         track->regs[dreg].selector = NULL;
 778         track->regs[dreg].bitwise = NULL;
 779         track->regs[dreg].num_reg = 0;
 780 }
 781 EXPORT_SYMBOL_GPL(__nft_reg_track_cancel);
 782
 783 /*
 784  * Tables
 785  */
 786
 787 static struct nft_table *nft_table_lookup(const struct net *net,
 788                                           const struct nlattr *nla,
 789                                           u8 family, u8 genmask, u32 nlpid)
 790 {
 791         struct nftables_pernet *nft_net;
 792         struct nft_table *table;
 793
 794         if (nla == NULL)
 795                 return ERR_PTR(-EINVAL);
 796
 797         nft_net = nft_pernet(net);
 798         list_for_each_entry_rcu(table, &nft_net->tables, list,
 799                                 lockdep_is_held(&nft_net->commit_mutex)) {
 800                 if (!nla_strcmp(nla, table->name) &&
 801                     table->family == family &&
 802                     nft_active_genmask(table, genmask)) {
 803                         if (nft_table_has_owner(table) &&
 804                             nlpid && table->nlpid != nlpid)
 805                                 return ERR_PTR(-EPERM);
 806
 807                         return table;
 808                 }
 809         }
 810
 811         return ERR_PTR(-ENOENT);
 812 }
 813
 814 static struct nft_table *nft_table_lookup_byhandle(const struct net *net,
 815                                                    const struct nlattr *nla,
 816                                                    int family, u8 genmask, u32 nlpid)
 817 {
 818         struct nftables_pernet *nft_net;
 819         struct nft_table *table;
 820
 821         nft_net = nft_pernet(net);
 822         list_for_each_entry(table, &nft_net->tables, list) {
 823                 if (be64_to_cpu(nla_get_be64(nla)) == table->handle &&
 824                     table->family == family &&
 825                     nft_active_genmask(table, genmask)) {
 826                         if (nft_table_has_owner(table) &&
 827                             nlpid && table->nlpid != nlpid)
 828                                 return ERR_PTR(-EPERM);
 829
 830                         return table;
 831                 }
 832         }
 833
 834         return ERR_PTR(-ENOENT);
 835 }
 836
 837 static inline u64 nf_tables_alloc_handle(struct nft_table *table)
 838 {
 839         return ++table->hgenerator;
 840 }
 841
 842 static const struct nft_chain_type *chain_type[NFPROTO_NUMPROTO][NFT_CHAIN_T_MAX];
 843
 844 static const struct nft_chain_type *
 845 __nft_chain_type_get(u8 family, enum nft_chain_types type)
 846 {
 847         if (family >= NFPROTO_NUMPROTO ||
 848             type >= NFT_CHAIN_T_MAX)
 849                 return NULL;
 850
 851         return chain_type[family][type];
 852 }
 853
 854 static const struct nft_chain_type *
 855 __nf_tables_chain_type_lookup(const struct nlattr *nla, u8 family)
 856 {
 857         const struct nft_chain_type *type;
 858         int i;
 859
 860         for (i = 0; i < NFT_CHAIN_T_MAX; i++) {
 861                 type = __nft_chain_type_get(family, i);
 862                 if (!type)
 863                         continue;
 864                 if (!nla_strcmp(nla, type->name))
 865                         return type;
 866         }
 867         return NULL;
 868 }
 869
 870 struct nft_module_request {
 871         struct list_head        list;
 872         char                    module[MODULE_NAME_LEN];
 873         bool                    done;
 874 };
 875
 876 #ifdef CONFIG_MODULES
 877 __printf(2, 3) int nft_request_module(struct net *net, const char *fmt,
 878                                       ...)
 879 {
 880         char module_name[MODULE_NAME_LEN];
 881         struct nftables_pernet *nft_net;
 882         struct nft_module_request *req;
 883         va_list args;
 884         int ret;
 885
 886         va_start(args, fmt);
 887         ret = vsnprintf(module_name, MODULE_NAME_LEN, fmt, args);
 888         va_end(args);
 889         if (ret >= MODULE_NAME_LEN)
 890                 return 0;
 891
 892         nft_net = nft_pernet(net);
 893         list_for_each_entry(req, &nft_net->module_list, list) {
 894                 if (!strcmp(req->module, module_name)) {
 895                         if (req->done)
 896                                 return 0;
 897
 898                         /* A request to load this module already exists. */
 899                         return -EAGAIN;
 900                 }
 901         }
 902
 903         req = kmalloc(sizeof(*req), GFP_KERNEL);
 904         if (!req)
 905                 return -ENOMEM;
 906
 907         req->done = false;
 908         strscpy(req->module, module_name, MODULE_NAME_LEN);
 909         list_add_tail(&req->list, &nft_net->module_list);
 910
 911         return -EAGAIN;
 912 }
 913 EXPORT_SYMBOL_GPL(nft_request_module);
 914 #endif
 915
 916 static void lockdep_nfnl_nft_mutex_not_held(void)
 917 {
 918 #ifdef CONFIG_PROVE_LOCKING
 919         if (debug_locks)
 920                 WARN_ON_ONCE(lockdep_nfnl_is_held(NFNL_SUBSYS_NFTABLES));
 921 #endif
 922 }
 923
 924 static const struct nft_chain_type *
 925 nf_tables_chain_type_lookup(struct net *net, const struct nlattr *nla,
 926                             u8 family, bool autoload)
 927 {
 928         const struct nft_chain_type *type;
 929
 930         type = __nf_tables_chain_type_lookup(nla, family);
 931         if (type != NULL)
 932                 return type;
 933
 934         lockdep_nfnl_nft_mutex_not_held();
 935 #ifdef CONFIG_MODULES
 936         if (autoload) {
 937                 if (nft_request_module(net, "nft-chain-%u-%.*s", family,
 938                                        nla_len(nla),
 939                                        (const char *)nla_data(nla)) == -EAGAIN)
 940                         return ERR_PTR(-EAGAIN);
 941         }
 942 #endif
 943         return ERR_PTR(-ENOENT);
 944 }
 945
 946 static __be16 nft_base_seq(const struct net *net)
 947 {
 948         struct nftables_pernet *nft_net = nft_pernet(net);
 949
 950         return htons(nft_net->base_seq & 0xffff);
 951 }
 952
 953 static const struct nla_policy nft_table_policy[NFTA_TABLE_MAX + 1] = {
 954         [NFTA_TABLE_NAME]       = { .type = NLA_STRING,
 955                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
 956         [NFTA_TABLE_FLAGS]      = { .type = NLA_U32 },
 957         [NFTA_TABLE_HANDLE]     = { .type = NLA_U64 },
 958         [NFTA_TABLE_USERDATA]   = { .type = NLA_BINARY,
 959                                     .len = NFT_USERDATA_MAXLEN }
 960 };
 961
 962 static int nf_tables_fill_table_info(struct sk_buff *skb, struct net *net,
 963                                      u32 portid, u32 seq, int event, u32 flags,
 964                                      int family, const struct nft_table *table)
 965 {
 966         struct nlmsghdr *nlh;
 967
 968         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
 969         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
 970                            NFNETLINK_V0, nft_base_seq(net));
 971         if (!nlh)
 972                 goto nla_put_failure;
 973
 974         if (nla_put_string(skb, NFTA_TABLE_NAME, table->name) ||
 975             nla_put_be32(skb, NFTA_TABLE_USE, htonl(table->use)) ||
 976             nla_put_be64(skb, NFTA_TABLE_HANDLE, cpu_to_be64(table->handle),
 977                          NFTA_TABLE_PAD))
 978                 goto nla_put_failure;
 979
 980         if (event == NFT_MSG_DELTABLE) {
 981                 nlmsg_end(skb, nlh);
 982                 return 0;
 983         }
 984
 985         if (nla_put_be32(skb, NFTA_TABLE_FLAGS,
 986                          htonl(table->flags & NFT_TABLE_F_MASK)))
 987                 goto nla_put_failure;
 988
 989         if (nft_table_has_owner(table) &&
 990             nla_put_be32(skb, NFTA_TABLE_OWNER, htonl(table->nlpid)))
 991                 goto nla_put_failure;
 992
 993         if (table->udata) {
 994                 if (nla_put(skb, NFTA_TABLE_USERDATA, table->udlen, table->udata))
 995                         goto nla_put_failure;
 996         }
 997
 998         nlmsg_end(skb, nlh);
 999         return 0;
1000
1001 nla_put_failure:
1002         nlmsg_trim(skb, nlh);
1003         return -1;
1004 }
1005
1006 struct nftnl_skb_parms {
1007         bool report;
1008 };
1009 #define NFT_CB(skb)     (*(struct nftnl_skb_parms*)&((skb)->cb))
1010
1011 static void nft_notify_enqueue(struct sk_buff *skb, bool report,
1012                                struct list_head *notify_list)
1013 {
1014         NFT_CB(skb).report = report;
1015         list_add_tail(&skb->list, notify_list);
1016 }
1017
1018 static void nf_tables_table_notify(const struct nft_ctx *ctx, int event)
1019 {
1020         struct nftables_pernet *nft_net;
1021         struct sk_buff *skb;
1022         u16 flags = 0;
1023         int err;
1024
1025         if (!ctx->report &&
1026             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
1027                 return;
1028
1029         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
1030         if (skb == NULL)
1031                 goto err;
1032
1033         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
1034                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
1035
1036         err = nf_tables_fill_table_info(skb, ctx->net, ctx->portid, ctx->seq,
1037                                         event, flags, ctx->family, ctx->table);
1038         if (err < 0) {
1039                 kfree_skb(skb);
1040                 goto err;
1041         }
1042
1043         nft_net = nft_pernet(ctx->net);
1044         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
1045         return;
1046 err:
1047         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
1048 }
1049
1050 static int nf_tables_dump_tables(struct sk_buff *skb,
1051                                  struct netlink_callback *cb)
1052 {
1053         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
1054         struct nftables_pernet *nft_net;
1055         const struct nft_table *table;
1056         unsigned int idx = 0, s_idx = cb->args[0];
1057         struct net *net = sock_net(skb->sk);
1058         int family = nfmsg->nfgen_family;
1059
1060         rcu_read_lock();
1061         nft_net = nft_pernet(net);
1062         cb->seq = READ_ONCE(nft_net->base_seq);
1063
1064         list_for_each_entry_rcu(table, &nft_net->tables, list) {
1065                 if (family != NFPROTO_UNSPEC && family != table->family)
1066                         continue;
1067
1068                 if (idx < s_idx)
1069                         goto cont;
1070                 if (idx > s_idx)
1071                         memset(&cb->args[1], 0,
1072                                sizeof(cb->args) - sizeof(cb->args[0]));
1073                 if (!nft_is_active(net, table))
1074                         continue;
1075                 if (nf_tables_fill_table_info(skb, net,
1076                                               NETLINK_CB(cb->skb).portid,
1077                                               cb->nlh->nlmsg_seq,
1078                                               NFT_MSG_NEWTABLE, NLM_F_MULTI,
1079                                               table->family, table) < 0)
1080                         goto done;
1081
1082                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
1083 cont:
1084                 idx++;
1085         }
1086 done:
1087         rcu_read_unlock();
1088         cb->args[0] = idx;
1089         return skb->len;
1090 }
1091
1092 static int nft_netlink_dump_start_rcu(struct sock *nlsk, struct sk_buff *skb,
1093                                       const struct nlmsghdr *nlh,
1094                                       struct netlink_dump_control *c)
1095 {
1096         int err;
1097
1098         if (!try_module_get(THIS_MODULE))
1099                 return -EINVAL;
1100
1101         rcu_read_unlock();
1102         err = netlink_dump_start(nlsk, skb, nlh, c);
1103         rcu_read_lock();
1104         module_put(THIS_MODULE);
1105
1106         return err;
1107 }
1108
1109 /* called with rcu_read_lock held */
1110 static int nf_tables_gettable(struct sk_buff *skb, const struct nfnl_info *info,
1111                               const struct nlattr * const nla[])
1112 {
1113         struct netlink_ext_ack *extack = info->extack;
1114         u8 genmask = nft_genmask_cur(info->net);
1115         u8 family = info->nfmsg->nfgen_family;
1116         const struct nft_table *table;
1117         struct net *net = info->net;
1118         struct sk_buff *skb2;
1119         int err;
1120
1121         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
1122                 struct netlink_dump_control c = {
1123                         .dump = nf_tables_dump_tables,
1124                         .module = THIS_MODULE,
1125                 };
1126
1127                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
1128         }
1129
1130         table = nft_table_lookup(net, nla[NFTA_TABLE_NAME], family, genmask, 0);
1131         if (IS_ERR(table)) {
1132                 NL_SET_BAD_ATTR(extack, nla[NFTA_TABLE_NAME]);
1133                 return PTR_ERR(table);
1134         }
1135
1136         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
1137         if (!skb2)
1138                 return -ENOMEM;
1139
1140         err = nf_tables_fill_table_info(skb2, net, NETLINK_CB(skb).portid,
1141                                         info->nlh->nlmsg_seq, NFT_MSG_NEWTABLE,
1142                                         0, family, table);
1143         if (err < 0)
1144                 goto err_fill_table_info;
1145
1146         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
1147
1148 err_fill_table_info:
1149         kfree_skb(skb2);
1150         return err;
1151 }
1152
1153 static void nft_table_disable(struct net *net, struct nft_table *table, u32 cnt)
1154 {
1155         struct nft_chain *chain;
1156         u32 i = 0;
1157
1158         list_for_each_entry(chain, &table->chains, list) {
1159                 if (!nft_is_active_next(net, chain))
1160                         continue;
1161                 if (!nft_is_base_chain(chain))
1162                         continue;
1163
1164                 if (cnt && i++ == cnt)
1165                         break;
1166
1167                 nf_tables_unregister_hook(net, table, chain);
1168         }
1169 }
1170
1171 static int nf_tables_table_enable(struct net *net, struct nft_table *table)
1172 {
1173         struct nft_chain *chain;
1174         int err, i = 0;
1175
1176         list_for_each_entry(chain, &table->chains, list) {
1177                 if (!nft_is_active_next(net, chain))
1178                         continue;
1179                 if (!nft_is_base_chain(chain))
1180                         continue;
1181
1182                 err = nf_tables_register_hook(net, table, chain);
1183                 if (err < 0)
1184                         goto err_register_hooks;
1185
1186                 i++;
1187         }
1188         return 0;
1189
1190 err_register_hooks:
1191         if (i)
1192                 nft_table_disable(net, table, i);
1193         return err;
1194 }
1195
1196 static void nf_tables_table_disable(struct net *net, struct nft_table *table)
1197 {
1198         table->flags &= ~NFT_TABLE_F_DORMANT;
1199         nft_table_disable(net, table, 0);
1200         table->flags |= NFT_TABLE_F_DORMANT;
1201 }
1202
1203 #define __NFT_TABLE_F_INTERNAL          (NFT_TABLE_F_MASK + 1)
1204 #define __NFT_TABLE_F_WAS_DORMANT       (__NFT_TABLE_F_INTERNAL << 0)
1205 #define __NFT_TABLE_F_WAS_AWAKEN        (__NFT_TABLE_F_INTERNAL << 1)
1206 #define __NFT_TABLE_F_WAS_ORPHAN        (__NFT_TABLE_F_INTERNAL << 2)
1207 #define __NFT_TABLE_F_UPDATE            (__NFT_TABLE_F_WAS_DORMANT | \
1208                                          __NFT_TABLE_F_WAS_AWAKEN | \
1209                                          __NFT_TABLE_F_WAS_ORPHAN)
1210
1211 static bool nft_table_pending_update(const struct nft_ctx *ctx)
1212 {
1213         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
1214         struct nft_trans *trans;
1215
1216         if (ctx->table->flags & __NFT_TABLE_F_UPDATE)
1217                 return true;
1218
1219         list_for_each_entry(trans, &nft_net->commit_list, list) {
1220                 if (trans->ctx.table == ctx->table &&
1221                     ((trans->msg_type == NFT_MSG_NEWCHAIN &&
1222                       nft_trans_chain_update(trans)) ||
1223                      (trans->msg_type == NFT_MSG_DELCHAIN &&
1224                       nft_is_base_chain(trans->ctx.chain))))
1225                         return true;
1226         }
1227
1228         return false;
1229 }
1230
1231 static int nf_tables_updtable(struct nft_ctx *ctx)
1232 {
1233         struct nft_trans *trans;
1234         u32 flags;
1235         int ret;
1236
1237         if (!ctx->nla[NFTA_TABLE_FLAGS])
1238                 return 0;
1239
1240         flags = ntohl(nla_get_be32(ctx->nla[NFTA_TABLE_FLAGS]));
1241         if (flags & ~NFT_TABLE_F_MASK)
1242                 return -EOPNOTSUPP;
1243
1244         if (flags == (ctx->table->flags & NFT_TABLE_F_MASK))
1245                 return 0;
1246
1247         if ((nft_table_has_owner(ctx->table) &&
1248              !(flags & NFT_TABLE_F_OWNER)) ||
1249             (flags & NFT_TABLE_F_OWNER &&
1250              !nft_table_is_orphan(ctx->table)))
1251                 return -EOPNOTSUPP;
1252
1253         if ((flags ^ ctx->table->flags) & NFT_TABLE_F_PERSIST)
1254                 return -EOPNOTSUPP;
1255
1256         /* No dormant off/on/off/on games in single transaction */
1257         if (nft_table_pending_update(ctx))
1258                 return -EINVAL;
1259
1260         trans = nft_trans_alloc(ctx, NFT_MSG_NEWTABLE,
1261                                 sizeof(struct nft_trans_table));
1262         if (trans == NULL)
1263                 return -ENOMEM;
1264
1265         if ((flags & NFT_TABLE_F_DORMANT) &&
1266             !(ctx->table->flags & NFT_TABLE_F_DORMANT)) {
1267                 ctx->table->flags |= NFT_TABLE_F_DORMANT;
1268                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE))
1269                         ctx->table->flags |= __NFT_TABLE_F_WAS_AWAKEN;
1270         } else if (!(flags & NFT_TABLE_F_DORMANT) &&
1271                    ctx->table->flags & NFT_TABLE_F_DORMANT) {
1272                 ctx->table->flags &= ~NFT_TABLE_F_DORMANT;
1273                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE)) {
1274                         ret = nf_tables_table_enable(ctx->net, ctx->table);
1275                         if (ret < 0)
1276                                 goto err_register_hooks;
1277
1278                         ctx->table->flags |= __NFT_TABLE_F_WAS_DORMANT;
1279                 }
1280         }
1281
1282         if ((flags & NFT_TABLE_F_OWNER) &&
1283             !nft_table_has_owner(ctx->table)) {
1284                 ctx->table->nlpid = ctx->portid;
1285                 ctx->table->flags |= NFT_TABLE_F_OWNER |
1286                                      __NFT_TABLE_F_WAS_ORPHAN;
1287         }
1288
1289         nft_trans_table_update(trans) = true;
1290         nft_trans_commit_list_add_tail(ctx->net, trans);
1291
1292         return 0;
1293
1294 err_register_hooks:
1295         ctx->table->flags |= NFT_TABLE_F_DORMANT;
1296         nft_trans_destroy(trans);
1297         return ret;
1298 }
1299
1300 static u32 nft_chain_hash(const void *data, u32 len, u32 seed)
1301 {
1302         const char *name = data;
1303
1304         return jhash(name, strlen(name), seed);
1305 }
1306
1307 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed)
1308 {
1309         const struct nft_chain *chain = data;
1310
1311         return nft_chain_hash(chain->name, 0, seed);
1312 }
1313
1314 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *arg,
1315                               const void *ptr)
1316 {
1317         const struct nft_chain *chain = ptr;
1318         const char *name = arg->key;
1319
1320         return strcmp(chain->name, name);
1321 }
1322
1323 static u32 nft_objname_hash(const void *data, u32 len, u32 seed)
1324 {
1325         const struct nft_object_hash_key *k = data;
1326
1327         seed ^= hash_ptr(k->table, 32);
1328
1329         return jhash(k->name, strlen(k->name), seed);
1330 }
1331
1332 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed)
1333 {
1334         const struct nft_object *obj = data;
1335
1336         return nft_objname_hash(&obj->key, 0, seed);
1337 }
1338
1339 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *arg,
1340                                 const void *ptr)
1341 {
1342         const struct nft_object_hash_key *k = arg->key;
1343         const struct nft_object *obj = ptr;
1344
1345         if (obj->key.table != k->table)
1346                 return -1;
1347
1348         return strcmp(obj->key.name, k->name);
1349 }
1350
1351 static bool nft_supported_family(u8 family)
1352 {
1353         return false
1354 #ifdef CONFIG_NF_TABLES_INET
1355                 || family == NFPROTO_INET
1356 #endif
1357 #ifdef CONFIG_NF_TABLES_IPV4
1358                 || family == NFPROTO_IPV4
1359 #endif
1360 #ifdef CONFIG_NF_TABLES_ARP
1361                 || family == NFPROTO_ARP
1362 #endif
1363 #ifdef CONFIG_NF_TABLES_NETDEV
1364                 || family == NFPROTO_NETDEV
1365 #endif
1366 #if IS_ENABLED(CONFIG_NF_TABLES_BRIDGE)
1367                 || family == NFPROTO_BRIDGE
1368 #endif
1369 #ifdef CONFIG_NF_TABLES_IPV6
1370                 || family == NFPROTO_IPV6
1371 #endif
1372                 ;
1373 }
1374
1375 static int nf_tables_newtable(struct sk_buff *skb, const struct nfnl_info *info,
1376                               const struct nlattr * const nla[])
1377 {
1378         struct nftables_pernet *nft_net = nft_pernet(info->net);
1379         struct netlink_ext_ack *extack = info->extack;
1380         u8 genmask = nft_genmask_next(info->net);
1381         u8 family = info->nfmsg->nfgen_family;
1382         struct net *net = info->net;
1383         const struct nlattr *attr;
1384         struct nft_table *table;
1385         struct nft_ctx ctx;
1386         u32 flags = 0;
1387         int err;
1388
1389         if (!nft_supported_family(family))
1390                 return -EOPNOTSUPP;
1391
1392         lockdep_assert_held(&nft_net->commit_mutex);
1393         attr = nla[NFTA_TABLE_NAME];
1394         table = nft_table_lookup(net, attr, family, genmask,
1395                                  NETLINK_CB(skb).portid);
1396         if (IS_ERR(table)) {
1397                 if (PTR_ERR(table) != -ENOENT)
1398                         return PTR_ERR(table);
1399         } else {
1400                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
1401                         NL_SET_BAD_ATTR(extack, attr);
1402                         return -EEXIST;
1403                 }
1404                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
1405                         return -EOPNOTSUPP;
1406
1407                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1408
1409                 return nf_tables_updtable(&ctx);
1410         }
1411
1412         if (nla[NFTA_TABLE_FLAGS]) {
1413                 flags = ntohl(nla_get_be32(nla[NFTA_TABLE_FLAGS]));
1414                 if (flags & ~NFT_TABLE_F_MASK)
1415                         return -EOPNOTSUPP;
1416         }
1417
1418         err = -ENOMEM;
1419         table = kzalloc(sizeof(*table), GFP_KERNEL_ACCOUNT);
1420         if (table == NULL)
1421                 goto err_kzalloc;
1422
1423         table->validate_state = nft_net->validate_state;
1424         table->name = nla_strdup(attr, GFP_KERNEL_ACCOUNT);
1425         if (table->name == NULL)
1426                 goto err_strdup;
1427
1428         if (nla[NFTA_TABLE_USERDATA]) {
1429                 table->udata = nla_memdup(nla[NFTA_TABLE_USERDATA], GFP_KERNEL_ACCOUNT);
1430                 if (table->udata == NULL)
1431                         goto err_table_udata;
1432
1433                 table->udlen = nla_len(nla[NFTA_TABLE_USERDATA]);
1434         }
1435
1436         err = rhltable_init(&table->chains_ht, &nft_chain_ht_params);
1437         if (err)
1438                 goto err_chain_ht;
1439
1440         INIT_LIST_HEAD(&table->chains);
1441         INIT_LIST_HEAD(&table->sets);
1442         INIT_LIST_HEAD(&table->objects);
1443         INIT_LIST_HEAD(&table->flowtables);
1444         table->family = family;
1445         table->flags = flags;
1446         table->handle = ++nft_net->table_handle;
1447         if (table->flags & NFT_TABLE_F_OWNER)
1448                 table->nlpid = NETLINK_CB(skb).portid;
1449
1450         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1451         err = nft_trans_table_add(&ctx, NFT_MSG_NEWTABLE);
1452         if (err < 0)
1453                 goto err_trans;
1454
1455         list_add_tail_rcu(&table->list, &nft_net->tables);
1456         return 0;
1457 err_trans:
1458         rhltable_destroy(&table->chains_ht);
1459 err_chain_ht:
1460         kfree(table->udata);
1461 err_table_udata:
1462         kfree(table->name);
1463 err_strdup:
1464         kfree(table);
1465 err_kzalloc:
1466         return err;
1467 }
1468
1469 static int nft_flush_table(struct nft_ctx *ctx)
1470 {
1471         struct nft_flowtable *flowtable, *nft;
1472         struct nft_chain *chain, *nc;
1473         struct nft_object *obj, *ne;
1474         struct nft_set *set, *ns;
1475         int err;
1476
1477         list_for_each_entry(chain, &ctx->table->chains, list) {
1478                 if (!nft_is_active_next(ctx->net, chain))
1479                         continue;
1480
1481                 if (nft_chain_binding(chain))
1482                         continue;
1483
1484                 ctx->chain = chain;
1485
1486                 err = nft_delrule_by_chain(ctx);
1487                 if (err < 0)
1488                         goto out;
1489         }
1490
1491         list_for_each_entry_safe(set, ns, &ctx->table->sets, list) {
1492                 if (!nft_is_active_next(ctx->net, set))
1493                         continue;
1494
1495                 if (nft_set_is_anonymous(set))
1496                         continue;
1497
1498                 err = nft_delset(ctx, set);
1499                 if (err < 0)
1500                         goto out;
1501         }
1502
1503         list_for_each_entry_safe(flowtable, nft, &ctx->table->flowtables, list) {
1504                 if (!nft_is_active_next(ctx->net, flowtable))
1505                         continue;
1506
1507                 err = nft_delflowtable(ctx, flowtable);
1508                 if (err < 0)
1509                         goto out;
1510         }
1511
1512         list_for_each_entry_safe(obj, ne, &ctx->table->objects, list) {
1513                 if (!nft_is_active_next(ctx->net, obj))
1514                         continue;
1515
1516                 err = nft_delobj(ctx, obj);
1517                 if (err < 0)
1518                         goto out;
1519         }
1520
1521         list_for_each_entry_safe(chain, nc, &ctx->table->chains, list) {
1522                 if (!nft_is_active_next(ctx->net, chain))
1523                         continue;
1524
1525                 if (nft_chain_binding(chain))
1526                         continue;
1527
1528                 ctx->chain = chain;
1529
1530                 err = nft_delchain(ctx);
1531                 if (err < 0)
1532                         goto out;
1533         }
1534
1535         err = nft_deltable(ctx);
1536 out:
1537         return err;
1538 }
1539
1540 static int nft_flush(struct nft_ctx *ctx, int family)
1541 {
1542         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
1543         const struct nlattr * const *nla = ctx->nla;
1544         struct nft_table *table, *nt;
1545         int err = 0;
1546
1547         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
1548                 if (family != AF_UNSPEC && table->family != family)
1549                         continue;
1550
1551                 ctx->family = table->family;
1552
1553                 if (!nft_is_active_next(ctx->net, table))
1554                         continue;
1555
1556                 if (nft_table_has_owner(table) && table->nlpid != ctx->portid)
1557                         continue;
1558
1559                 if (nla[NFTA_TABLE_NAME] &&
1560                     nla_strcmp(nla[NFTA_TABLE_NAME], table->name) != 0)
1561                         continue;
1562
1563                 ctx->table = table;
1564
1565                 err = nft_flush_table(ctx);
1566                 if (err < 0)
1567                         goto out;
1568         }
1569 out:
1570         return err;
1571 }
1572
1573 static int nf_tables_deltable(struct sk_buff *skb, const struct nfnl_info *info,
1574                               const struct nlattr * const nla[])
1575 {
1576         struct netlink_ext_ack *extack = info->extack;
1577         u8 genmask = nft_genmask_next(info->net);
1578         u8 family = info->nfmsg->nfgen_family;
1579         struct net *net = info->net;
1580         const struct nlattr *attr;
1581         struct nft_table *table;
1582         struct nft_ctx ctx;
1583
1584         nft_ctx_init(&ctx, net, skb, info->nlh, 0, NULL, NULL, nla);
1585         if (family == AF_UNSPEC ||
1586             (!nla[NFTA_TABLE_NAME] && !nla[NFTA_TABLE_HANDLE]))
1587                 return nft_flush(&ctx, family);
1588
1589         if (nla[NFTA_TABLE_HANDLE]) {
1590                 attr = nla[NFTA_TABLE_HANDLE];
1591                 table = nft_table_lookup_byhandle(net, attr, family, genmask,
1592                                                   NETLINK_CB(skb).portid);
1593         } else {
1594                 attr = nla[NFTA_TABLE_NAME];
1595                 table = nft_table_lookup(net, attr, family, genmask,
1596                                          NETLINK_CB(skb).portid);
1597         }
1598
1599         if (IS_ERR(table)) {
1600                 if (PTR_ERR(table) == -ENOENT &&
1601                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYTABLE)
1602                         return 0;
1603
1604                 NL_SET_BAD_ATTR(extack, attr);
1605                 return PTR_ERR(table);
1606         }
1607
1608         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
1609             table->use > 0)
1610                 return -EBUSY;
1611
1612         ctx.family = family;
1613         ctx.table = table;
1614
1615         return nft_flush_table(&ctx);
1616 }
1617
1618 static void nf_tables_table_destroy(struct nft_ctx *ctx)
1619 {
1620         if (WARN_ON(ctx->table->use > 0))
1621                 return;
1622
1623         rhltable_destroy(&ctx->table->chains_ht);
1624         kfree(ctx->table->name);
1625         kfree(ctx->table->udata);
1626         kfree(ctx->table);
1627 }
1628
1629 void nft_register_chain_type(const struct nft_chain_type *ctype)
1630 {
1631         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1632         if (WARN_ON(__nft_chain_type_get(ctype->family, ctype->type))) {
1633                 nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1634                 return;
1635         }
1636         chain_type[ctype->family][ctype->type] = ctype;
1637         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1638 }
1639 EXPORT_SYMBOL_GPL(nft_register_chain_type);
1640
1641 void nft_unregister_chain_type(const struct nft_chain_type *ctype)
1642 {
1643         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1644         chain_type[ctype->family][ctype->type] = NULL;
1645         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1646 }
1647 EXPORT_SYMBOL_GPL(nft_unregister_chain_type);
1648
1649 /*
1650  * Chains
1651  */
1652
1653 static struct nft_chain *
1654 nft_chain_lookup_byhandle(const struct nft_table *table, u64 handle, u8 genmask)
1655 {
1656         struct nft_chain *chain;
1657
1658         list_for_each_entry(chain, &table->chains, list) {
1659                 if (chain->handle == handle &&
1660                     nft_active_genmask(chain, genmask))
1661                         return chain;
1662         }
1663
1664         return ERR_PTR(-ENOENT);
1665 }
1666
1667 static bool lockdep_commit_lock_is_held(const struct net *net)
1668 {
1669 #ifdef CONFIG_PROVE_LOCKING
1670         struct nftables_pernet *nft_net = nft_pernet(net);
1671
1672         return lockdep_is_held(&nft_net->commit_mutex);
1673 #else
1674         return true;
1675 #endif
1676 }
1677
1678 static struct nft_chain *nft_chain_lookup(struct net *net,
1679                                           struct nft_table *table,
1680                                           const struct nlattr *nla, u8 genmask)
1681 {
1682         char search[NFT_CHAIN_MAXNAMELEN + 1];
1683         struct rhlist_head *tmp, *list;
1684         struct nft_chain *chain;
1685
1686         if (nla == NULL)
1687                 return ERR_PTR(-EINVAL);
1688
1689         nla_strscpy(search, nla, sizeof(search));
1690
1691         WARN_ON(!rcu_read_lock_held() &&
1692                 !lockdep_commit_lock_is_held(net));
1693
1694         chain = ERR_PTR(-ENOENT);
1695         rcu_read_lock();
1696         list = rhltable_lookup(&table->chains_ht, search, nft_chain_ht_params);
1697         if (!list)
1698                 goto out_unlock;
1699
1700         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
1701                 if (nft_active_genmask(chain, genmask))
1702                         goto out_unlock;
1703         }
1704         chain = ERR_PTR(-ENOENT);
1705 out_unlock:
1706         rcu_read_unlock();
1707         return chain;
1708 }
1709
1710 static const struct nla_policy nft_chain_policy[NFTA_CHAIN_MAX + 1] = {
1711         [NFTA_CHAIN_TABLE]      = { .type = NLA_STRING,
1712                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
1713         [NFTA_CHAIN_HANDLE]     = { .type = NLA_U64 },
1714         [NFTA_CHAIN_NAME]       = { .type = NLA_STRING,
1715                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
1716         [NFTA_CHAIN_HOOK]       = { .type = NLA_NESTED },
1717         [NFTA_CHAIN_POLICY]     = { .type = NLA_U32 },
1718         [NFTA_CHAIN_TYPE]       = { .type = NLA_STRING,
1719                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
1720         [NFTA_CHAIN_COUNTERS]   = { .type = NLA_NESTED },
1721         [NFTA_CHAIN_FLAGS]      = { .type = NLA_U32 },
1722         [NFTA_CHAIN_ID]         = { .type = NLA_U32 },
1723         [NFTA_CHAIN_USERDATA]   = { .type = NLA_BINARY,
1724                                     .len = NFT_USERDATA_MAXLEN },
1725 };
1726
1727 static const struct nla_policy nft_hook_policy[NFTA_HOOK_MAX + 1] = {
1728         [NFTA_HOOK_HOOKNUM]     = { .type = NLA_U32 },
1729         [NFTA_HOOK_PRIORITY]    = { .type = NLA_U32 },
1730         [NFTA_HOOK_DEV]         = { .type = NLA_STRING,
1731                                     .len = IFNAMSIZ - 1 },
1732 };
1733
1734 static int nft_dump_stats(struct sk_buff *skb, struct nft_stats __percpu *stats)
1735 {
1736         struct nft_stats *cpu_stats, total;
1737         struct nlattr *nest;
1738         unsigned int seq;
1739         u64 pkts, bytes;
1740         int cpu;
1741
1742         if (!stats)
1743                 return 0;
1744
1745         memset(&total, 0, sizeof(total));
1746         for_each_possible_cpu(cpu) {
1747                 cpu_stats = per_cpu_ptr(stats, cpu);
1748                 do {
1749                         seq = u64_stats_fetch_begin(&cpu_stats->syncp);
1750                         pkts = cpu_stats->pkts;
1751                         bytes = cpu_stats->bytes;
1752                 } while (u64_stats_fetch_retry(&cpu_stats->syncp, seq));
1753                 total.pkts += pkts;
1754                 total.bytes += bytes;
1755         }
1756         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_COUNTERS);
1757         if (nest == NULL)
1758                 goto nla_put_failure;
1759
1760         if (nla_put_be64(skb, NFTA_COUNTER_PACKETS, cpu_to_be64(total.pkts),
1761                          NFTA_COUNTER_PAD) ||
1762             nla_put_be64(skb, NFTA_COUNTER_BYTES, cpu_to_be64(total.bytes),
1763                          NFTA_COUNTER_PAD))
1764                 goto nla_put_failure;
1765
1766         nla_nest_end(skb, nest);
1767         return 0;
1768
1769 nla_put_failure:
1770         return -ENOSPC;
1771 }
1772
1773 static int nft_dump_basechain_hook(struct sk_buff *skb, int family,
1774                                    const struct nft_base_chain *basechain,
1775                                    const struct list_head *hook_list)
1776 {
1777         const struct nf_hook_ops *ops = &basechain->ops;
1778         struct nft_hook *hook, *first = NULL;
1779         struct nlattr *nest, *nest_devs;
1780         int n = 0;
1781
1782         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_HOOK);
1783         if (nest == NULL)
1784                 goto nla_put_failure;
1785         if (nla_put_be32(skb, NFTA_HOOK_HOOKNUM, htonl(ops->hooknum)))
1786                 goto nla_put_failure;
1787         if (nla_put_be32(skb, NFTA_HOOK_PRIORITY, htonl(ops->priority)))
1788                 goto nla_put_failure;
1789
1790         if (nft_base_chain_netdev(family, ops->hooknum)) {
1791                 nest_devs = nla_nest_start_noflag(skb, NFTA_HOOK_DEVS);
1792                 if (!nest_devs)
1793                         goto nla_put_failure;
1794
1795                 if (!hook_list)
1796                         hook_list = &basechain->hook_list;
1797
1798                 list_for_each_entry(hook, hook_list, list) {
1799                         if (!first)
1800                                 first = hook;
1801
1802                         if (nla_put_string(skb, NFTA_DEVICE_NAME,
1803                                            hook->ops.dev->name))
1804                                 goto nla_put_failure;
1805                         n++;
1806                 }
1807                 nla_nest_end(skb, nest_devs);
1808
1809                 if (n == 1 &&
1810                     nla_put_string(skb, NFTA_HOOK_DEV, first->ops.dev->name))
1811                         goto nla_put_failure;
1812         }
1813         nla_nest_end(skb, nest);
1814
1815         return 0;
1816 nla_put_failure:
1817         return -1;
1818 }
1819
1820 static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net,
1821                                      u32 portid, u32 seq, int event, u32 flags,
1822                                      int family, const struct nft_table *table,
1823                                      const struct nft_chain *chain,
1824                                      const struct list_head *hook_list)
1825 {
1826         struct nlmsghdr *nlh;
1827
1828         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
1829         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
1830                            NFNETLINK_V0, nft_base_seq(net));
1831         if (!nlh)
1832                 goto nla_put_failure;
1833
1834         if (nla_put_string(skb, NFTA_CHAIN_TABLE, table->name) ||
1835             nla_put_string(skb, NFTA_CHAIN_NAME, chain->name) ||
1836             nla_put_be64(skb, NFTA_CHAIN_HANDLE, cpu_to_be64(chain->handle),
1837                          NFTA_CHAIN_PAD))
1838                 goto nla_put_failure;
1839
1840         if (event == NFT_MSG_DELCHAIN && !hook_list) {
1841                 nlmsg_end(skb, nlh);
1842                 return 0;
1843         }
1844
1845         if (nft_is_base_chain(chain)) {
1846                 const struct nft_base_chain *basechain = nft_base_chain(chain);
1847                 struct nft_stats __percpu *stats;
1848
1849                 if (nft_dump_basechain_hook(skb, family, basechain, hook_list))
1850                         goto nla_put_failure;
1851
1852                 if (nla_put_be32(skb, NFTA_CHAIN_POLICY,
1853                                  htonl(basechain->policy)))
1854                         goto nla_put_failure;
1855
1856                 if (nla_put_string(skb, NFTA_CHAIN_TYPE, basechain->type->name))
1857                         goto nla_put_failure;
1858
1859                 stats = rcu_dereference_check(basechain->stats,
1860                                               lockdep_commit_lock_is_held(net));
1861                 if (nft_dump_stats(skb, stats))
1862                         goto nla_put_failure;
1863         }
1864
1865         if (chain->flags &&
1866             nla_put_be32(skb, NFTA_CHAIN_FLAGS, htonl(chain->flags)))
1867                 goto nla_put_failure;
1868
1869         if (nla_put_be32(skb, NFTA_CHAIN_USE, htonl(chain->use)))
1870                 goto nla_put_failure;
1871
1872         if (chain->udata &&
1873             nla_put(skb, NFTA_CHAIN_USERDATA, chain->udlen, chain->udata))
1874                 goto nla_put_failure;
1875
1876         nlmsg_end(skb, nlh);
1877         return 0;
1878
1879 nla_put_failure:
1880         nlmsg_trim(skb, nlh);
1881         return -1;
1882 }
1883
1884 static void nf_tables_chain_notify(const struct nft_ctx *ctx, int event,
1885                                    const struct list_head *hook_list)
1886 {
1887         struct nftables_pernet *nft_net;
1888         struct sk_buff *skb;
1889         u16 flags = 0;
1890         int err;
1891
1892         if (!ctx->report &&
1893             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
1894                 return;
1895
1896         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
1897         if (skb == NULL)
1898                 goto err;
1899
1900         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
1901                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
1902
1903         err = nf_tables_fill_chain_info(skb, ctx->net, ctx->portid, ctx->seq,
1904                                         event, flags, ctx->family, ctx->table,
1905                                         ctx->chain, hook_list);
1906         if (err < 0) {
1907                 kfree_skb(skb);
1908                 goto err;
1909         }
1910
1911         nft_net = nft_pernet(ctx->net);
1912         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
1913         return;
1914 err:
1915         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
1916 }
1917
1918 static int nf_tables_dump_chains(struct sk_buff *skb,
1919                                  struct netlink_callback *cb)
1920 {
1921         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
1922         unsigned int idx = 0, s_idx = cb->args[0];
1923         struct net *net = sock_net(skb->sk);
1924         int family = nfmsg->nfgen_family;
1925         struct nftables_pernet *nft_net;
1926         const struct nft_table *table;
1927         const struct nft_chain *chain;
1928
1929         rcu_read_lock();
1930         nft_net = nft_pernet(net);
1931         cb->seq = READ_ONCE(nft_net->base_seq);
1932
1933         list_for_each_entry_rcu(table, &nft_net->tables, list) {
1934                 if (family != NFPROTO_UNSPEC && family != table->family)
1935                         continue;
1936
1937                 list_for_each_entry_rcu(chain, &table->chains, list) {
1938                         if (idx < s_idx)
1939                                 goto cont;
1940                         if (idx > s_idx)
1941                                 memset(&cb->args[1], 0,
1942                                        sizeof(cb->args) - sizeof(cb->args[0]));
1943                         if (!nft_is_active(net, chain))
1944                                 continue;
1945                         if (nf_tables_fill_chain_info(skb, net,
1946                                                       NETLINK_CB(cb->skb).portid,
1947                                                       cb->nlh->nlmsg_seq,
1948                                                       NFT_MSG_NEWCHAIN,
1949                                                       NLM_F_MULTI,
1950                                                       table->family, table,
1951                                                       chain, NULL) < 0)
1952                                 goto done;
1953
1954                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
1955 cont:
1956                         idx++;
1957                 }
1958         }
1959 done:
1960         rcu_read_unlock();
1961         cb->args[0] = idx;
1962         return skb->len;
1963 }
1964
1965 /* called with rcu_read_lock held */
1966 static int nf_tables_getchain(struct sk_buff *skb, const struct nfnl_info *info,
1967                               const struct nlattr * const nla[])
1968 {
1969         struct netlink_ext_ack *extack = info->extack;
1970         u8 genmask = nft_genmask_cur(info->net);
1971         u8 family = info->nfmsg->nfgen_family;
1972         const struct nft_chain *chain;
1973         struct net *net = info->net;
1974         struct nft_table *table;
1975         struct sk_buff *skb2;
1976         int err;
1977
1978         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
1979                 struct netlink_dump_control c = {
1980                         .dump = nf_tables_dump_chains,
1981                         .module = THIS_MODULE,
1982                 };
1983
1984                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
1985         }
1986
1987         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask, 0);
1988         if (IS_ERR(table)) {
1989                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
1990                 return PTR_ERR(table);
1991         }
1992
1993         chain = nft_chain_lookup(net, table, nla[NFTA_CHAIN_NAME], genmask);
1994         if (IS_ERR(chain)) {
1995                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
1996                 return PTR_ERR(chain);
1997         }
1998
1999         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
2000         if (!skb2)
2001                 return -ENOMEM;
2002
2003         err = nf_tables_fill_chain_info(skb2, net, NETLINK_CB(skb).portid,
2004                                         info->nlh->nlmsg_seq, NFT_MSG_NEWCHAIN,
2005                                         0, family, table, chain, NULL);
2006         if (err < 0)
2007                 goto err_fill_chain_info;
2008
2009         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
2010
2011 err_fill_chain_info:
2012         kfree_skb(skb2);
2013         return err;
2014 }
2015
2016 static const struct nla_policy nft_counter_policy[NFTA_COUNTER_MAX + 1] = {
2017         [NFTA_COUNTER_PACKETS]  = { .type = NLA_U64 },
2018         [NFTA_COUNTER_BYTES]    = { .type = NLA_U64 },
2019 };
2020
2021 static struct nft_stats __percpu *nft_stats_alloc(const struct nlattr *attr)
2022 {
2023         struct nlattr *tb[NFTA_COUNTER_MAX+1];
2024         struct nft_stats __percpu *newstats;
2025         struct nft_stats *stats;
2026         int err;
2027
2028         err = nla_parse_nested_deprecated(tb, NFTA_COUNTER_MAX, attr,
2029                                           nft_counter_policy, NULL);
2030         if (err < 0)
2031                 return ERR_PTR(err);
2032
2033         if (!tb[NFTA_COUNTER_BYTES] || !tb[NFTA_COUNTER_PACKETS])
2034                 return ERR_PTR(-EINVAL);
2035
2036         newstats = netdev_alloc_pcpu_stats(struct nft_stats);
2037         if (newstats == NULL)
2038                 return ERR_PTR(-ENOMEM);
2039
2040         /* Restore old counters on this cpu, no problem. Per-cpu statistics
2041          * are not exposed to userspace.
2042          */
2043         preempt_disable();
2044         stats = this_cpu_ptr(newstats);
2045         stats->bytes = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_BYTES]));
2046         stats->pkts = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_PACKETS]));
2047         preempt_enable();
2048
2049         return newstats;
2050 }
2051
2052 static void nft_chain_stats_replace(struct nft_trans *trans)
2053 {
2054         struct nft_base_chain *chain = nft_base_chain(trans->ctx.chain);
2055
2056         if (!nft_trans_chain_stats(trans))
2057                 return;
2058
2059         nft_trans_chain_stats(trans) =
2060                 rcu_replace_pointer(chain->stats, nft_trans_chain_stats(trans),
2061                                     lockdep_commit_lock_is_held(trans->ctx.net));
2062
2063         if (!nft_trans_chain_stats(trans))
2064                 static_branch_inc(&nft_counters_enabled);
2065 }
2066
2067 static void nf_tables_chain_free_chain_rules(struct nft_chain *chain)
2068 {
2069         struct nft_rule_blob *g0 = rcu_dereference_raw(chain->blob_gen_0);
2070         struct nft_rule_blob *g1 = rcu_dereference_raw(chain->blob_gen_1);
2071
2072         if (g0 != g1)
2073                 kvfree(g1);
2074         kvfree(g0);
2075
2076         /* should be NULL either via abort or via successful commit */
2077         WARN_ON_ONCE(chain->blob_next);
2078         kvfree(chain->blob_next);
2079 }
2080
2081 void nf_tables_chain_destroy(struct nft_ctx *ctx)
2082 {
2083         struct nft_chain *chain = ctx->chain;
2084         struct nft_hook *hook, *next;
2085
2086         if (WARN_ON(chain->use > 0))
2087                 return;
2088
2089         /* no concurrent access possible anymore */
2090         nf_tables_chain_free_chain_rules(chain);
2091
2092         if (nft_is_base_chain(chain)) {
2093                 struct nft_base_chain *basechain = nft_base_chain(chain);
2094
2095                 if (nft_base_chain_netdev(ctx->family, basechain->ops.hooknum)) {
2096                         list_for_each_entry_safe(hook, next,
2097                                                  &basechain->hook_list, list) {
2098                                 list_del_rcu(&hook->list);
2099                                 kfree_rcu(hook, rcu);
2100                         }
2101                 }
2102                 module_put(basechain->type->owner);
2103                 if (rcu_access_pointer(basechain->stats)) {
2104                         static_branch_dec(&nft_counters_enabled);
2105                         free_percpu(rcu_dereference_raw(basechain->stats));
2106                 }
2107                 kfree(chain->name);
2108                 kfree(chain->udata);
2109                 kfree(basechain);
2110         } else {
2111                 kfree(chain->name);
2112                 kfree(chain->udata);
2113                 kfree(chain);
2114         }
2115 }
2116
2117 static struct nft_hook *nft_netdev_hook_alloc(struct net *net,
2118                                               const struct nlattr *attr)
2119 {
2120         struct net_device *dev;
2121         char ifname[IFNAMSIZ];
2122         struct nft_hook *hook;
2123         int err;
2124
2125         hook = kzalloc(sizeof(struct nft_hook), GFP_KERNEL_ACCOUNT);
2126         if (!hook) {
2127                 err = -ENOMEM;
2128                 goto err_hook_alloc;
2129         }
2130
2131         nla_strscpy(ifname, attr, IFNAMSIZ);
2132         /* nf_tables_netdev_event() is called under rtnl_mutex, this is
2133          * indirectly serializing all the other holders of the commit_mutex with
2134          * the rtnl_mutex.
2135          */
2136         dev = __dev_get_by_name(net, ifname);
2137         if (!dev) {
2138                 err = -ENOENT;
2139                 goto err_hook_dev;
2140         }
2141         hook->ops.dev = dev;
2142
2143         return hook;
2144
2145 err_hook_dev:
2146         kfree(hook);
2147 err_hook_alloc:
2148         return ERR_PTR(err);
2149 }
2150
2151 static struct nft_hook *nft_hook_list_find(struct list_head *hook_list,
2152                                            const struct nft_hook *this)
2153 {
2154         struct nft_hook *hook;
2155
2156         list_for_each_entry(hook, hook_list, list) {
2157                 if (this->ops.dev == hook->ops.dev)
2158                         return hook;
2159         }
2160
2161         return NULL;
2162 }
2163
2164 static int nf_tables_parse_netdev_hooks(struct net *net,
2165                                         const struct nlattr *attr,
2166                                         struct list_head *hook_list,
2167                                         struct netlink_ext_ack *extack)
2168 {
2169         struct nft_hook *hook, *next;
2170         const struct nlattr *tmp;
2171         int rem, n = 0, err;
2172
2173         nla_for_each_nested(tmp, attr, rem) {
2174                 if (nla_type(tmp) != NFTA_DEVICE_NAME) {
2175                         err = -EINVAL;
2176                         goto err_hook;
2177                 }
2178
2179                 hook = nft_netdev_hook_alloc(net, tmp);
2180                 if (IS_ERR(hook)) {
2181                         NL_SET_BAD_ATTR(extack, tmp);
2182                         err = PTR_ERR(hook);
2183                         goto err_hook;
2184                 }
2185                 if (nft_hook_list_find(hook_list, hook)) {
2186                         NL_SET_BAD_ATTR(extack, tmp);
2187                         kfree(hook);
2188                         err = -EEXIST;
2189                         goto err_hook;
2190                 }
2191                 list_add_tail(&hook->list, hook_list);
2192                 n++;
2193
2194                 if (n == NFT_NETDEVICE_MAX) {
2195                         err = -EFBIG;
2196                         goto err_hook;
2197                 }
2198         }
2199
2200         return 0;
2201
2202 err_hook:
2203         list_for_each_entry_safe(hook, next, hook_list, list) {
2204                 list_del(&hook->list);
2205                 kfree(hook);
2206         }
2207         return err;
2208 }
2209
2210 struct nft_chain_hook {
2211         u32                             num;
2212         s32                             priority;
2213         const struct nft_chain_type     *type;
2214         struct list_head                list;
2215 };
2216
2217 static int nft_chain_parse_netdev(struct net *net, struct nlattr *tb[],
2218                                   struct list_head *hook_list,
2219                                   struct netlink_ext_ack *extack, u32 flags)
2220 {
2221         struct nft_hook *hook;
2222         int err;
2223
2224         if (tb[NFTA_HOOK_DEV]) {
2225                 hook = nft_netdev_hook_alloc(net, tb[NFTA_HOOK_DEV]);
2226                 if (IS_ERR(hook)) {
2227                         NL_SET_BAD_ATTR(extack, tb[NFTA_HOOK_DEV]);
2228                         return PTR_ERR(hook);
2229                 }
2230
2231                 list_add_tail(&hook->list, hook_list);
2232         } else if (tb[NFTA_HOOK_DEVS]) {
2233                 err = nf_tables_parse_netdev_hooks(net, tb[NFTA_HOOK_DEVS],
2234                                                    hook_list, extack);
2235                 if (err < 0)
2236                         return err;
2237
2238         }
2239
2240         if (flags & NFT_CHAIN_HW_OFFLOAD &&
2241             list_empty(hook_list))
2242                 return -EINVAL;
2243
2244         return 0;
2245 }
2246
2247 static int nft_chain_parse_hook(struct net *net,
2248                                 struct nft_base_chain *basechain,
2249                                 const struct nlattr * const nla[],
2250                                 struct nft_chain_hook *hook, u8 family,
2251                                 u32 flags, struct netlink_ext_ack *extack)
2252 {
2253         struct nftables_pernet *nft_net = nft_pernet(net);
2254         struct nlattr *ha[NFTA_HOOK_MAX + 1];
2255         const struct nft_chain_type *type;
2256         int err;
2257
2258         lockdep_assert_held(&nft_net->commit_mutex);
2259         lockdep_nfnl_nft_mutex_not_held();
2260
2261         err = nla_parse_nested_deprecated(ha, NFTA_HOOK_MAX,
2262                                           nla[NFTA_CHAIN_HOOK],
2263                                           nft_hook_policy, NULL);
2264         if (err < 0)
2265                 return err;
2266
2267         if (!basechain) {
2268                 if (!ha[NFTA_HOOK_HOOKNUM] ||
2269                     !ha[NFTA_HOOK_PRIORITY]) {
2270                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2271                         return -ENOENT;
2272                 }
2273
2274                 hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
2275                 hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
2276
2277                 type = __nft_chain_type_get(family, NFT_CHAIN_T_DEFAULT);
2278                 if (!type)
2279                         return -EOPNOTSUPP;
2280
2281                 if (nla[NFTA_CHAIN_TYPE]) {
2282                         type = nf_tables_chain_type_lookup(net, nla[NFTA_CHAIN_TYPE],
2283                                                            family, true);
2284                         if (IS_ERR(type)) {
2285                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2286                                 return PTR_ERR(type);
2287                         }
2288                 }
2289                 if (hook->num >= NFT_MAX_HOOKS || !(type->hook_mask & (1 << hook->num)))
2290                         return -EOPNOTSUPP;
2291
2292                 if (type->type == NFT_CHAIN_T_NAT &&
2293                     hook->priority <= NF_IP_PRI_CONNTRACK)
2294                         return -EOPNOTSUPP;
2295         } else {
2296                 if (ha[NFTA_HOOK_HOOKNUM]) {
2297                         hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
2298                         if (hook->num != basechain->ops.hooknum)
2299                                 return -EOPNOTSUPP;
2300                 }
2301                 if (ha[NFTA_HOOK_PRIORITY]) {
2302                         hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
2303                         if (hook->priority != basechain->ops.priority)
2304                                 return -EOPNOTSUPP;
2305                 }
2306
2307                 if (nla[NFTA_CHAIN_TYPE]) {
2308                         type = __nf_tables_chain_type_lookup(nla[NFTA_CHAIN_TYPE],
2309                                                              family);
2310                         if (!type) {
2311                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2312                                 return -ENOENT;
2313                         }
2314                 } else {
2315                         type = basechain->type;
2316                 }
2317         }
2318
2319         if (!try_module_get(type->owner)) {
2320                 if (nla[NFTA_CHAIN_TYPE])
2321                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2322                 return -ENOENT;
2323         }
2324
2325         hook->type = type;
2326
2327         INIT_LIST_HEAD(&hook->list);
2328         if (nft_base_chain_netdev(family, hook->num)) {
2329                 err = nft_chain_parse_netdev(net, ha, &hook->list, extack, flags);
2330                 if (err < 0) {
2331                         module_put(type->owner);
2332                         return err;
2333                 }
2334         } else if (ha[NFTA_HOOK_DEV] || ha[NFTA_HOOK_DEVS]) {
2335                 module_put(type->owner);
2336                 return -EOPNOTSUPP;
2337         }
2338
2339         return 0;
2340 }
2341
2342 static void nft_chain_release_hook(struct nft_chain_hook *hook)
2343 {
2344         struct nft_hook *h, *next;
2345
2346         list_for_each_entry_safe(h, next, &hook->list, list) {
2347                 list_del(&h->list);
2348                 kfree(h);
2349         }
2350         module_put(hook->type->owner);
2351 }
2352
2353 static void nft_last_rule(const struct nft_chain *chain, const void *ptr)
2354 {
2355         struct nft_rule_dp_last *lrule;
2356
2357         BUILD_BUG_ON(offsetof(struct nft_rule_dp_last, end) != 0);
2358
2359         lrule = (struct nft_rule_dp_last *)ptr;
2360         lrule->end.is_last = 1;
2361         lrule->chain = chain;
2362         /* blob size does not include the trailer rule */
2363 }
2364
2365 static struct nft_rule_blob *nf_tables_chain_alloc_rules(const struct nft_chain *chain,
2366                                                          unsigned int size)
2367 {
2368         struct nft_rule_blob *blob;
2369
2370         if (size > INT_MAX)
2371                 return NULL;
2372
2373         size += sizeof(struct nft_rule_blob) + sizeof(struct nft_rule_dp_last);
2374
2375         blob = kvmalloc(size, GFP_KERNEL_ACCOUNT);
2376         if (!blob)
2377                 return NULL;
2378
2379         blob->size = 0;
2380         nft_last_rule(chain, blob->data);
2381
2382         return blob;
2383 }
2384
2385 static void nft_basechain_hook_init(struct nf_hook_ops *ops, u8 family,
2386                                     const struct nft_chain_hook *hook,
2387                                     struct nft_chain *chain)
2388 {
2389         ops->pf                 = family;
2390         ops->hooknum            = hook->num;
2391         ops->priority           = hook->priority;
2392         ops->priv               = chain;
2393         ops->hook               = hook->type->hooks[ops->hooknum];
2394         ops->hook_ops_type      = NF_HOOK_OP_NF_TABLES;
2395 }
2396
2397 static int nft_basechain_init(struct nft_base_chain *basechain, u8 family,
2398                               struct nft_chain_hook *hook, u32 flags)
2399 {
2400         struct nft_chain *chain;
2401         struct nft_hook *h;
2402
2403         basechain->type = hook->type;
2404         INIT_LIST_HEAD(&basechain->hook_list);
2405         chain = &basechain->chain;
2406
2407         if (nft_base_chain_netdev(family, hook->num)) {
2408                 list_splice_init(&hook->list, &basechain->hook_list);
2409                 list_for_each_entry(h, &basechain->hook_list, list)
2410                         nft_basechain_hook_init(&h->ops, family, hook, chain);
2411         }
2412         nft_basechain_hook_init(&basechain->ops, family, hook, chain);
2413
2414         chain->flags |= NFT_CHAIN_BASE | flags;
2415         basechain->policy = NF_ACCEPT;
2416         if (chain->flags & NFT_CHAIN_HW_OFFLOAD &&
2417             !nft_chain_offload_support(basechain)) {
2418                 list_splice_init(&basechain->hook_list, &hook->list);
2419                 return -EOPNOTSUPP;
2420         }
2421
2422         flow_block_init(&basechain->flow_block);
2423
2424         return 0;
2425 }
2426
2427 int nft_chain_add(struct nft_table *table, struct nft_chain *chain)
2428 {
2429         int err;
2430
2431         err = rhltable_insert_key(&table->chains_ht, chain->name,
2432                                   &chain->rhlhead, nft_chain_ht_params);
2433         if (err)
2434                 return err;
2435
2436         list_add_tail_rcu(&chain->list, &table->chains);
2437
2438         return 0;
2439 }
2440
2441 static u64 chain_id;
2442
2443 static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 genmask,
2444                               u8 policy, u32 flags,
2445                               struct netlink_ext_ack *extack)
2446 {
2447         const struct nlattr * const *nla = ctx->nla;
2448         struct nft_table *table = ctx->table;
2449         struct nft_base_chain *basechain;
2450         struct net *net = ctx->net;
2451         char name[NFT_NAME_MAXLEN];
2452         struct nft_rule_blob *blob;
2453         struct nft_trans *trans;
2454         struct nft_chain *chain;
2455         int err;
2456
2457         if (nla[NFTA_CHAIN_HOOK]) {
2458                 struct nft_stats __percpu *stats = NULL;
2459                 struct nft_chain_hook hook = {};
2460
2461                 if (table->flags & __NFT_TABLE_F_UPDATE)
2462                         return -EINVAL;
2463
2464                 if (flags & NFT_CHAIN_BINDING)
2465                         return -EOPNOTSUPP;
2466
2467                 err = nft_chain_parse_hook(net, NULL, nla, &hook, family, flags,
2468                                            extack);
2469                 if (err < 0)
2470                         return err;
2471
2472                 basechain = kzalloc(sizeof(*basechain), GFP_KERNEL_ACCOUNT);
2473                 if (basechain == NULL) {
2474                         nft_chain_release_hook(&hook);
2475                         return -ENOMEM;
2476                 }
2477                 chain = &basechain->chain;
2478
2479                 if (nla[NFTA_CHAIN_COUNTERS]) {
2480                         stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2481                         if (IS_ERR(stats)) {
2482                                 nft_chain_release_hook(&hook);
2483                                 kfree(basechain);
2484                                 return PTR_ERR(stats);
2485                         }
2486                         rcu_assign_pointer(basechain->stats, stats);
2487                 }
2488
2489                 err = nft_basechain_init(basechain, family, &hook, flags);
2490                 if (err < 0) {
2491                         nft_chain_release_hook(&hook);
2492                         kfree(basechain);
2493                         free_percpu(stats);
2494                         return err;
2495                 }
2496                 if (stats)
2497                         static_branch_inc(&nft_counters_enabled);
2498         } else {
2499                 if (flags & NFT_CHAIN_BASE)
2500                         return -EINVAL;
2501                 if (flags & NFT_CHAIN_HW_OFFLOAD)
2502                         return -EOPNOTSUPP;
2503
2504                 chain = kzalloc(sizeof(*chain), GFP_KERNEL_ACCOUNT);
2505                 if (chain == NULL)
2506                         return -ENOMEM;
2507
2508                 chain->flags = flags;
2509         }
2510         ctx->chain = chain;
2511
2512         INIT_LIST_HEAD(&chain->rules);
2513         chain->handle = nf_tables_alloc_handle(table);
2514         chain->table = table;
2515
2516         if (nla[NFTA_CHAIN_NAME]) {
2517                 chain->name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
2518         } else {
2519                 if (!(flags & NFT_CHAIN_BINDING)) {
2520                         err = -EINVAL;
2521                         goto err_destroy_chain;
2522                 }
2523
2524                 snprintf(name, sizeof(name), "__chain%llu", ++chain_id);
2525                 chain->name = kstrdup(name, GFP_KERNEL_ACCOUNT);
2526         }
2527
2528         if (!chain->name) {
2529                 err = -ENOMEM;
2530                 goto err_destroy_chain;
2531         }
2532
2533         if (nla[NFTA_CHAIN_USERDATA]) {
2534                 chain->udata = nla_memdup(nla[NFTA_CHAIN_USERDATA], GFP_KERNEL_ACCOUNT);
2535                 if (chain->udata == NULL) {
2536                         err = -ENOMEM;
2537                         goto err_destroy_chain;
2538                 }
2539                 chain->udlen = nla_len(nla[NFTA_CHAIN_USERDATA]);
2540         }
2541
2542         blob = nf_tables_chain_alloc_rules(chain, 0);
2543         if (!blob) {
2544                 err = -ENOMEM;
2545                 goto err_destroy_chain;
2546         }
2547
2548         RCU_INIT_POINTER(chain->blob_gen_0, blob);
2549         RCU_INIT_POINTER(chain->blob_gen_1, blob);
2550
2551         if (!nft_use_inc(&table->use)) {
2552                 err = -EMFILE;
2553                 goto err_destroy_chain;
2554         }
2555
2556         trans = nft_trans_chain_add(ctx, NFT_MSG_NEWCHAIN);
2557         if (IS_ERR(trans)) {
2558                 err = PTR_ERR(trans);
2559                 goto err_trans;
2560         }
2561
2562         nft_trans_chain_policy(trans) = NFT_CHAIN_POLICY_UNSET;
2563         if (nft_is_base_chain(chain))
2564                 nft_trans_chain_policy(trans) = policy;
2565
2566         err = nft_chain_add(table, chain);
2567         if (err < 0)
2568                 goto err_chain_add;
2569
2570         /* This must be LAST to ensure no packets are walking over this chain. */
2571         err = nf_tables_register_hook(net, table, chain);
2572         if (err < 0)
2573                 goto err_register_hook;
2574
2575         return 0;
2576
2577 err_register_hook:
2578         nft_chain_del(chain);
2579 err_chain_add:
2580         nft_trans_destroy(trans);
2581 err_trans:
2582         nft_use_dec_restore(&table->use);
2583 err_destroy_chain:
2584         nf_tables_chain_destroy(ctx);
2585
2586         return err;
2587 }
2588
2589 static int nf_tables_updchain(struct nft_ctx *ctx, u8 genmask, u8 policy,
2590                               u32 flags, const struct nlattr *attr,
2591                               struct netlink_ext_ack *extack)
2592 {
2593         const struct nlattr * const *nla = ctx->nla;
2594         struct nft_base_chain *basechain = NULL;
2595         struct nft_table *table = ctx->table;
2596         struct nft_chain *chain = ctx->chain;
2597         struct nft_chain_hook hook = {};
2598         struct nft_stats *stats = NULL;
2599         struct nft_hook *h, *next;
2600         struct nf_hook_ops *ops;
2601         struct nft_trans *trans;
2602         bool unregister = false;
2603         int err;
2604
2605         if (chain->flags ^ flags)
2606                 return -EOPNOTSUPP;
2607
2608         INIT_LIST_HEAD(&hook.list);
2609
2610         if (nla[NFTA_CHAIN_HOOK]) {
2611                 if (!nft_is_base_chain(chain)) {
2612                         NL_SET_BAD_ATTR(extack, attr);
2613                         return -EEXIST;
2614                 }
2615
2616                 basechain = nft_base_chain(chain);
2617                 err = nft_chain_parse_hook(ctx->net, basechain, nla, &hook,
2618                                            ctx->family, flags, extack);
2619                 if (err < 0)
2620                         return err;
2621
2622                 if (basechain->type != hook.type) {
2623                         nft_chain_release_hook(&hook);
2624                         NL_SET_BAD_ATTR(extack, attr);
2625                         return -EEXIST;
2626                 }
2627
2628                 if (nft_base_chain_netdev(ctx->family, basechain->ops.hooknum)) {
2629                         list_for_each_entry_safe(h, next, &hook.list, list) {
2630                                 h->ops.pf       = basechain->ops.pf;
2631                                 h->ops.hooknum  = basechain->ops.hooknum;
2632                                 h->ops.priority = basechain->ops.priority;
2633                                 h->ops.priv     = basechain->ops.priv;
2634                                 h->ops.hook     = basechain->ops.hook;
2635
2636                                 if (nft_hook_list_find(&basechain->hook_list, h)) {
2637                                         list_del(&h->list);
2638                                         kfree(h);
2639                                 }
2640                         }
2641                 } else {
2642                         ops = &basechain->ops;
2643                         if (ops->hooknum != hook.num ||
2644                             ops->priority != hook.priority) {
2645                                 nft_chain_release_hook(&hook);
2646                                 NL_SET_BAD_ATTR(extack, attr);
2647                                 return -EEXIST;
2648                         }
2649                 }
2650         }
2651
2652         if (nla[NFTA_CHAIN_HANDLE] &&
2653             nla[NFTA_CHAIN_NAME]) {
2654                 struct nft_chain *chain2;
2655
2656                 chain2 = nft_chain_lookup(ctx->net, table,
2657                                           nla[NFTA_CHAIN_NAME], genmask);
2658                 if (!IS_ERR(chain2)) {
2659                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2660                         err = -EEXIST;
2661                         goto err_hooks;
2662                 }
2663         }
2664
2665         if (table->flags & __NFT_TABLE_F_UPDATE &&
2666             !list_empty(&hook.list)) {
2667                 NL_SET_BAD_ATTR(extack, attr);
2668                 err = -EOPNOTSUPP;
2669                 goto err_hooks;
2670         }
2671
2672         if (!(table->flags & NFT_TABLE_F_DORMANT) &&
2673             nft_is_base_chain(chain) &&
2674             !list_empty(&hook.list)) {
2675                 basechain = nft_base_chain(chain);
2676                 ops = &basechain->ops;
2677
2678                 if (nft_base_chain_netdev(table->family, basechain->ops.hooknum)) {
2679                         err = nft_netdev_register_hooks(ctx->net, &hook.list);
2680                         if (err < 0)
2681                                 goto err_hooks;
2682                 }
2683         }
2684
2685         unregister = true;
2686
2687         if (nla[NFTA_CHAIN_COUNTERS]) {
2688                 if (!nft_is_base_chain(chain)) {
2689                         err = -EOPNOTSUPP;
2690                         goto err_hooks;
2691                 }
2692
2693                 stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2694                 if (IS_ERR(stats)) {
2695                         err = PTR_ERR(stats);
2696                         goto err_hooks;
2697                 }
2698         }
2699
2700         err = -ENOMEM;
2701         trans = nft_trans_alloc(ctx, NFT_MSG_NEWCHAIN,
2702                                 sizeof(struct nft_trans_chain));
2703         if (trans == NULL)
2704                 goto err_trans;
2705
2706         nft_trans_chain_stats(trans) = stats;
2707         nft_trans_chain_update(trans) = true;
2708
2709         if (nla[NFTA_CHAIN_POLICY])
2710                 nft_trans_chain_policy(trans) = policy;
2711         else
2712                 nft_trans_chain_policy(trans) = -1;
2713
2714         if (nla[NFTA_CHAIN_HANDLE] &&
2715             nla[NFTA_CHAIN_NAME]) {
2716                 struct nftables_pernet *nft_net = nft_pernet(ctx->net);
2717                 struct nft_trans *tmp;
2718                 char *name;
2719
2720                 err = -ENOMEM;
2721                 name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
2722                 if (!name)
2723                         goto err_trans;
2724
2725                 err = -EEXIST;
2726                 list_for_each_entry(tmp, &nft_net->commit_list, list) {
2727                         if (tmp->msg_type == NFT_MSG_NEWCHAIN &&
2728                             tmp->ctx.table == table &&
2729                             nft_trans_chain_update(tmp) &&
2730                             nft_trans_chain_name(tmp) &&
2731                             strcmp(name, nft_trans_chain_name(tmp)) == 0) {
2732                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2733                                 kfree(name);
2734                                 goto err_trans;
2735                         }
2736                 }
2737
2738                 nft_trans_chain_name(trans) = name;
2739         }
2740
2741         nft_trans_basechain(trans) = basechain;
2742         INIT_LIST_HEAD(&nft_trans_chain_hooks(trans));
2743         list_splice(&hook.list, &nft_trans_chain_hooks(trans));
2744         if (nla[NFTA_CHAIN_HOOK])
2745                 module_put(hook.type->owner);
2746
2747         nft_trans_commit_list_add_tail(ctx->net, trans);
2748
2749         return 0;
2750
2751 err_trans:
2752         free_percpu(stats);
2753         kfree(trans);
2754 err_hooks:
2755         if (nla[NFTA_CHAIN_HOOK]) {
2756                 list_for_each_entry_safe(h, next, &hook.list, list) {
2757                         if (unregister)
2758                                 nf_unregister_net_hook(ctx->net, &h->ops);
2759                         list_del(&h->list);
2760                         kfree_rcu(h, rcu);
2761                 }
2762                 module_put(hook.type->owner);
2763         }
2764
2765         return err;
2766 }
2767
2768 static struct nft_chain *nft_chain_lookup_byid(const struct net *net,
2769                                                const struct nft_table *table,
2770                                                const struct nlattr *nla, u8 genmask)
2771 {
2772         struct nftables_pernet *nft_net = nft_pernet(net);
2773         u32 id = ntohl(nla_get_be32(nla));
2774         struct nft_trans *trans;
2775
2776         list_for_each_entry(trans, &nft_net->commit_list, list) {
2777                 struct nft_chain *chain = trans->ctx.chain;
2778
2779                 if (trans->msg_type == NFT_MSG_NEWCHAIN &&
2780                     chain->table == table &&
2781                     id == nft_trans_chain_id(trans) &&
2782                     nft_active_genmask(chain, genmask))
2783                         return chain;
2784         }
2785         return ERR_PTR(-ENOENT);
2786 }
2787
2788 static int nf_tables_newchain(struct sk_buff *skb, const struct nfnl_info *info,
2789                               const struct nlattr * const nla[])
2790 {
2791         struct nftables_pernet *nft_net = nft_pernet(info->net);
2792         struct netlink_ext_ack *extack = info->extack;
2793         u8 genmask = nft_genmask_next(info->net);
2794         u8 family = info->nfmsg->nfgen_family;
2795         struct nft_chain *chain = NULL;
2796         struct net *net = info->net;
2797         const struct nlattr *attr;
2798         struct nft_table *table;
2799         u8 policy = NF_ACCEPT;
2800         struct nft_ctx ctx;
2801         u64 handle = 0;
2802         u32 flags = 0;
2803
2804         lockdep_assert_held(&nft_net->commit_mutex);
2805
2806         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
2807                                  NETLINK_CB(skb).portid);
2808         if (IS_ERR(table)) {
2809                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2810                 return PTR_ERR(table);
2811         }
2812
2813         chain = NULL;
2814         attr = nla[NFTA_CHAIN_NAME];
2815
2816         if (nla[NFTA_CHAIN_HANDLE]) {
2817                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_CHAIN_HANDLE]));
2818                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2819                 if (IS_ERR(chain)) {
2820                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_HANDLE]);
2821                         return PTR_ERR(chain);
2822                 }
2823                 attr = nla[NFTA_CHAIN_HANDLE];
2824         } else if (nla[NFTA_CHAIN_NAME]) {
2825                 chain = nft_chain_lookup(net, table, attr, genmask);
2826                 if (IS_ERR(chain)) {
2827                         if (PTR_ERR(chain) != -ENOENT) {
2828                                 NL_SET_BAD_ATTR(extack, attr);
2829                                 return PTR_ERR(chain);
2830                         }
2831                         chain = NULL;
2832                 }
2833         } else if (!nla[NFTA_CHAIN_ID]) {
2834                 return -EINVAL;
2835         }
2836
2837         if (nla[NFTA_CHAIN_POLICY]) {
2838                 if (chain != NULL &&
2839                     !nft_is_base_chain(chain)) {
2840                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2841                         return -EOPNOTSUPP;
2842                 }
2843
2844                 if (chain == NULL &&
2845                     nla[NFTA_CHAIN_HOOK] == NULL) {
2846                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2847                         return -EOPNOTSUPP;
2848                 }
2849
2850                 policy = ntohl(nla_get_be32(nla[NFTA_CHAIN_POLICY]));
2851                 switch (policy) {
2852                 case NF_DROP:
2853                 case NF_ACCEPT:
2854                         break;
2855                 default:
2856                         return -EINVAL;
2857                 }
2858         }
2859
2860         if (nla[NFTA_CHAIN_FLAGS])
2861                 flags = ntohl(nla_get_be32(nla[NFTA_CHAIN_FLAGS]));
2862         else if (chain)
2863                 flags = chain->flags;
2864
2865         if (flags & ~NFT_CHAIN_FLAGS)
2866                 return -EOPNOTSUPP;
2867
2868         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
2869
2870         if (chain != NULL) {
2871                 if (chain->flags & NFT_CHAIN_BINDING)
2872                         return -EINVAL;
2873
2874                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
2875                         NL_SET_BAD_ATTR(extack, attr);
2876                         return -EEXIST;
2877                 }
2878                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
2879                         return -EOPNOTSUPP;
2880
2881                 flags |= chain->flags & NFT_CHAIN_BASE;
2882                 return nf_tables_updchain(&ctx, genmask, policy, flags, attr,
2883                                           extack);
2884         }
2885
2886         return nf_tables_addchain(&ctx, family, genmask, policy, flags, extack);
2887 }
2888
2889 static int nft_delchain_hook(struct nft_ctx *ctx,
2890                              struct nft_base_chain *basechain,
2891                              struct netlink_ext_ack *extack)
2892 {
2893         const struct nft_chain *chain = &basechain->chain;
2894         const struct nlattr * const *nla = ctx->nla;
2895         struct nft_chain_hook chain_hook = {};
2896         struct nft_hook *this, *hook;
2897         LIST_HEAD(chain_del_list);
2898         struct nft_trans *trans;
2899         int err;
2900
2901         if (ctx->table->flags & __NFT_TABLE_F_UPDATE)
2902                 return -EOPNOTSUPP;
2903
2904         err = nft_chain_parse_hook(ctx->net, basechain, nla, &chain_hook,
2905                                    ctx->family, chain->flags, extack);
2906         if (err < 0)
2907                 return err;
2908
2909         list_for_each_entry(this, &chain_hook.list, list) {
2910                 hook = nft_hook_list_find(&basechain->hook_list, this);
2911                 if (!hook) {
2912                         err = -ENOENT;
2913                         goto err_chain_del_hook;
2914                 }
2915                 list_move(&hook->list, &chain_del_list);
2916         }
2917
2918         trans = nft_trans_alloc(ctx, NFT_MSG_DELCHAIN,
2919                                 sizeof(struct nft_trans_chain));
2920         if (!trans) {
2921                 err = -ENOMEM;
2922                 goto err_chain_del_hook;
2923         }
2924
2925         nft_trans_basechain(trans) = basechain;
2926         nft_trans_chain_update(trans) = true;
2927         INIT_LIST_HEAD(&nft_trans_chain_hooks(trans));
2928         list_splice(&chain_del_list, &nft_trans_chain_hooks(trans));
2929         nft_chain_release_hook(&chain_hook);
2930
2931         nft_trans_commit_list_add_tail(ctx->net, trans);
2932
2933         return 0;
2934
2935 err_chain_del_hook:
2936         list_splice(&chain_del_list, &basechain->hook_list);
2937         nft_chain_release_hook(&chain_hook);
2938
2939         return err;
2940 }
2941
2942 static int nf_tables_delchain(struct sk_buff *skb, const struct nfnl_info *info,
2943                               const struct nlattr * const nla[])
2944 {
2945         struct netlink_ext_ack *extack = info->extack;
2946         u8 genmask = nft_genmask_next(info->net);
2947         u8 family = info->nfmsg->nfgen_family;
2948         struct net *net = info->net;
2949         const struct nlattr *attr;
2950         struct nft_table *table;
2951         struct nft_chain *chain;
2952         struct nft_rule *rule;
2953         struct nft_ctx ctx;
2954         u64 handle;
2955         u32 use;
2956         int err;
2957
2958         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
2959                                  NETLINK_CB(skb).portid);
2960         if (IS_ERR(table)) {
2961                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2962                 return PTR_ERR(table);
2963         }
2964
2965         if (nla[NFTA_CHAIN_HANDLE]) {
2966                 attr = nla[NFTA_CHAIN_HANDLE];
2967                 handle = be64_to_cpu(nla_get_be64(attr));
2968                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2969         } else {
2970                 attr = nla[NFTA_CHAIN_NAME];
2971                 chain = nft_chain_lookup(net, table, attr, genmask);
2972         }
2973         if (IS_ERR(chain)) {
2974                 if (PTR_ERR(chain) == -ENOENT &&
2975                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYCHAIN)
2976                         return 0;
2977
2978                 NL_SET_BAD_ATTR(extack, attr);
2979                 return PTR_ERR(chain);
2980         }
2981
2982         if (nft_chain_binding(chain))
2983                 return -EOPNOTSUPP;
2984
2985         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
2986
2987         if (nla[NFTA_CHAIN_HOOK]) {
2988                 if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYCHAIN ||
2989                     chain->flags & NFT_CHAIN_HW_OFFLOAD)
2990                         return -EOPNOTSUPP;
2991
2992                 if (nft_is_base_chain(chain)) {
2993                         struct nft_base_chain *basechain = nft_base_chain(chain);
2994
2995                         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
2996                                 return nft_delchain_hook(&ctx, basechain, extack);
2997                 }
2998         }
2999
3000         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
3001             chain->use > 0)
3002                 return -EBUSY;
3003
3004         use = chain->use;
3005         list_for_each_entry(rule, &chain->rules, list) {
3006                 if (!nft_is_active_next(net, rule))
3007                         continue;
3008                 use--;
3009
3010                 err = nft_delrule(&ctx, rule);
3011                 if (err < 0)
3012                         return err;
3013         }
3014
3015         /* There are rules and elements that are still holding references to us,
3016          * we cannot do a recursive removal in this case.
3017          */
3018         if (use > 0) {
3019                 NL_SET_BAD_ATTR(extack, attr);
3020                 return -EBUSY;
3021         }
3022
3023         return nft_delchain(&ctx);
3024 }
3025
3026 /*
3027  * Expressions
3028  */
3029
3030 /**
3031  *      nft_register_expr - register nf_tables expr type
3032  *      @type: expr type
3033  *
3034  *      Registers the expr type for use with nf_tables. Returns zero on
3035  *      success or a negative errno code otherwise.
3036  */
3037 int nft_register_expr(struct nft_expr_type *type)
3038 {
3039         if (WARN_ON_ONCE(type->maxattr > NFT_EXPR_MAXATTR))
3040                 return -ENOMEM;
3041
3042         nfnl_lock(NFNL_SUBSYS_NFTABLES);
3043         if (type->family == NFPROTO_UNSPEC)
3044                 list_add_tail_rcu(&type->list, &nf_tables_expressions);
3045         else
3046                 list_add_rcu(&type->list, &nf_tables_expressions);
3047         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
3048         return 0;
3049 }
3050 EXPORT_SYMBOL_GPL(nft_register_expr);
3051
3052 /**
3053  *      nft_unregister_expr - unregister nf_tables expr type
3054  *      @type: expr type
3055  *
3056  *      Unregisters the expr typefor use with nf_tables.
3057  */
3058 void nft_unregister_expr(struct nft_expr_type *type)
3059 {
3060         nfnl_lock(NFNL_SUBSYS_NFTABLES);
3061         list_del_rcu(&type->list);
3062         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
3063 }
3064 EXPORT_SYMBOL_GPL(nft_unregister_expr);
3065
3066 static const struct nft_expr_type *__nft_expr_type_get(u8 family,
3067                                                        struct nlattr *nla)
3068 {
3069         const struct nft_expr_type *type, *candidate = NULL;
3070
3071         list_for_each_entry_rcu(type, &nf_tables_expressions, list) {
3072                 if (!nla_strcmp(nla, type->name)) {
3073                         if (!type->family && !candidate)
3074                                 candidate = type;
3075                         else if (type->family == family)
3076                                 candidate = type;
3077                 }
3078         }
3079         return candidate;
3080 }
3081
3082 #ifdef CONFIG_MODULES
3083 static int nft_expr_type_request_module(struct net *net, u8 family,
3084                                         struct nlattr *nla)
3085 {
3086         if (nft_request_module(net, "nft-expr-%u-%.*s", family,
3087                                nla_len(nla), (char *)nla_data(nla)) == -EAGAIN)
3088                 return -EAGAIN;
3089
3090         return 0;
3091 }
3092 #endif
3093
3094 static const struct nft_expr_type *nft_expr_type_get(struct net *net,
3095                                                      u8 family,
3096                                                      struct nlattr *nla)
3097 {
3098         const struct nft_expr_type *type;
3099
3100         if (nla == NULL)
3101                 return ERR_PTR(-EINVAL);
3102
3103         rcu_read_lock();
3104         type = __nft_expr_type_get(family, nla);
3105         if (type != NULL && try_module_get(type->owner)) {
3106                 rcu_read_unlock();
3107                 return type;
3108         }
3109         rcu_read_unlock();
3110
3111         lockdep_nfnl_nft_mutex_not_held();
3112 #ifdef CONFIG_MODULES
3113         if (type == NULL) {
3114                 if (nft_expr_type_request_module(net, family, nla) == -EAGAIN)
3115                         return ERR_PTR(-EAGAIN);
3116
3117                 if (nft_request_module(net, "nft-expr-%.*s",
3118                                        nla_len(nla),
3119                                        (char *)nla_data(nla)) == -EAGAIN)
3120                         return ERR_PTR(-EAGAIN);
3121         }
3122 #endif
3123         return ERR_PTR(-ENOENT);
3124 }
3125
3126 static const struct nla_policy nft_expr_policy[NFTA_EXPR_MAX + 1] = {
3127         [NFTA_EXPR_NAME]        = { .type = NLA_STRING,
3128                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
3129         [NFTA_EXPR_DATA]        = { .type = NLA_NESTED },
3130 };
3131
3132 static int nf_tables_fill_expr_info(struct sk_buff *skb,
3133                                     const struct nft_expr *expr, bool reset)
3134 {
3135         if (nla_put_string(skb, NFTA_EXPR_NAME, expr->ops->type->name))
3136                 goto nla_put_failure;
3137
3138         if (expr->ops->dump) {
3139                 struct nlattr *data = nla_nest_start_noflag(skb,
3140                                                             NFTA_EXPR_DATA);
3141                 if (data == NULL)
3142                         goto nla_put_failure;
3143                 if (expr->ops->dump(skb, expr, reset) < 0)
3144                         goto nla_put_failure;
3145                 nla_nest_end(skb, data);
3146         }
3147
3148         return skb->len;
3149
3150 nla_put_failure:
3151         return -1;
3152 };
3153
3154 int nft_expr_dump(struct sk_buff *skb, unsigned int attr,
3155                   const struct nft_expr *expr, bool reset)
3156 {
3157         struct nlattr *nest;
3158
3159         nest = nla_nest_start_noflag(skb, attr);
3160         if (!nest)
3161                 goto nla_put_failure;
3162         if (nf_tables_fill_expr_info(skb, expr, reset) < 0)
3163                 goto nla_put_failure;
3164         nla_nest_end(skb, nest);
3165         return 0;
3166
3167 nla_put_failure:
3168         return -1;
3169 }
3170
3171 struct nft_expr_info {
3172         const struct nft_expr_ops       *ops;
3173         const struct nlattr             *attr;
3174         struct nlattr                   *tb[NFT_EXPR_MAXATTR + 1];
3175 };
3176
3177 static int nf_tables_expr_parse(const struct nft_ctx *ctx,
3178                                 const struct nlattr *nla,
3179                                 struct nft_expr_info *info)
3180 {
3181         const struct nft_expr_type *type;
3182         const struct nft_expr_ops *ops;
3183         struct nlattr *tb[NFTA_EXPR_MAX + 1];
3184         int err;
3185
3186         err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
3187                                           nft_expr_policy, NULL);
3188         if (err < 0)
3189                 return err;
3190
3191         type = nft_expr_type_get(ctx->net, ctx->family, tb[NFTA_EXPR_NAME]);
3192         if (IS_ERR(type))
3193                 return PTR_ERR(type);
3194
3195         if (tb[NFTA_EXPR_DATA]) {
3196                 err = nla_parse_nested_deprecated(info->tb, type->maxattr,
3197                                                   tb[NFTA_EXPR_DATA],
3198                                                   type->policy, NULL);
3199                 if (err < 0)
3200                         goto err1;
3201         } else
3202                 memset(info->tb, 0, sizeof(info->tb[0]) * (type->maxattr + 1));
3203
3204         if (type->select_ops != NULL) {
3205                 ops = type->select_ops(ctx,
3206                                        (const struct nlattr * const *)info->tb);
3207                 if (IS_ERR(ops)) {
3208                         err = PTR_ERR(ops);
3209 #ifdef CONFIG_MODULES
3210                         if (err == -EAGAIN)
3211                                 if (nft_expr_type_request_module(ctx->net,
3212                                                                  ctx->family,
3213                                                                  tb[NFTA_EXPR_NAME]) != -EAGAIN)
3214                                         err = -ENOENT;
3215 #endif
3216                         goto err1;
3217                 }
3218         } else
3219                 ops = type->ops;
3220
3221         info->attr = nla;
3222         info->ops = ops;
3223
3224         return 0;
3225
3226 err1:
3227         module_put(type->owner);
3228         return err;
3229 }
3230
3231 int nft_expr_inner_parse(const struct nft_ctx *ctx, const struct nlattr *nla,
3232                          struct nft_expr_info *info)
3233 {
3234         struct nlattr *tb[NFTA_EXPR_MAX + 1];
3235         const struct nft_expr_type *type;
3236         int err;
3237
3238         err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
3239                                           nft_expr_policy, NULL);
3240         if (err < 0)
3241                 return err;
3242
3243         if (!tb[NFTA_EXPR_DATA] || !tb[NFTA_EXPR_NAME])
3244                 return -EINVAL;
3245
3246         type = __nft_expr_type_get(ctx->family, tb[NFTA_EXPR_NAME]);
3247         if (!type)
3248                 return -ENOENT;
3249
3250         if (!type->inner_ops)
3251                 return -EOPNOTSUPP;
3252
3253         err = nla_parse_nested_deprecated(info->tb, type->maxattr,
3254                                           tb[NFTA_EXPR_DATA],
3255                                           type->policy, NULL);
3256         if (err < 0)
3257                 goto err_nla_parse;
3258
3259         info->attr = nla;
3260         info->ops = type->inner_ops;
3261
3262         return 0;
3263
3264 err_nla_parse:
3265         return err;
3266 }
3267
3268 static int nf_tables_newexpr(const struct nft_ctx *ctx,
3269                              const struct nft_expr_info *expr_info,
3270                              struct nft_expr *expr)
3271 {
3272         const struct nft_expr_ops *ops = expr_info->ops;
3273         int err;
3274
3275         expr->ops = ops;
3276         if (ops->init) {
3277                 err = ops->init(ctx, expr, (const struct nlattr **)expr_info->tb);
3278                 if (err < 0)
3279                         goto err1;
3280         }
3281
3282         return 0;
3283 err1:
3284         expr->ops = NULL;
3285         return err;
3286 }
3287
3288 static void nf_tables_expr_destroy(const struct nft_ctx *ctx,
3289                                    struct nft_expr *expr)
3290 {
3291         const struct nft_expr_type *type = expr->ops->type;
3292
3293         if (expr->ops->destroy)
3294                 expr->ops->destroy(ctx, expr);
3295         module_put(type->owner);
3296 }
3297
3298 static struct nft_expr *nft_expr_init(const struct nft_ctx *ctx,
3299                                       const struct nlattr *nla)
3300 {
3301         struct nft_expr_info expr_info;
3302         struct nft_expr *expr;
3303         struct module *owner;
3304         int err;
3305
3306         err = nf_tables_expr_parse(ctx, nla, &expr_info);
3307         if (err < 0)
3308                 goto err_expr_parse;
3309
3310         err = -EOPNOTSUPP;
3311         if (!(expr_info.ops->type->flags & NFT_EXPR_STATEFUL))
3312                 goto err_expr_stateful;
3313
3314         err = -ENOMEM;
3315         expr = kzalloc(expr_info.ops->size, GFP_KERNEL_ACCOUNT);
3316         if (expr == NULL)
3317                 goto err_expr_stateful;
3318
3319         err = nf_tables_newexpr(ctx, &expr_info, expr);
3320         if (err < 0)
3321                 goto err_expr_new;
3322
3323         return expr;
3324 err_expr_new:
3325         kfree(expr);
3326 err_expr_stateful:
3327         owner = expr_info.ops->type->owner;
3328         if (expr_info.ops->type->release_ops)
3329                 expr_info.ops->type->release_ops(expr_info.ops);
3330
3331         module_put(owner);
3332 err_expr_parse:
3333         return ERR_PTR(err);
3334 }
3335
3336 int nft_expr_clone(struct nft_expr *dst, struct nft_expr *src, gfp_t gfp)
3337 {
3338         int err;
3339
3340         if (WARN_ON_ONCE(!src->ops->clone))
3341                 return -EINVAL;
3342
3343         dst->ops = src->ops;
3344         err = src->ops->clone(dst, src, gfp);
3345         if (err < 0)
3346                 return err;
3347
3348         __module_get(src->ops->type->owner);
3349
3350         return 0;
3351 }
3352
3353 void nft_expr_destroy(const struct nft_ctx *ctx, struct nft_expr *expr)
3354 {
3355         nf_tables_expr_destroy(ctx, expr);
3356         kfree(expr);
3357 }
3358
3359 /*
3360  * Rules
3361  */
3362
3363 static struct nft_rule *__nft_rule_lookup(const struct nft_chain *chain,
3364                                           u64 handle)
3365 {
3366         struct nft_rule *rule;
3367
3368         // FIXME: this sucks
3369         list_for_each_entry_rcu(rule, &chain->rules, list) {
3370                 if (handle == rule->handle)
3371                         return rule;
3372         }
3373
3374         return ERR_PTR(-ENOENT);
3375 }
3376
3377 static struct nft_rule *nft_rule_lookup(const struct nft_chain *chain,
3378                                         const struct nlattr *nla)
3379 {
3380         if (nla == NULL)
3381                 return ERR_PTR(-EINVAL);
3382
3383         return __nft_rule_lookup(chain, be64_to_cpu(nla_get_be64(nla)));
3384 }
3385
3386 static const struct nla_policy nft_rule_policy[NFTA_RULE_MAX + 1] = {
3387         [NFTA_RULE_TABLE]       = { .type = NLA_STRING,
3388                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
3389         [NFTA_RULE_CHAIN]       = { .type = NLA_STRING,
3390                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
3391         [NFTA_RULE_HANDLE]      = { .type = NLA_U64 },
3392         [NFTA_RULE_EXPRESSIONS] = NLA_POLICY_NESTED_ARRAY(nft_expr_policy),
3393         [NFTA_RULE_COMPAT]      = { .type = NLA_NESTED },
3394         [NFTA_RULE_POSITION]    = { .type = NLA_U64 },
3395         [NFTA_RULE_USERDATA]    = { .type = NLA_BINARY,
3396                                     .len = NFT_USERDATA_MAXLEN },
3397         [NFTA_RULE_ID]          = { .type = NLA_U32 },
3398         [NFTA_RULE_POSITION_ID] = { .type = NLA_U32 },
3399         [NFTA_RULE_CHAIN_ID]    = { .type = NLA_U32 },
3400 };
3401
3402 static int nf_tables_fill_rule_info(struct sk_buff *skb, struct net *net,
3403                                     u32 portid, u32 seq, int event,
3404                                     u32 flags, int family,
3405                                     const struct nft_table *table,
3406                                     const struct nft_chain *chain,
3407                                     const struct nft_rule *rule, u64 handle,
3408                                     bool reset)
3409 {
3410         struct nlmsghdr *nlh;
3411         const struct nft_expr *expr, *next;
3412         struct nlattr *list;
3413         u16 type = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
3414
3415         nlh = nfnl_msg_put(skb, portid, seq, type, flags, family, NFNETLINK_V0,
3416                            nft_base_seq(net));
3417         if (!nlh)
3418                 goto nla_put_failure;
3419
3420         if (nla_put_string(skb, NFTA_RULE_TABLE, table->name))
3421                 goto nla_put_failure;
3422         if (nla_put_string(skb, NFTA_RULE_CHAIN, chain->name))
3423                 goto nla_put_failure;
3424         if (nla_put_be64(skb, NFTA_RULE_HANDLE, cpu_to_be64(rule->handle),
3425                          NFTA_RULE_PAD))
3426                 goto nla_put_failure;
3427
3428         if (event != NFT_MSG_DELRULE && handle) {
3429                 if (nla_put_be64(skb, NFTA_RULE_POSITION, cpu_to_be64(handle),
3430                                  NFTA_RULE_PAD))
3431                         goto nla_put_failure;
3432         }
3433
3434         if (chain->flags & NFT_CHAIN_HW_OFFLOAD)
3435                 nft_flow_rule_stats(chain, rule);
3436
3437         list = nla_nest_start_noflag(skb, NFTA_RULE_EXPRESSIONS);
3438         if (list == NULL)
3439                 goto nla_put_failure;
3440         nft_rule_for_each_expr(expr, next, rule) {
3441                 if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr, reset) < 0)
3442                         goto nla_put_failure;
3443         }
3444         nla_nest_end(skb, list);
3445
3446         if (rule->udata) {
3447                 struct nft_userdata *udata = nft_userdata(rule);
3448                 if (nla_put(skb, NFTA_RULE_USERDATA, udata->len + 1,
3449                             udata->data) < 0)
3450                         goto nla_put_failure;
3451         }
3452
3453         nlmsg_end(skb, nlh);
3454         return 0;
3455
3456 nla_put_failure:
3457         nlmsg_trim(skb, nlh);
3458         return -1;
3459 }
3460
3461 static void nf_tables_rule_notify(const struct nft_ctx *ctx,
3462                                   const struct nft_rule *rule, int event)
3463 {
3464         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
3465         const struct nft_rule *prule;
3466         struct sk_buff *skb;
3467         u64 handle = 0;
3468         u16 flags = 0;
3469         int err;
3470
3471         if (!ctx->report &&
3472             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
3473                 return;
3474
3475         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
3476         if (skb == NULL)
3477                 goto err;
3478
3479         if (event == NFT_MSG_NEWRULE &&
3480             !list_is_first(&rule->list, &ctx->chain->rules) &&
3481             !list_is_last(&rule->list, &ctx->chain->rules)) {
3482                 prule = list_prev_entry(rule, list);
3483                 handle = prule->handle;
3484         }
3485         if (ctx->flags & (NLM_F_APPEND | NLM_F_REPLACE))
3486                 flags |= NLM_F_APPEND;
3487         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
3488                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
3489
3490         err = nf_tables_fill_rule_info(skb, ctx->net, ctx->portid, ctx->seq,
3491                                        event, flags, ctx->family, ctx->table,
3492                                        ctx->chain, rule, handle, false);
3493         if (err < 0) {
3494                 kfree_skb(skb);
3495                 goto err;
3496         }
3497
3498         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
3499         return;
3500 err:
3501         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
3502 }
3503
3504 static void audit_log_rule_reset(const struct nft_table *table,
3505                                  unsigned int base_seq,
3506                                  unsigned int nentries)
3507 {
3508         char *buf = kasprintf(GFP_ATOMIC, "%s:%u",
3509                               table->name, base_seq);
3510
3511         audit_log_nfcfg(buf, table->family, nentries,
3512                         AUDIT_NFT_OP_RULE_RESET, GFP_ATOMIC);
3513         kfree(buf);
3514 }
3515
3516 struct nft_rule_dump_ctx {
3517         unsigned int s_idx;
3518         char *table;
3519         char *chain;
3520         bool reset;
3521 };
3522
3523 static int __nf_tables_dump_rules(struct sk_buff *skb,
3524                                   unsigned int *idx,
3525                                   struct netlink_callback *cb,
3526                                   const struct nft_table *table,
3527                                   const struct nft_chain *chain)
3528 {
3529         struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
3530         struct net *net = sock_net(skb->sk);
3531         const struct nft_rule *rule, *prule;
3532         unsigned int entries = 0;
3533         int ret = 0;
3534         u64 handle;
3535
3536         prule = NULL;
3537         list_for_each_entry_rcu(rule, &chain->rules, list) {
3538                 if (!nft_is_active(net, rule))
3539                         goto cont_skip;
3540                 if (*idx < ctx->s_idx)
3541                         goto cont;
3542                 if (prule)
3543                         handle = prule->handle;
3544                 else
3545                         handle = 0;
3546
3547                 if (nf_tables_fill_rule_info(skb, net, NETLINK_CB(cb->skb).portid,
3548                                         cb->nlh->nlmsg_seq,
3549                                         NFT_MSG_NEWRULE,
3550                                         NLM_F_MULTI | NLM_F_APPEND,
3551                                         table->family,
3552                                         table, chain, rule, handle, ctx->reset) < 0) {
3553                         ret = 1;
3554                         break;
3555                 }
3556                 entries++;
3557                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
3558 cont:
3559                 prule = rule;
3560 cont_skip:
3561                 (*idx)++;
3562         }
3563
3564         if (ctx->reset && entries)
3565                 audit_log_rule_reset(table, cb->seq, entries);
3566
3567         return ret;
3568 }
3569
3570 static int nf_tables_dump_rules(struct sk_buff *skb,
3571                                 struct netlink_callback *cb)
3572 {
3573         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
3574         struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
3575         struct nft_table *table;
3576         const struct nft_chain *chain;
3577         unsigned int idx = 0;
3578         struct net *net = sock_net(skb->sk);
3579         int family = nfmsg->nfgen_family;
3580         struct nftables_pernet *nft_net;
3581
3582         rcu_read_lock();
3583         nft_net = nft_pernet(net);
3584         cb->seq = READ_ONCE(nft_net->base_seq);
3585
3586         list_for_each_entry_rcu(table, &nft_net->tables, list) {
3587                 if (family != NFPROTO_UNSPEC && family != table->family)
3588                         continue;
3589
3590                 if (ctx->table && strcmp(ctx->table, table->name) != 0)
3591                         continue;
3592
3593                 if (ctx->table && ctx->chain) {
3594                         struct rhlist_head *list, *tmp;
3595
3596                         list = rhltable_lookup(&table->chains_ht, ctx->chain,
3597                                                nft_chain_ht_params);
3598                         if (!list)
3599                                 goto done;
3600
3601                         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
3602                                 if (!nft_is_active(net, chain))
3603                                         continue;
3604                                 __nf_tables_dump_rules(skb, &idx,
3605                                                        cb, table, chain);
3606                                 break;
3607                         }
3608                         goto done;
3609                 }
3610
3611                 list_for_each_entry_rcu(chain, &table->chains, list) {
3612                         if (__nf_tables_dump_rules(skb, &idx,
3613                                                    cb, table, chain))
3614                                 goto done;
3615                 }
3616
3617                 if (ctx->table)
3618                         break;
3619         }
3620 done:
3621         rcu_read_unlock();
3622
3623         ctx->s_idx = idx;
3624         return skb->len;
3625 }
3626
3627 static int nf_tables_dumpreset_rules(struct sk_buff *skb,
3628                                      struct netlink_callback *cb)
3629 {
3630         struct nftables_pernet *nft_net = nft_pernet(sock_net(skb->sk));
3631         int ret;
3632
3633         /* Mutex is held is to prevent that two concurrent dump-and-reset calls
3634          * do not underrun counters and quotas. The commit_mutex is used for
3635          * the lack a better lock, this is not transaction path.
3636          */
3637         mutex_lock(&nft_net->commit_mutex);
3638         ret = nf_tables_dump_rules(skb, cb);
3639         mutex_unlock(&nft_net->commit_mutex);
3640
3641         return ret;
3642 }
3643
3644 static int nf_tables_dump_rules_start(struct netlink_callback *cb)
3645 {
3646         struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
3647         const struct nlattr * const *nla = cb->data;
3648
3649         BUILD_BUG_ON(sizeof(*ctx) > sizeof(cb->ctx));
3650
3651         if (nla[NFTA_RULE_TABLE]) {
3652                 ctx->table = nla_strdup(nla[NFTA_RULE_TABLE], GFP_ATOMIC);
3653                 if (!ctx->table)
3654                         return -ENOMEM;
3655         }
3656         if (nla[NFTA_RULE_CHAIN]) {
3657                 ctx->chain = nla_strdup(nla[NFTA_RULE_CHAIN], GFP_ATOMIC);
3658                 if (!ctx->chain) {
3659                         kfree(ctx->table);
3660                         return -ENOMEM;
3661                 }
3662         }
3663         return 0;
3664 }
3665
3666 static int nf_tables_dumpreset_rules_start(struct netlink_callback *cb)
3667 {
3668         struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
3669
3670         ctx->reset = true;
3671
3672         return nf_tables_dump_rules_start(cb);
3673 }
3674
3675 static int nf_tables_dump_rules_done(struct netlink_callback *cb)
3676 {
3677         struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
3678
3679         kfree(ctx->table);
3680         kfree(ctx->chain);
3681         return 0;
3682 }
3683
3684 /* called with rcu_read_lock held */
3685 static struct sk_buff *
3686 nf_tables_getrule_single(u32 portid, const struct nfnl_info *info,
3687                          const struct nlattr * const nla[], bool reset)
3688 {
3689         struct netlink_ext_ack *extack = info->extack;
3690         u8 genmask = nft_genmask_cur(info->net);
3691         u8 family = info->nfmsg->nfgen_family;
3692         const struct nft_chain *chain;
3693         const struct nft_rule *rule;
3694         struct net *net = info->net;
3695         struct nft_table *table;
3696         struct sk_buff *skb2;
3697         int err;
3698
3699         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask, 0);
3700         if (IS_ERR(table)) {
3701                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3702                 return ERR_CAST(table);
3703         }
3704
3705         chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN], genmask);
3706         if (IS_ERR(chain)) {
3707                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3708                 return ERR_CAST(chain);
3709         }
3710
3711         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
3712         if (IS_ERR(rule)) {
3713                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3714                 return ERR_CAST(rule);
3715         }
3716
3717         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
3718         if (!skb2)
3719                 return ERR_PTR(-ENOMEM);
3720
3721         err = nf_tables_fill_rule_info(skb2, net, portid,
3722                                        info->nlh->nlmsg_seq, NFT_MSG_NEWRULE, 0,
3723                                        family, table, chain, rule, 0, reset);
3724         if (err < 0) {
3725                 kfree_skb(skb2);
3726                 return ERR_PTR(err);
3727         }
3728
3729         return skb2;
3730 }
3731
3732 static int nf_tables_getrule(struct sk_buff *skb, const struct nfnl_info *info,
3733                              const struct nlattr * const nla[])
3734 {
3735         u32 portid = NETLINK_CB(skb).portid;
3736         struct net *net = info->net;
3737         struct sk_buff *skb2;
3738
3739         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
3740                 struct netlink_dump_control c = {
3741                         .start= nf_tables_dump_rules_start,
3742                         .dump = nf_tables_dump_rules,
3743                         .done = nf_tables_dump_rules_done,
3744                         .module = THIS_MODULE,
3745                         .data = (void *)nla,
3746                 };
3747
3748                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
3749         }
3750
3751         skb2 = nf_tables_getrule_single(portid, info, nla, false);
3752         if (IS_ERR(skb2))
3753                 return PTR_ERR(skb2);
3754
3755         return nfnetlink_unicast(skb2, net, portid);
3756 }
3757
3758 static int nf_tables_getrule_reset(struct sk_buff *skb,
3759                                    const struct nfnl_info *info,
3760                                    const struct nlattr * const nla[])
3761 {
3762         struct nftables_pernet *nft_net = nft_pernet(info->net);
3763         u32 portid = NETLINK_CB(skb).portid;
3764         struct net *net = info->net;
3765         struct sk_buff *skb2;
3766         char *buf;
3767
3768         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
3769                 struct netlink_dump_control c = {
3770                         .start= nf_tables_dumpreset_rules_start,
3771                         .dump = nf_tables_dumpreset_rules,
3772                         .done = nf_tables_dump_rules_done,
3773                         .module = THIS_MODULE,
3774                         .data = (void *)nla,
3775                 };
3776
3777                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
3778         }
3779
3780         if (!try_module_get(THIS_MODULE))
3781                 return -EINVAL;
3782         rcu_read_unlock();
3783         mutex_lock(&nft_net->commit_mutex);
3784         skb2 = nf_tables_getrule_single(portid, info, nla, true);
3785         mutex_unlock(&nft_net->commit_mutex);
3786         rcu_read_lock();
3787         module_put(THIS_MODULE);
3788
3789         if (IS_ERR(skb2))
3790                 return PTR_ERR(skb2);
3791
3792         buf = kasprintf(GFP_ATOMIC, "%.*s:%u",
3793                         nla_len(nla[NFTA_RULE_TABLE]),
3794                         (char *)nla_data(nla[NFTA_RULE_TABLE]),
3795                         nft_net->base_seq);
3796         audit_log_nfcfg(buf, info->nfmsg->nfgen_family, 1,
3797                         AUDIT_NFT_OP_RULE_RESET, GFP_ATOMIC);
3798         kfree(buf);
3799
3800         return nfnetlink_unicast(skb2, net, portid);
3801 }
3802
3803 void nf_tables_rule_destroy(const struct nft_ctx *ctx, struct nft_rule *rule)
3804 {
3805         struct nft_expr *expr, *next;
3806
3807         /*
3808          * Careful: some expressions might not be initialized in case this
3809          * is called on error from nf_tables_newrule().
3810          */
3811         expr = nft_expr_first(rule);
3812         while (nft_expr_more(rule, expr)) {
3813                 next = nft_expr_next(expr);
3814                 nf_tables_expr_destroy(ctx, expr);
3815                 expr = next;
3816         }
3817         kfree(rule);
3818 }
3819
3820 static void nf_tables_rule_release(const struct nft_ctx *ctx, struct nft_rule *rule)
3821 {
3822         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_RELEASE);
3823         nf_tables_rule_destroy(ctx, rule);
3824 }
3825
3826 int nft_chain_validate(const struct nft_ctx *ctx, const struct nft_chain *chain)
3827 {
3828         struct nft_expr *expr, *last;
3829         const struct nft_data *data;
3830         struct nft_rule *rule;
3831         int err;
3832
3833         if (ctx->level == NFT_JUMP_STACK_SIZE)
3834                 return -EMLINK;
3835
3836         list_for_each_entry(rule, &chain->rules, list) {
3837                 if (fatal_signal_pending(current))
3838                         return -EINTR;
3839
3840                 if (!nft_is_active_next(ctx->net, rule))
3841                         continue;
3842
3843                 nft_rule_for_each_expr(expr, last, rule) {
3844                         if (!expr->ops->validate)
3845                                 continue;
3846
3847                         err = expr->ops->validate(ctx, expr, &data);
3848                         if (err < 0)
3849                                 return err;
3850                 }
3851         }
3852
3853         return 0;
3854 }
3855 EXPORT_SYMBOL_GPL(nft_chain_validate);
3856
3857 static int nft_table_validate(struct net *net, const struct nft_table *table)
3858 {
3859         struct nft_chain *chain;
3860         struct nft_ctx ctx = {
3861                 .net    = net,
3862                 .family = table->family,
3863         };
3864         int err;
3865
3866         list_for_each_entry(chain, &table->chains, list) {
3867                 if (!nft_is_base_chain(chain))
3868                         continue;
3869
3870                 ctx.chain = chain;
3871                 err = nft_chain_validate(&ctx, chain);
3872                 if (err < 0)
3873                         return err;
3874
3875                 cond_resched();
3876         }
3877
3878         return 0;
3879 }
3880
3881 int nft_setelem_validate(const struct nft_ctx *ctx, struct nft_set *set,
3882                          const struct nft_set_iter *iter,
3883                          struct nft_elem_priv *elem_priv)
3884 {
3885         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
3886         struct nft_ctx *pctx = (struct nft_ctx *)ctx;
3887         const struct nft_data *data;
3888         int err;
3889
3890         if (!nft_set_elem_active(ext, iter->genmask))
3891                 return 0;
3892
3893         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
3894             *nft_set_ext_flags(ext) & NFT_SET_ELEM_INTERVAL_END)
3895                 return 0;
3896
3897         data = nft_set_ext_data(ext);
3898         switch (data->verdict.code) {
3899         case NFT_JUMP:
3900         case NFT_GOTO:
3901                 pctx->level++;
3902                 err = nft_chain_validate(ctx, data->verdict.chain);
3903                 if (err < 0)
3904                         return err;
3905                 pctx->level--;
3906                 break;
3907         default:
3908                 break;
3909         }
3910
3911         return 0;
3912 }
3913
3914 int nft_set_catchall_validate(const struct nft_ctx *ctx, struct nft_set *set)
3915 {
3916         struct nft_set_iter dummy_iter = {
3917                 .genmask        = nft_genmask_next(ctx->net),
3918         };
3919         struct nft_set_elem_catchall *catchall;
3920
3921         struct nft_set_ext *ext;
3922         int ret = 0;
3923
3924         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
3925                 ext = nft_set_elem_ext(set, catchall->elem);
3926                 if (!nft_set_elem_active(ext, dummy_iter.genmask))
3927                         continue;
3928
3929                 ret = nft_setelem_validate(ctx, set, &dummy_iter, catchall->elem);
3930                 if (ret < 0)
3931                         return ret;
3932         }
3933
3934         return ret;
3935 }
3936
3937 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
3938                                              const struct nft_chain *chain,
3939                                              const struct nlattr *nla);
3940
3941 #define NFT_RULE_MAXEXPRS       128
3942
3943 static int nf_tables_newrule(struct sk_buff *skb, const struct nfnl_info *info,
3944                              const struct nlattr * const nla[])
3945 {
3946         struct nftables_pernet *nft_net = nft_pernet(info->net);
3947         struct netlink_ext_ack *extack = info->extack;
3948         unsigned int size, i, n, ulen = 0, usize = 0;
3949         u8 genmask = nft_genmask_next(info->net);
3950         struct nft_rule *rule, *old_rule = NULL;
3951         struct nft_expr_info *expr_info = NULL;
3952         u8 family = info->nfmsg->nfgen_family;
3953         struct nft_flow_rule *flow = NULL;
3954         struct net *net = info->net;
3955         struct nft_userdata *udata;
3956         struct nft_table *table;
3957         struct nft_chain *chain;
3958         struct nft_trans *trans;
3959         u64 handle, pos_handle;
3960         struct nft_expr *expr;
3961         struct nft_ctx ctx;
3962         struct nlattr *tmp;
3963         int err, rem;
3964
3965         lockdep_assert_held(&nft_net->commit_mutex);
3966
3967         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
3968                                  NETLINK_CB(skb).portid);
3969         if (IS_ERR(table)) {
3970                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3971                 return PTR_ERR(table);
3972         }
3973
3974         if (nla[NFTA_RULE_CHAIN]) {
3975                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
3976                                          genmask);
3977                 if (IS_ERR(chain)) {
3978                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3979                         return PTR_ERR(chain);
3980                 }
3981
3982         } else if (nla[NFTA_RULE_CHAIN_ID]) {
3983                 chain = nft_chain_lookup_byid(net, table, nla[NFTA_RULE_CHAIN_ID],
3984                                               genmask);
3985                 if (IS_ERR(chain)) {
3986                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN_ID]);
3987                         return PTR_ERR(chain);
3988                 }
3989         } else {
3990                 return -EINVAL;
3991         }
3992
3993         if (nft_chain_is_bound(chain))
3994                 return -EOPNOTSUPP;
3995
3996         if (nla[NFTA_RULE_HANDLE]) {
3997                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_HANDLE]));
3998                 rule = __nft_rule_lookup(chain, handle);
3999                 if (IS_ERR(rule)) {
4000                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
4001                         return PTR_ERR(rule);
4002                 }
4003
4004                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
4005                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
4006                         return -EEXIST;
4007                 }
4008                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
4009                         old_rule = rule;
4010                 else
4011                         return -EOPNOTSUPP;
4012         } else {
4013                 if (!(info->nlh->nlmsg_flags & NLM_F_CREATE) ||
4014                     info->nlh->nlmsg_flags & NLM_F_REPLACE)
4015                         return -EINVAL;
4016                 handle = nf_tables_alloc_handle(table);
4017
4018                 if (nla[NFTA_RULE_POSITION]) {
4019                         pos_handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_POSITION]));
4020                         old_rule = __nft_rule_lookup(chain, pos_handle);
4021                         if (IS_ERR(old_rule)) {
4022                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION]);
4023                                 return PTR_ERR(old_rule);
4024                         }
4025                 } else if (nla[NFTA_RULE_POSITION_ID]) {
4026                         old_rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_POSITION_ID]);
4027                         if (IS_ERR(old_rule)) {
4028                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION_ID]);
4029                                 return PTR_ERR(old_rule);
4030                         }
4031                 }
4032         }
4033
4034         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
4035
4036         n = 0;
4037         size = 0;
4038         if (nla[NFTA_RULE_EXPRESSIONS]) {
4039                 expr_info = kvmalloc_array(NFT_RULE_MAXEXPRS,
4040                                            sizeof(struct nft_expr_info),
4041                                            GFP_KERNEL);
4042                 if (!expr_info)
4043                         return -ENOMEM;
4044
4045                 nla_for_each_nested(tmp, nla[NFTA_RULE_EXPRESSIONS], rem) {
4046                         err = -EINVAL;
4047                         if (nla_type(tmp) != NFTA_LIST_ELEM)
4048                                 goto err_release_expr;
4049                         if (n == NFT_RULE_MAXEXPRS)
4050                                 goto err_release_expr;
4051                         err = nf_tables_expr_parse(&ctx, tmp, &expr_info[n]);
4052                         if (err < 0) {
4053                                 NL_SET_BAD_ATTR(extack, tmp);
4054                                 goto err_release_expr;
4055                         }
4056                         size += expr_info[n].ops->size;
4057                         n++;
4058                 }
4059         }
4060         /* Check for overflow of dlen field */
4061         err = -EFBIG;
4062         if (size >= 1 << 12)
4063                 goto err_release_expr;
4064
4065         if (nla[NFTA_RULE_USERDATA]) {
4066                 ulen = nla_len(nla[NFTA_RULE_USERDATA]);
4067                 if (ulen > 0)
4068                         usize = sizeof(struct nft_userdata) + ulen;
4069         }
4070
4071         err = -ENOMEM;
4072         rule = kzalloc(sizeof(*rule) + size + usize, GFP_KERNEL_ACCOUNT);
4073         if (rule == NULL)
4074                 goto err_release_expr;
4075
4076         nft_activate_next(net, rule);
4077
4078         rule->handle = handle;
4079         rule->dlen   = size;
4080         rule->udata  = ulen ? 1 : 0;
4081
4082         if (ulen) {
4083                 udata = nft_userdata(rule);
4084                 udata->len = ulen - 1;
4085                 nla_memcpy(udata->data, nla[NFTA_RULE_USERDATA], ulen);
4086         }
4087
4088         expr = nft_expr_first(rule);
4089         for (i = 0; i < n; i++) {
4090                 err = nf_tables_newexpr(&ctx, &expr_info[i], expr);
4091                 if (err < 0) {
4092                         NL_SET_BAD_ATTR(extack, expr_info[i].attr);
4093                         goto err_release_rule;
4094                 }
4095
4096                 if (expr_info[i].ops->validate)
4097                         nft_validate_state_update(table, NFT_VALIDATE_NEED);
4098
4099                 expr_info[i].ops = NULL;
4100                 expr = nft_expr_next(expr);
4101         }
4102
4103         if (chain->flags & NFT_CHAIN_HW_OFFLOAD) {
4104                 flow = nft_flow_rule_create(net, rule);
4105                 if (IS_ERR(flow)) {
4106                         err = PTR_ERR(flow);
4107                         goto err_release_rule;
4108                 }
4109         }
4110
4111         if (!nft_use_inc(&chain->use)) {
4112                 err = -EMFILE;
4113                 goto err_release_rule;
4114         }
4115
4116         if (info->nlh->nlmsg_flags & NLM_F_REPLACE) {
4117                 if (nft_chain_binding(chain)) {
4118                         err = -EOPNOTSUPP;
4119                         goto err_destroy_flow_rule;
4120                 }
4121
4122                 err = nft_delrule(&ctx, old_rule);
4123                 if (err < 0)
4124                         goto err_destroy_flow_rule;
4125
4126                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
4127                 if (trans == NULL) {
4128                         err = -ENOMEM;
4129                         goto err_destroy_flow_rule;
4130                 }
4131                 list_add_tail_rcu(&rule->list, &old_rule->list);
4132         } else {
4133                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
4134                 if (!trans) {
4135                         err = -ENOMEM;
4136                         goto err_destroy_flow_rule;
4137                 }
4138
4139                 if (info->nlh->nlmsg_flags & NLM_F_APPEND) {
4140                         if (old_rule)
4141                                 list_add_rcu(&rule->list, &old_rule->list);
4142                         else
4143                                 list_add_tail_rcu(&rule->list, &chain->rules);
4144                  } else {
4145                         if (old_rule)
4146                                 list_add_tail_rcu(&rule->list, &old_rule->list);
4147                         else
4148                                 list_add_rcu(&rule->list, &chain->rules);
4149                 }
4150         }
4151         kvfree(expr_info);
4152
4153         if (flow)
4154                 nft_trans_flow_rule(trans) = flow;
4155
4156         if (table->validate_state == NFT_VALIDATE_DO)
4157                 return nft_table_validate(net, table);
4158
4159         return 0;
4160
4161 err_destroy_flow_rule:
4162         nft_use_dec_restore(&chain->use);
4163         if (flow)
4164                 nft_flow_rule_destroy(flow);
4165 err_release_rule:
4166         nft_rule_expr_deactivate(&ctx, rule, NFT_TRANS_PREPARE_ERROR);
4167         nf_tables_rule_destroy(&ctx, rule);
4168 err_release_expr:
4169         for (i = 0; i < n; i++) {
4170                 if (expr_info[i].ops) {
4171                         module_put(expr_info[i].ops->type->owner);
4172                         if (expr_info[i].ops->type->release_ops)
4173                                 expr_info[i].ops->type->release_ops(expr_info[i].ops);
4174                 }
4175         }
4176         kvfree(expr_info);
4177
4178         return err;
4179 }
4180
4181 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
4182                                              const struct nft_chain *chain,
4183                                              const struct nlattr *nla)
4184 {
4185         struct nftables_pernet *nft_net = nft_pernet(net);
4186         u32 id = ntohl(nla_get_be32(nla));
4187         struct nft_trans *trans;
4188
4189         list_for_each_entry(trans, &nft_net->commit_list, list) {
4190                 if (trans->msg_type == NFT_MSG_NEWRULE &&
4191                     trans->ctx.chain == chain &&
4192                     id == nft_trans_rule_id(trans))
4193                         return nft_trans_rule(trans);
4194         }
4195         return ERR_PTR(-ENOENT);
4196 }
4197
4198 static int nf_tables_delrule(struct sk_buff *skb, const struct nfnl_info *info,
4199                              const struct nlattr * const nla[])
4200 {
4201         struct netlink_ext_ack *extack = info->extack;
4202         u8 genmask = nft_genmask_next(info->net);
4203         u8 family = info->nfmsg->nfgen_family;
4204         struct nft_chain *chain = NULL;
4205         struct net *net = info->net;
4206         struct nft_table *table;
4207         struct nft_rule *rule;
4208         struct nft_ctx ctx;
4209         int err = 0;
4210
4211         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
4212                                  NETLINK_CB(skb).portid);
4213         if (IS_ERR(table)) {
4214                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
4215                 return PTR_ERR(table);
4216         }
4217
4218         if (nla[NFTA_RULE_CHAIN]) {
4219                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
4220                                          genmask);
4221                 if (IS_ERR(chain)) {
4222                         if (PTR_ERR(chain) == -ENOENT &&
4223                             NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYRULE)
4224                                 return 0;
4225
4226                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
4227                         return PTR_ERR(chain);
4228                 }
4229                 if (nft_chain_binding(chain))
4230                         return -EOPNOTSUPP;
4231         }
4232
4233         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
4234
4235         if (chain) {
4236                 if (nla[NFTA_RULE_HANDLE]) {
4237                         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
4238                         if (IS_ERR(rule)) {
4239                                 if (PTR_ERR(rule) == -ENOENT &&
4240                                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYRULE)
4241                                         return 0;
4242
4243                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
4244                                 return PTR_ERR(rule);
4245                         }
4246
4247                         err = nft_delrule(&ctx, rule);
4248                 } else if (nla[NFTA_RULE_ID]) {
4249                         rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_ID]);
4250                         if (IS_ERR(rule)) {
4251                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_ID]);
4252                                 return PTR_ERR(rule);
4253                         }
4254
4255                         err = nft_delrule(&ctx, rule);
4256                 } else {
4257                         err = nft_delrule_by_chain(&ctx);
4258                 }
4259         } else {
4260                 list_for_each_entry(chain, &table->chains, list) {
4261                         if (!nft_is_active_next(net, chain))
4262                                 continue;
4263                         if (nft_chain_binding(chain))
4264                                 continue;
4265
4266                         ctx.chain = chain;
4267                         err = nft_delrule_by_chain(&ctx);
4268                         if (err < 0)
4269                                 break;
4270                 }
4271         }
4272
4273         return err;
4274 }
4275
4276 /*
4277  * Sets
4278  */
4279 static const struct nft_set_type *nft_set_types[] = {
4280         &nft_set_hash_fast_type,
4281         &nft_set_hash_type,
4282         &nft_set_rhash_type,
4283         &nft_set_bitmap_type,
4284         &nft_set_rbtree_type,
4285 #if defined(CONFIG_X86_64) && !defined(CONFIG_UML)
4286         &nft_set_pipapo_avx2_type,
4287 #endif
4288         &nft_set_pipapo_type,
4289 };
4290
4291 #define NFT_SET_FEATURES        (NFT_SET_INTERVAL | NFT_SET_MAP | \
4292                                  NFT_SET_TIMEOUT | NFT_SET_OBJECT | \
4293                                  NFT_SET_EVAL)
4294
4295 static bool nft_set_ops_candidate(const struct nft_set_type *type, u32 flags)
4296 {
4297         return (flags & type->features) == (flags & NFT_SET_FEATURES);
4298 }
4299
4300 /*
4301  * Select a set implementation based on the data characteristics and the
4302  * given policy. The total memory use might not be known if no size is
4303  * given, in that case the amount of memory per element is used.
4304  */
4305 static const struct nft_set_ops *
4306 nft_select_set_ops(const struct nft_ctx *ctx, u32 flags,
4307                    const struct nft_set_desc *desc)
4308 {
4309         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
4310         const struct nft_set_ops *ops, *bops;
4311         struct nft_set_estimate est, best;
4312         const struct nft_set_type *type;
4313         int i;
4314
4315         lockdep_assert_held(&nft_net->commit_mutex);
4316         lockdep_nfnl_nft_mutex_not_held();
4317
4318         bops        = NULL;
4319         best.size   = ~0;
4320         best.lookup = ~0;
4321         best.space  = ~0;
4322
4323         for (i = 0; i < ARRAY_SIZE(nft_set_types); i++) {
4324                 type = nft_set_types[i];
4325                 ops = &type->ops;
4326
4327                 if (!nft_set_ops_candidate(type, flags))
4328                         continue;
4329                 if (!ops->estimate(desc, flags, &est))
4330                         continue;
4331
4332                 switch (desc->policy) {
4333                 case NFT_SET_POL_PERFORMANCE:
4334                         if (est.lookup < best.lookup)
4335                                 break;
4336                         if (est.lookup == best.lookup &&
4337                             est.space < best.space)
4338                                 break;
4339                         continue;
4340                 case NFT_SET_POL_MEMORY:
4341                         if (!desc->size) {
4342                                 if (est.space < best.space)
4343                                         break;
4344                                 if (est.space == best.space &&
4345                                     est.lookup < best.lookup)
4346                                         break;
4347                         } else if (est.size < best.size || !bops) {
4348                                 break;
4349                         }
4350                         continue;
4351                 default:
4352                         break;
4353                 }
4354
4355                 bops = ops;
4356                 best = est;
4357         }
4358
4359         if (bops != NULL)
4360                 return bops;
4361
4362         return ERR_PTR(-EOPNOTSUPP);
4363 }
4364
4365 static const struct nla_policy nft_set_policy[NFTA_SET_MAX + 1] = {
4366         [NFTA_SET_TABLE]                = { .type = NLA_STRING,
4367                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
4368         [NFTA_SET_NAME]                 = { .type = NLA_STRING,
4369                                             .len = NFT_SET_MAXNAMELEN - 1 },
4370         [NFTA_SET_FLAGS]                = { .type = NLA_U32 },
4371         [NFTA_SET_KEY_TYPE]             = { .type = NLA_U32 },
4372         [NFTA_SET_KEY_LEN]              = { .type = NLA_U32 },
4373         [NFTA_SET_DATA_TYPE]            = { .type = NLA_U32 },
4374         [NFTA_SET_DATA_LEN]             = { .type = NLA_U32 },
4375         [NFTA_SET_POLICY]               = { .type = NLA_U32 },
4376         [NFTA_SET_DESC]                 = { .type = NLA_NESTED },
4377         [NFTA_SET_ID]                   = { .type = NLA_U32 },
4378         [NFTA_SET_TIMEOUT]              = { .type = NLA_U64 },
4379         [NFTA_SET_GC_INTERVAL]          = { .type = NLA_U32 },
4380         [NFTA_SET_USERDATA]             = { .type = NLA_BINARY,
4381                                             .len  = NFT_USERDATA_MAXLEN },
4382         [NFTA_SET_OBJ_TYPE]             = { .type = NLA_U32 },
4383         [NFTA_SET_HANDLE]               = { .type = NLA_U64 },
4384         [NFTA_SET_EXPR]                 = { .type = NLA_NESTED },
4385         [NFTA_SET_EXPRESSIONS]          = NLA_POLICY_NESTED_ARRAY(nft_expr_policy),
4386 };
4387
4388 static const struct nla_policy nft_concat_policy[NFTA_SET_FIELD_MAX + 1] = {
4389         [NFTA_SET_FIELD_LEN]    = { .type = NLA_U32 },
4390 };
4391
4392 static const struct nla_policy nft_set_desc_policy[NFTA_SET_DESC_MAX + 1] = {
4393         [NFTA_SET_DESC_SIZE]            = { .type = NLA_U32 },
4394         [NFTA_SET_DESC_CONCAT]          = NLA_POLICY_NESTED_ARRAY(nft_concat_policy),
4395 };
4396
4397 static struct nft_set *nft_set_lookup(const struct nft_table *table,
4398                                       const struct nlattr *nla, u8 genmask)
4399 {
4400         struct nft_set *set;
4401
4402         if (nla == NULL)
4403                 return ERR_PTR(-EINVAL);
4404
4405         list_for_each_entry_rcu(set, &table->sets, list) {
4406                 if (!nla_strcmp(nla, set->name) &&
4407                     nft_active_genmask(set, genmask))
4408                         return set;
4409         }
4410         return ERR_PTR(-ENOENT);
4411 }
4412
4413 static struct nft_set *nft_set_lookup_byhandle(const struct nft_table *table,
4414                                                const struct nlattr *nla,
4415                                                u8 genmask)
4416 {
4417         struct nft_set *set;
4418
4419         list_for_each_entry(set, &table->sets, list) {
4420                 if (be64_to_cpu(nla_get_be64(nla)) == set->handle &&
4421                     nft_active_genmask(set, genmask))
4422                         return set;
4423         }
4424         return ERR_PTR(-ENOENT);
4425 }
4426
4427 static struct nft_set *nft_set_lookup_byid(const struct net *net,
4428                                            const struct nft_table *table,
4429                                            const struct nlattr *nla, u8 genmask)
4430 {
4431         struct nftables_pernet *nft_net = nft_pernet(net);
4432         u32 id = ntohl(nla_get_be32(nla));
4433         struct nft_trans *trans;
4434
4435         list_for_each_entry(trans, &nft_net->commit_list, list) {
4436                 if (trans->msg_type == NFT_MSG_NEWSET) {
4437                         struct nft_set *set = nft_trans_set(trans);
4438
4439                         if (id == nft_trans_set_id(trans) &&
4440                             set->table == table &&
4441                             nft_active_genmask(set, genmask))
4442                                 return set;
4443                 }
4444         }
4445         return ERR_PTR(-ENOENT);
4446 }
4447
4448 struct nft_set *nft_set_lookup_global(const struct net *net,
4449                                       const struct nft_table *table,
4450                                       const struct nlattr *nla_set_name,
4451                                       const struct nlattr *nla_set_id,
4452                                       u8 genmask)
4453 {
4454         struct nft_set *set;
4455
4456         set = nft_set_lookup(table, nla_set_name, genmask);
4457         if (IS_ERR(set)) {
4458                 if (!nla_set_id)
4459                         return set;
4460
4461                 set = nft_set_lookup_byid(net, table, nla_set_id, genmask);
4462         }
4463         return set;
4464 }
4465 EXPORT_SYMBOL_GPL(nft_set_lookup_global);
4466
4467 static int nf_tables_set_alloc_name(struct nft_ctx *ctx, struct nft_set *set,
4468                                     const char *name)
4469 {
4470         const struct nft_set *i;
4471         const char *p;
4472         unsigned long *inuse;
4473         unsigned int n = 0, min = 0;
4474
4475         p = strchr(name, '%');
4476         if (p != NULL) {
4477                 if (p[1] != 'd' || strchr(p + 2, '%'))
4478                         return -EINVAL;
4479
4480                 if (strnlen(name, NFT_SET_MAX_ANONLEN) >= NFT_SET_MAX_ANONLEN)
4481                         return -EINVAL;
4482
4483                 inuse = (unsigned long *)get_zeroed_page(GFP_KERNEL);
4484                 if (inuse == NULL)
4485                         return -ENOMEM;
4486 cont:
4487                 list_for_each_entry(i, &ctx->table->sets, list) {
4488                         int tmp;
4489
4490                         if (!nft_is_active_next(ctx->net, i))
4491                                 continue;
4492                         if (!sscanf(i->name, name, &tmp))
4493                                 continue;
4494                         if (tmp < min || tmp >= min + BITS_PER_BYTE * PAGE_SIZE)
4495                                 continue;
4496
4497                         set_bit(tmp - min, inuse);
4498                 }
4499
4500                 n = find_first_zero_bit(inuse, BITS_PER_BYTE * PAGE_SIZE);
4501                 if (n >= BITS_PER_BYTE * PAGE_SIZE) {
4502                         min += BITS_PER_BYTE * PAGE_SIZE;
4503                         memset(inuse, 0, PAGE_SIZE);
4504                         goto cont;
4505                 }
4506                 free_page((unsigned long)inuse);
4507         }
4508
4509         set->name = kasprintf(GFP_KERNEL_ACCOUNT, name, min + n);
4510         if (!set->name)
4511                 return -ENOMEM;
4512
4513         list_for_each_entry(i, &ctx->table->sets, list) {
4514                 if (!nft_is_active_next(ctx->net, i))
4515                         continue;
4516                 if (!strcmp(set->name, i->name)) {
4517                         kfree(set->name);
4518                         set->name = NULL;
4519                         return -ENFILE;
4520                 }
4521         }
4522         return 0;
4523 }
4524
4525 int nf_msecs_to_jiffies64(const struct nlattr *nla, u64 *result)
4526 {
4527         u64 ms = be64_to_cpu(nla_get_be64(nla));
4528         u64 max = (u64)(~((u64)0));
4529
4530         max = div_u64(max, NSEC_PER_MSEC);
4531         if (ms >= max)
4532                 return -ERANGE;
4533
4534         ms *= NSEC_PER_MSEC;
4535         *result = nsecs_to_jiffies64(ms);
4536         return 0;
4537 }
4538
4539 __be64 nf_jiffies64_to_msecs(u64 input)
4540 {
4541         return cpu_to_be64(jiffies64_to_msecs(input));
4542 }
4543
4544 static int nf_tables_fill_set_concat(struct sk_buff *skb,
4545                                      const struct nft_set *set)
4546 {
4547         struct nlattr *concat, *field;
4548         int i;
4549
4550         concat = nla_nest_start_noflag(skb, NFTA_SET_DESC_CONCAT);
4551         if (!concat)
4552                 return -ENOMEM;
4553
4554         for (i = 0; i < set->field_count; i++) {
4555                 field = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
4556                 if (!field)
4557                         return -ENOMEM;
4558
4559                 if (nla_put_be32(skb, NFTA_SET_FIELD_LEN,
4560                                  htonl(set->field_len[i])))
4561                         return -ENOMEM;
4562
4563                 nla_nest_end(skb, field);
4564         }
4565
4566         nla_nest_end(skb, concat);
4567
4568         return 0;
4569 }
4570
4571 static int nf_tables_fill_set(struct sk_buff *skb, const struct nft_ctx *ctx,
4572                               const struct nft_set *set, u16 event, u16 flags)
4573 {
4574         u64 timeout = READ_ONCE(set->timeout);
4575         u32 gc_int = READ_ONCE(set->gc_int);
4576         u32 portid = ctx->portid;
4577         struct nlmsghdr *nlh;
4578         struct nlattr *nest;
4579         u32 seq = ctx->seq;
4580         int i;
4581
4582         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
4583         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
4584                            NFNETLINK_V0, nft_base_seq(ctx->net));
4585         if (!nlh)
4586                 goto nla_put_failure;
4587
4588         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
4589                 goto nla_put_failure;
4590         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
4591                 goto nla_put_failure;
4592         if (nla_put_be64(skb, NFTA_SET_HANDLE, cpu_to_be64(set->handle),
4593                          NFTA_SET_PAD))
4594                 goto nla_put_failure;
4595
4596         if (event == NFT_MSG_DELSET) {
4597                 nlmsg_end(skb, nlh);
4598                 return 0;
4599         }
4600
4601         if (set->flags != 0)
4602                 if (nla_put_be32(skb, NFTA_SET_FLAGS, htonl(set->flags)))
4603                         goto nla_put_failure;
4604
4605         if (nla_put_be32(skb, NFTA_SET_KEY_TYPE, htonl(set->ktype)))
4606                 goto nla_put_failure;
4607         if (nla_put_be32(skb, NFTA_SET_KEY_LEN, htonl(set->klen)))
4608                 goto nla_put_failure;
4609         if (set->flags & NFT_SET_MAP) {
4610                 if (nla_put_be32(skb, NFTA_SET_DATA_TYPE, htonl(set->dtype)))
4611                         goto nla_put_failure;
4612                 if (nla_put_be32(skb, NFTA_SET_DATA_LEN, htonl(set->dlen)))
4613                         goto nla_put_failure;
4614         }
4615         if (set->flags & NFT_SET_OBJECT &&
4616             nla_put_be32(skb, NFTA_SET_OBJ_TYPE, htonl(set->objtype)))
4617                 goto nla_put_failure;
4618
4619         if (timeout &&
4620             nla_put_be64(skb, NFTA_SET_TIMEOUT,
4621                          nf_jiffies64_to_msecs(timeout),
4622                          NFTA_SET_PAD))
4623                 goto nla_put_failure;
4624         if (gc_int &&
4625             nla_put_be32(skb, NFTA_SET_GC_INTERVAL, htonl(gc_int)))
4626                 goto nla_put_failure;
4627
4628         if (set->policy != NFT_SET_POL_PERFORMANCE) {
4629                 if (nla_put_be32(skb, NFTA_SET_POLICY, htonl(set->policy)))
4630                         goto nla_put_failure;
4631         }
4632
4633         if (set->udata &&
4634             nla_put(skb, NFTA_SET_USERDATA, set->udlen, set->udata))
4635                 goto nla_put_failure;
4636
4637         nest = nla_nest_start_noflag(skb, NFTA_SET_DESC);
4638         if (!nest)
4639                 goto nla_put_failure;
4640         if (set->size &&
4641             nla_put_be32(skb, NFTA_SET_DESC_SIZE, htonl(set->size)))
4642                 goto nla_put_failure;
4643
4644         if (set->field_count > 1 &&
4645             nf_tables_fill_set_concat(skb, set))
4646                 goto nla_put_failure;
4647
4648         nla_nest_end(skb, nest);
4649
4650         if (set->num_exprs == 1) {
4651                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPR);
4652                 if (nf_tables_fill_expr_info(skb, set->exprs[0], false) < 0)
4653                         goto nla_put_failure;
4654
4655                 nla_nest_end(skb, nest);
4656         } else if (set->num_exprs > 1) {
4657                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPRESSIONS);
4658                 if (nest == NULL)
4659                         goto nla_put_failure;
4660
4661                 for (i = 0; i < set->num_exprs; i++) {
4662                         if (nft_expr_dump(skb, NFTA_LIST_ELEM,
4663                                           set->exprs[i], false) < 0)
4664                                 goto nla_put_failure;
4665                 }
4666                 nla_nest_end(skb, nest);
4667         }
4668
4669         nlmsg_end(skb, nlh);
4670         return 0;
4671
4672 nla_put_failure:
4673         nlmsg_trim(skb, nlh);
4674         return -1;
4675 }
4676
4677 static void nf_tables_set_notify(const struct nft_ctx *ctx,
4678                                  const struct nft_set *set, int event,
4679                                  gfp_t gfp_flags)
4680 {
4681         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
4682         u32 portid = ctx->portid;
4683         struct sk_buff *skb;
4684         u16 flags = 0;
4685         int err;
4686
4687         if (!ctx->report &&
4688             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
4689                 return;
4690
4691         skb = nlmsg_new(NLMSG_GOODSIZE, gfp_flags);
4692         if (skb == NULL)
4693                 goto err;
4694
4695         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
4696                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
4697
4698         err = nf_tables_fill_set(skb, ctx, set, event, flags);
4699         if (err < 0) {
4700                 kfree_skb(skb);
4701                 goto err;
4702         }
4703
4704         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
4705         return;
4706 err:
4707         nfnetlink_set_err(ctx->net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
4708 }
4709
4710 static int nf_tables_dump_sets(struct sk_buff *skb, struct netlink_callback *cb)
4711 {
4712         const struct nft_set *set;
4713         unsigned int idx, s_idx = cb->args[0];
4714         struct nft_table *table, *cur_table = (struct nft_table *)cb->args[2];
4715         struct net *net = sock_net(skb->sk);
4716         struct nft_ctx *ctx = cb->data, ctx_set;
4717         struct nftables_pernet *nft_net;
4718
4719         if (cb->args[1])
4720                 return skb->len;
4721
4722         rcu_read_lock();
4723         nft_net = nft_pernet(net);
4724         cb->seq = READ_ONCE(nft_net->base_seq);
4725
4726         list_for_each_entry_rcu(table, &nft_net->tables, list) {
4727                 if (ctx->family != NFPROTO_UNSPEC &&
4728                     ctx->family != table->family)
4729                         continue;
4730
4731                 if (ctx->table && ctx->table != table)
4732                         continue;
4733
4734                 if (cur_table) {
4735                         if (cur_table != table)
4736                                 continue;
4737
4738                         cur_table = NULL;
4739                 }
4740                 idx = 0;
4741                 list_for_each_entry_rcu(set, &table->sets, list) {
4742                         if (idx < s_idx)
4743                                 goto cont;
4744                         if (!nft_is_active(net, set))
4745                                 goto cont;
4746
4747                         ctx_set = *ctx;
4748                         ctx_set.table = table;
4749                         ctx_set.family = table->family;
4750
4751                         if (nf_tables_fill_set(skb, &ctx_set, set,
4752                                                NFT_MSG_NEWSET,
4753                                                NLM_F_MULTI) < 0) {
4754                                 cb->args[0] = idx;
4755                                 cb->args[2] = (unsigned long) table;
4756                                 goto done;
4757                         }
4758                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
4759 cont:
4760                         idx++;
4761                 }
4762                 if (s_idx)
4763                         s_idx = 0;
4764         }
4765         cb->args[1] = 1;
4766 done:
4767         rcu_read_unlock();
4768         return skb->len;
4769 }
4770
4771 static int nf_tables_dump_sets_start(struct netlink_callback *cb)
4772 {
4773         struct nft_ctx *ctx_dump = NULL;
4774
4775         ctx_dump = kmemdup(cb->data, sizeof(*ctx_dump), GFP_ATOMIC);
4776         if (ctx_dump == NULL)
4777                 return -ENOMEM;
4778
4779         cb->data = ctx_dump;
4780         return 0;
4781 }
4782
4783 static int nf_tables_dump_sets_done(struct netlink_callback *cb)
4784 {
4785         kfree(cb->data);
4786         return 0;
4787 }
4788
4789 /* called with rcu_read_lock held */
4790 static int nf_tables_getset(struct sk_buff *skb, const struct nfnl_info *info,
4791                             const struct nlattr * const nla[])
4792 {
4793         struct netlink_ext_ack *extack = info->extack;
4794         u8 genmask = nft_genmask_cur(info->net);
4795         u8 family = info->nfmsg->nfgen_family;
4796         struct nft_table *table = NULL;
4797         struct net *net = info->net;
4798         const struct nft_set *set;
4799         struct sk_buff *skb2;
4800         struct nft_ctx ctx;
4801         int err;
4802
4803         if (nla[NFTA_SET_TABLE]) {
4804                 table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
4805                                          genmask, 0);
4806                 if (IS_ERR(table)) {
4807                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
4808                         return PTR_ERR(table);
4809                 }
4810         }
4811
4812         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
4813
4814         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
4815                 struct netlink_dump_control c = {
4816                         .start = nf_tables_dump_sets_start,
4817                         .dump = nf_tables_dump_sets,
4818                         .done = nf_tables_dump_sets_done,
4819                         .data = &ctx,
4820                         .module = THIS_MODULE,
4821                 };
4822
4823                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
4824         }
4825
4826         /* Only accept unspec with dump */
4827         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
4828                 return -EAFNOSUPPORT;
4829         if (!nla[NFTA_SET_TABLE])
4830                 return -EINVAL;
4831
4832         set = nft_set_lookup(table, nla[NFTA_SET_NAME], genmask);
4833         if (IS_ERR(set)) {
4834                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
4835                 return PTR_ERR(set);
4836         }
4837
4838         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
4839         if (skb2 == NULL)
4840                 return -ENOMEM;
4841
4842         err = nf_tables_fill_set(skb2, &ctx, set, NFT_MSG_NEWSET, 0);
4843         if (err < 0)
4844                 goto err_fill_set_info;
4845
4846         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
4847
4848 err_fill_set_info:
4849         kfree_skb(skb2);
4850         return err;
4851 }
4852
4853 static int nft_set_desc_concat_parse(const struct nlattr *attr,
4854                                      struct nft_set_desc *desc)
4855 {
4856         struct nlattr *tb[NFTA_SET_FIELD_MAX + 1];
4857         u32 len;
4858         int err;
4859
4860         if (desc->field_count >= ARRAY_SIZE(desc->field_len))
4861                 return -E2BIG;
4862
4863         err = nla_parse_nested_deprecated(tb, NFTA_SET_FIELD_MAX, attr,
4864                                           nft_concat_policy, NULL);
4865         if (err < 0)
4866                 return err;
4867
4868         if (!tb[NFTA_SET_FIELD_LEN])
4869                 return -EINVAL;
4870
4871         len = ntohl(nla_get_be32(tb[NFTA_SET_FIELD_LEN]));
4872         if (!len || len > U8_MAX)
4873                 return -EINVAL;
4874
4875         desc->field_len[desc->field_count++] = len;
4876
4877         return 0;
4878 }
4879
4880 static int nft_set_desc_concat(struct nft_set_desc *desc,
4881                                const struct nlattr *nla)
4882 {
4883         u32 num_regs = 0, key_num_regs = 0;
4884         struct nlattr *attr;
4885         int rem, err, i;
4886
4887         nla_for_each_nested(attr, nla, rem) {
4888                 if (nla_type(attr) != NFTA_LIST_ELEM)
4889                         return -EINVAL;
4890
4891                 err = nft_set_desc_concat_parse(attr, desc);
4892                 if (err < 0)
4893                         return err;
4894         }
4895
4896         for (i = 0; i < desc->field_count; i++)
4897                 num_regs += DIV_ROUND_UP(desc->field_len[i], sizeof(u32));
4898
4899         key_num_regs = DIV_ROUND_UP(desc->klen, sizeof(u32));
4900         if (key_num_regs != num_regs)
4901                 return -EINVAL;
4902
4903         if (num_regs > NFT_REG32_COUNT)
4904                 return -E2BIG;
4905
4906         return 0;
4907 }
4908
4909 static int nf_tables_set_desc_parse(struct nft_set_desc *desc,
4910                                     const struct nlattr *nla)
4911 {
4912         struct nlattr *da[NFTA_SET_DESC_MAX + 1];
4913         int err;
4914
4915         err = nla_parse_nested_deprecated(da, NFTA_SET_DESC_MAX, nla,
4916                                           nft_set_desc_policy, NULL);
4917         if (err < 0)
4918                 return err;
4919
4920         if (da[NFTA_SET_DESC_SIZE] != NULL)
4921                 desc->size = ntohl(nla_get_be32(da[NFTA_SET_DESC_SIZE]));
4922         if (da[NFTA_SET_DESC_CONCAT])
4923                 err = nft_set_desc_concat(desc, da[NFTA_SET_DESC_CONCAT]);
4924
4925         return err;
4926 }
4927
4928 static int nft_set_expr_alloc(struct nft_ctx *ctx, struct nft_set *set,
4929                               const struct nlattr * const *nla,
4930                               struct nft_expr **exprs, int *num_exprs,
4931                               u32 flags)
4932 {
4933         struct nft_expr *expr;
4934         int err, i;
4935
4936         if (nla[NFTA_SET_EXPR]) {
4937                 expr = nft_set_elem_expr_alloc(ctx, set, nla[NFTA_SET_EXPR]);
4938                 if (IS_ERR(expr)) {
4939                         err = PTR_ERR(expr);
4940                         goto err_set_expr_alloc;
4941                 }
4942                 exprs[0] = expr;
4943                 (*num_exprs)++;
4944         } else if (nla[NFTA_SET_EXPRESSIONS]) {
4945                 struct nlattr *tmp;
4946                 int left;
4947
4948                 if (!(flags & NFT_SET_EXPR)) {
4949                         err = -EINVAL;
4950                         goto err_set_expr_alloc;
4951                 }
4952                 i = 0;
4953                 nla_for_each_nested(tmp, nla[NFTA_SET_EXPRESSIONS], left) {
4954                         if (i == NFT_SET_EXPR_MAX) {
4955                                 err = -E2BIG;
4956                                 goto err_set_expr_alloc;
4957                         }
4958                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
4959                                 err = -EINVAL;
4960                                 goto err_set_expr_alloc;
4961                         }
4962                         expr = nft_set_elem_expr_alloc(ctx, set, tmp);
4963                         if (IS_ERR(expr)) {
4964                                 err = PTR_ERR(expr);
4965                                 goto err_set_expr_alloc;
4966                         }
4967                         exprs[i++] = expr;
4968                         (*num_exprs)++;
4969                 }
4970         }
4971
4972         return 0;
4973
4974 err_set_expr_alloc:
4975         for (i = 0; i < *num_exprs; i++)
4976                 nft_expr_destroy(ctx, exprs[i]);
4977
4978         return err;
4979 }
4980
4981 static bool nft_set_is_same(const struct nft_set *set,
4982                             const struct nft_set_desc *desc,
4983                             struct nft_expr *exprs[], u32 num_exprs, u32 flags)
4984 {
4985         int i;
4986
4987         if (set->ktype != desc->ktype ||
4988             set->dtype != desc->dtype ||
4989             set->flags != flags ||
4990             set->klen != desc->klen ||
4991             set->dlen != desc->dlen ||
4992             set->field_count != desc->field_count ||
4993             set->num_exprs != num_exprs)
4994                 return false;
4995
4996         for (i = 0; i < desc->field_count; i++) {
4997                 if (set->field_len[i] != desc->field_len[i])
4998                         return false;
4999         }
5000
5001         for (i = 0; i < num_exprs; i++) {
5002                 if (set->exprs[i]->ops != exprs[i]->ops)
5003                         return false;
5004         }
5005
5006         return true;
5007 }
5008
5009 static int nf_tables_newset(struct sk_buff *skb, const struct nfnl_info *info,
5010                             const struct nlattr * const nla[])
5011 {
5012         struct netlink_ext_ack *extack = info->extack;
5013         u8 genmask = nft_genmask_next(info->net);
5014         u8 family = info->nfmsg->nfgen_family;
5015         const struct nft_set_ops *ops;
5016         struct net *net = info->net;
5017         struct nft_set_desc desc;
5018         struct nft_table *table;
5019         unsigned char *udata;
5020         struct nft_set *set;
5021         struct nft_ctx ctx;
5022         size_t alloc_size;
5023         int num_exprs = 0;
5024         char *name;
5025         int err, i;
5026         u16 udlen;
5027         u32 flags;
5028         u64 size;
5029
5030         if (nla[NFTA_SET_TABLE] == NULL ||
5031             nla[NFTA_SET_NAME] == NULL ||
5032             nla[NFTA_SET_KEY_LEN] == NULL ||
5033             nla[NFTA_SET_ID] == NULL)
5034                 return -EINVAL;
5035
5036         memset(&desc, 0, sizeof(desc));
5037
5038         desc.ktype = NFT_DATA_VALUE;
5039         if (nla[NFTA_SET_KEY_TYPE] != NULL) {
5040                 desc.ktype = ntohl(nla_get_be32(nla[NFTA_SET_KEY_TYPE]));
5041                 if ((desc.ktype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK)
5042                         return -EINVAL;
5043         }
5044
5045         desc.klen = ntohl(nla_get_be32(nla[NFTA_SET_KEY_LEN]));
5046         if (desc.klen == 0 || desc.klen > NFT_DATA_VALUE_MAXLEN)
5047                 return -EINVAL;
5048
5049         flags = 0;
5050         if (nla[NFTA_SET_FLAGS] != NULL) {
5051                 flags = ntohl(nla_get_be32(nla[NFTA_SET_FLAGS]));
5052                 if (flags & ~(NFT_SET_ANONYMOUS | NFT_SET_CONSTANT |
5053                               NFT_SET_INTERVAL | NFT_SET_TIMEOUT |
5054                               NFT_SET_MAP | NFT_SET_EVAL |
5055                               NFT_SET_OBJECT | NFT_SET_CONCAT | NFT_SET_EXPR))
5056                         return -EOPNOTSUPP;
5057                 /* Only one of these operations is supported */
5058                 if ((flags & (NFT_SET_MAP | NFT_SET_OBJECT)) ==
5059                              (NFT_SET_MAP | NFT_SET_OBJECT))
5060                         return -EOPNOTSUPP;
5061                 if ((flags & (NFT_SET_EVAL | NFT_SET_OBJECT)) ==
5062                              (NFT_SET_EVAL | NFT_SET_OBJECT))
5063                         return -EOPNOTSUPP;
5064                 if ((flags & (NFT_SET_ANONYMOUS | NFT_SET_TIMEOUT | NFT_SET_EVAL)) ==
5065                              (NFT_SET_ANONYMOUS | NFT_SET_TIMEOUT))
5066                         return -EOPNOTSUPP;
5067                 if ((flags & (NFT_SET_CONSTANT | NFT_SET_TIMEOUT)) ==
5068                              (NFT_SET_CONSTANT | NFT_SET_TIMEOUT))
5069                         return -EOPNOTSUPP;
5070         }
5071
5072         desc.dtype = 0;
5073         if (nla[NFTA_SET_DATA_TYPE] != NULL) {
5074                 if (!(flags & NFT_SET_MAP))
5075                         return -EINVAL;
5076
5077                 desc.dtype = ntohl(nla_get_be32(nla[NFTA_SET_DATA_TYPE]));
5078                 if ((desc.dtype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK &&
5079                     desc.dtype != NFT_DATA_VERDICT)
5080                         return -EINVAL;
5081
5082                 if (desc.dtype != NFT_DATA_VERDICT) {
5083                         if (nla[NFTA_SET_DATA_LEN] == NULL)
5084                                 return -EINVAL;
5085                         desc.dlen = ntohl(nla_get_be32(nla[NFTA_SET_DATA_LEN]));
5086                         if (desc.dlen == 0 || desc.dlen > NFT_DATA_VALUE_MAXLEN)
5087                                 return -EINVAL;
5088                 } else
5089                         desc.dlen = sizeof(struct nft_verdict);
5090         } else if (flags & NFT_SET_MAP)
5091                 return -EINVAL;
5092
5093         if (nla[NFTA_SET_OBJ_TYPE] != NULL) {
5094                 if (!(flags & NFT_SET_OBJECT))
5095                         return -EINVAL;
5096
5097                 desc.objtype = ntohl(nla_get_be32(nla[NFTA_SET_OBJ_TYPE]));
5098                 if (desc.objtype == NFT_OBJECT_UNSPEC ||
5099                     desc.objtype > NFT_OBJECT_MAX)
5100                         return -EOPNOTSUPP;
5101         } else if (flags & NFT_SET_OBJECT)
5102                 return -EINVAL;
5103         else
5104                 desc.objtype = NFT_OBJECT_UNSPEC;
5105
5106         desc.timeout = 0;
5107         if (nla[NFTA_SET_TIMEOUT] != NULL) {
5108                 if (!(flags & NFT_SET_TIMEOUT))
5109                         return -EINVAL;
5110
5111                 if (flags & NFT_SET_ANONYMOUS)
5112                         return -EOPNOTSUPP;
5113
5114                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_TIMEOUT], &desc.timeout);
5115                 if (err)
5116                         return err;
5117         }
5118         desc.gc_int = 0;
5119         if (nla[NFTA_SET_GC_INTERVAL] != NULL) {
5120                 if (!(flags & NFT_SET_TIMEOUT))
5121                         return -EINVAL;
5122
5123                 if (flags & NFT_SET_ANONYMOUS)
5124                         return -EOPNOTSUPP;
5125
5126                 desc.gc_int = ntohl(nla_get_be32(nla[NFTA_SET_GC_INTERVAL]));
5127         }
5128
5129         desc.policy = NFT_SET_POL_PERFORMANCE;
5130         if (nla[NFTA_SET_POLICY] != NULL) {
5131                 desc.policy = ntohl(nla_get_be32(nla[NFTA_SET_POLICY]));
5132                 switch (desc.policy) {
5133                 case NFT_SET_POL_PERFORMANCE:
5134                 case NFT_SET_POL_MEMORY:
5135                         break;
5136                 default:
5137                         return -EOPNOTSUPP;
5138                 }
5139         }
5140
5141         if (nla[NFTA_SET_DESC] != NULL) {
5142                 err = nf_tables_set_desc_parse(&desc, nla[NFTA_SET_DESC]);
5143                 if (err < 0)
5144                         return err;
5145
5146                 if (desc.field_count > 1) {
5147                         if (!(flags & NFT_SET_CONCAT))
5148                                 return -EINVAL;
5149                 } else if (flags & NFT_SET_CONCAT) {
5150                         return -EINVAL;
5151                 }
5152         } else if (flags & NFT_SET_CONCAT) {
5153                 return -EINVAL;
5154         }
5155
5156         if (nla[NFTA_SET_EXPR] || nla[NFTA_SET_EXPRESSIONS])
5157                 desc.expr = true;
5158
5159         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family, genmask,
5160                                  NETLINK_CB(skb).portid);
5161         if (IS_ERR(table)) {
5162                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
5163                 return PTR_ERR(table);
5164         }
5165
5166         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
5167
5168         set = nft_set_lookup(table, nla[NFTA_SET_NAME], genmask);
5169         if (IS_ERR(set)) {
5170                 if (PTR_ERR(set) != -ENOENT) {
5171                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
5172                         return PTR_ERR(set);
5173                 }
5174         } else {
5175                 struct nft_expr *exprs[NFT_SET_EXPR_MAX] = {};
5176
5177                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
5178                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
5179                         return -EEXIST;
5180                 }
5181                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
5182                         return -EOPNOTSUPP;
5183
5184                 if (nft_set_is_anonymous(set))
5185                         return -EOPNOTSUPP;
5186
5187                 err = nft_set_expr_alloc(&ctx, set, nla, exprs, &num_exprs, flags);
5188                 if (err < 0)
5189                         return err;
5190
5191                 err = 0;
5192                 if (!nft_set_is_same(set, &desc, exprs, num_exprs, flags)) {
5193                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
5194                         err = -EEXIST;
5195                 }
5196
5197                 for (i = 0; i < num_exprs; i++)
5198                         nft_expr_destroy(&ctx, exprs[i]);
5199
5200                 if (err < 0)
5201                         return err;
5202
5203                 return __nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set, &desc);
5204         }
5205
5206         if (!(info->nlh->nlmsg_flags & NLM_F_CREATE))
5207                 return -ENOENT;
5208
5209         ops = nft_select_set_ops(&ctx, flags, &desc);
5210         if (IS_ERR(ops))
5211                 return PTR_ERR(ops);
5212
5213         udlen = 0;
5214         if (nla[NFTA_SET_USERDATA])
5215                 udlen = nla_len(nla[NFTA_SET_USERDATA]);
5216
5217         size = 0;
5218         if (ops->privsize != NULL)
5219                 size = ops->privsize(nla, &desc);
5220         alloc_size = sizeof(*set) + size + udlen;
5221         if (alloc_size < size || alloc_size > INT_MAX)
5222                 return -ENOMEM;
5223
5224         if (!nft_use_inc(&table->use))
5225                 return -EMFILE;
5226
5227         set = kvzalloc(alloc_size, GFP_KERNEL_ACCOUNT);
5228         if (!set) {
5229                 err = -ENOMEM;
5230                 goto err_alloc;
5231         }
5232
5233         name = nla_strdup(nla[NFTA_SET_NAME], GFP_KERNEL_ACCOUNT);
5234         if (!name) {
5235                 err = -ENOMEM;
5236                 goto err_set_name;
5237         }
5238
5239         err = nf_tables_set_alloc_name(&ctx, set, name);
5240         kfree(name);
5241         if (err < 0)
5242                 goto err_set_name;
5243
5244         udata = NULL;
5245         if (udlen) {
5246                 udata = set->data + size;
5247                 nla_memcpy(udata, nla[NFTA_SET_USERDATA], udlen);
5248         }
5249
5250         INIT_LIST_HEAD(&set->bindings);
5251         INIT_LIST_HEAD(&set->catchall_list);
5252         refcount_set(&set->refs, 1);
5253         set->table = table;
5254         write_pnet(&set->net, net);
5255         set->ops = ops;
5256         set->ktype = desc.ktype;
5257         set->klen = desc.klen;
5258         set->dtype = desc.dtype;
5259         set->objtype = desc.objtype;
5260         set->dlen = desc.dlen;
5261         set->flags = flags;
5262         set->size = desc.size;
5263         set->policy = desc.policy;
5264         set->udlen = udlen;
5265         set->udata = udata;
5266         set->timeout = desc.timeout;
5267         set->gc_int = desc.gc_int;
5268
5269         set->field_count = desc.field_count;
5270         for (i = 0; i < desc.field_count; i++)
5271                 set->field_len[i] = desc.field_len[i];
5272
5273         err = ops->init(set, &desc, nla);
5274         if (err < 0)
5275                 goto err_set_init;
5276
5277         err = nft_set_expr_alloc(&ctx, set, nla, set->exprs, &num_exprs, flags);
5278         if (err < 0)
5279                 goto err_set_destroy;
5280
5281         set->num_exprs = num_exprs;
5282         set->handle = nf_tables_alloc_handle(table);
5283         INIT_LIST_HEAD(&set->pending_update);
5284
5285         err = nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set);
5286         if (err < 0)
5287                 goto err_set_expr_alloc;
5288
5289         list_add_tail_rcu(&set->list, &table->sets);
5290
5291         return 0;
5292
5293 err_set_expr_alloc:
5294         for (i = 0; i < set->num_exprs; i++)
5295                 nft_expr_destroy(&ctx, set->exprs[i]);
5296 err_set_destroy:
5297         ops->destroy(&ctx, set);
5298 err_set_init:
5299         kfree(set->name);
5300 err_set_name:
5301         kvfree(set);
5302 err_alloc:
5303         nft_use_dec_restore(&table->use);
5304
5305         return err;
5306 }
5307
5308 static void nft_set_catchall_destroy(const struct nft_ctx *ctx,
5309                                      struct nft_set *set)
5310 {
5311         struct nft_set_elem_catchall *next, *catchall;
5312
5313         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
5314                 list_del_rcu(&catchall->list);
5315                 nf_tables_set_elem_destroy(ctx, set, catchall->elem);
5316                 kfree_rcu(catchall, rcu);
5317         }
5318 }
5319
5320 static void nft_set_put(struct nft_set *set)
5321 {
5322         if (refcount_dec_and_test(&set->refs)) {
5323                 kfree(set->name);
5324                 kvfree(set);
5325         }
5326 }
5327
5328 static void nft_set_destroy(const struct nft_ctx *ctx, struct nft_set *set)
5329 {
5330         int i;
5331
5332         if (WARN_ON(set->use > 0))
5333                 return;
5334
5335         for (i = 0; i < set->num_exprs; i++)
5336                 nft_expr_destroy(ctx, set->exprs[i]);
5337
5338         set->ops->destroy(ctx, set);
5339         nft_set_catchall_destroy(ctx, set);
5340         nft_set_put(set);
5341 }
5342
5343 static int nf_tables_delset(struct sk_buff *skb, const struct nfnl_info *info,
5344                             const struct nlattr * const nla[])
5345 {
5346         struct netlink_ext_ack *extack = info->extack;
5347         u8 genmask = nft_genmask_next(info->net);
5348         u8 family = info->nfmsg->nfgen_family;
5349         struct net *net = info->net;
5350         const struct nlattr *attr;
5351         struct nft_table *table;
5352         struct nft_set *set;
5353         struct nft_ctx ctx;
5354
5355         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
5356                 return -EAFNOSUPPORT;
5357
5358         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
5359                                  genmask, NETLINK_CB(skb).portid);
5360         if (IS_ERR(table)) {
5361                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
5362                 return PTR_ERR(table);
5363         }
5364
5365         if (nla[NFTA_SET_HANDLE]) {
5366                 attr = nla[NFTA_SET_HANDLE];
5367                 set = nft_set_lookup_byhandle(table, attr, genmask);
5368         } else {
5369                 attr = nla[NFTA_SET_NAME];
5370                 set = nft_set_lookup(table, attr, genmask);
5371         }
5372
5373         if (IS_ERR(set)) {
5374                 if (PTR_ERR(set) == -ENOENT &&
5375                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYSET)
5376                         return 0;
5377
5378                 NL_SET_BAD_ATTR(extack, attr);
5379                 return PTR_ERR(set);
5380         }
5381         if (set->use ||
5382             (info->nlh->nlmsg_flags & NLM_F_NONREC &&
5383              atomic_read(&set->nelems) > 0)) {
5384                 NL_SET_BAD_ATTR(extack, attr);
5385                 return -EBUSY;
5386         }
5387
5388         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
5389
5390         return nft_delset(&ctx, set);
5391 }
5392
5393 static int nft_validate_register_store(const struct nft_ctx *ctx,
5394                                        enum nft_registers reg,
5395                                        const struct nft_data *data,
5396                                        enum nft_data_types type,
5397                                        unsigned int len);
5398
5399 static int nft_setelem_data_validate(const struct nft_ctx *ctx,
5400                                      struct nft_set *set,
5401                                      struct nft_elem_priv *elem_priv)
5402 {
5403         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
5404         enum nft_registers dreg;
5405
5406         dreg = nft_type_to_reg(set->dtype);
5407         return nft_validate_register_store(ctx, dreg, nft_set_ext_data(ext),
5408                                            set->dtype == NFT_DATA_VERDICT ?
5409                                            NFT_DATA_VERDICT : NFT_DATA_VALUE,
5410                                            set->dlen);
5411 }
5412
5413 static int nf_tables_bind_check_setelem(const struct nft_ctx *ctx,
5414                                         struct nft_set *set,
5415                                         const struct nft_set_iter *iter,
5416                                         struct nft_elem_priv *elem_priv)
5417 {
5418         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
5419
5420         if (!nft_set_elem_active(ext, iter->genmask))
5421                 return 0;
5422
5423         return nft_setelem_data_validate(ctx, set, elem_priv);
5424 }
5425
5426 static int nft_set_catchall_bind_check(const struct nft_ctx *ctx,
5427                                        struct nft_set *set)
5428 {
5429         u8 genmask = nft_genmask_next(ctx->net);
5430         struct nft_set_elem_catchall *catchall;
5431         struct nft_set_ext *ext;
5432         int ret = 0;
5433
5434         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5435                 ext = nft_set_elem_ext(set, catchall->elem);
5436                 if (!nft_set_elem_active(ext, genmask))
5437                         continue;
5438
5439                 ret = nft_setelem_data_validate(ctx, set, catchall->elem);
5440                 if (ret < 0)
5441                         break;
5442         }
5443
5444         return ret;
5445 }
5446
5447 int nf_tables_bind_set(const struct nft_ctx *ctx, struct nft_set *set,
5448                        struct nft_set_binding *binding)
5449 {
5450         struct nft_set_binding *i;
5451         struct nft_set_iter iter;
5452
5453         if (!list_empty(&set->bindings) && nft_set_is_anonymous(set))
5454                 return -EBUSY;
5455
5456         if (binding->flags & NFT_SET_MAP) {
5457                 /* If the set is already bound to the same chain all
5458                  * jumps are already validated for that chain.
5459                  */
5460                 list_for_each_entry(i, &set->bindings, list) {
5461                         if (i->flags & NFT_SET_MAP &&
5462                             i->chain == binding->chain)
5463                                 goto bind;
5464                 }
5465
5466                 iter.genmask    = nft_genmask_next(ctx->net);
5467                 iter.type       = NFT_ITER_UPDATE;
5468                 iter.skip       = 0;
5469                 iter.count      = 0;
5470                 iter.err        = 0;
5471                 iter.fn         = nf_tables_bind_check_setelem;
5472
5473                 set->ops->walk(ctx, set, &iter);
5474                 if (!iter.err)
5475                         iter.err = nft_set_catchall_bind_check(ctx, set);
5476
5477                 if (iter.err < 0)
5478                         return iter.err;
5479         }
5480 bind:
5481         if (!nft_use_inc(&set->use))
5482                 return -EMFILE;
5483
5484         binding->chain = ctx->chain;
5485         list_add_tail_rcu(&binding->list, &set->bindings);
5486         nft_set_trans_bind(ctx, set);
5487
5488         return 0;
5489 }
5490 EXPORT_SYMBOL_GPL(nf_tables_bind_set);
5491
5492 static void nf_tables_unbind_set(const struct nft_ctx *ctx, struct nft_set *set,
5493                                  struct nft_set_binding *binding, bool event)
5494 {
5495         list_del_rcu(&binding->list);
5496
5497         if (list_empty(&set->bindings) && nft_set_is_anonymous(set)) {
5498                 list_del_rcu(&set->list);
5499                 set->dead = 1;
5500                 if (event)
5501                         nf_tables_set_notify(ctx, set, NFT_MSG_DELSET,
5502                                              GFP_KERNEL);
5503         }
5504 }
5505
5506 static void nft_setelem_data_activate(const struct net *net,
5507                                       const struct nft_set *set,
5508                                       struct nft_elem_priv *elem_priv);
5509
5510 static int nft_mapelem_activate(const struct nft_ctx *ctx,
5511                                 struct nft_set *set,
5512                                 const struct nft_set_iter *iter,
5513                                 struct nft_elem_priv *elem_priv)
5514 {
5515         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
5516
5517         /* called from abort path, reverse check to undo changes. */
5518         if (nft_set_elem_active(ext, iter->genmask))
5519                 return 0;
5520
5521         nft_clear(ctx->net, ext);
5522         nft_setelem_data_activate(ctx->net, set, elem_priv);
5523
5524         return 0;
5525 }
5526
5527 static void nft_map_catchall_activate(const struct nft_ctx *ctx,
5528                                       struct nft_set *set)
5529 {
5530         u8 genmask = nft_genmask_next(ctx->net);
5531         struct nft_set_elem_catchall *catchall;
5532         struct nft_set_ext *ext;
5533
5534         list_for_each_entry(catchall, &set->catchall_list, list) {
5535                 ext = nft_set_elem_ext(set, catchall->elem);
5536                 if (!nft_set_elem_active(ext, genmask))
5537                         continue;
5538
5539                 nft_clear(ctx->net, ext);
5540                 nft_setelem_data_activate(ctx->net, set, catchall->elem);
5541                 break;
5542         }
5543 }
5544
5545 static void nft_map_activate(const struct nft_ctx *ctx, struct nft_set *set)
5546 {
5547         struct nft_set_iter iter = {
5548                 .genmask        = nft_genmask_next(ctx->net),
5549                 .type           = NFT_ITER_UPDATE,
5550                 .fn             = nft_mapelem_activate,
5551         };
5552
5553         set->ops->walk(ctx, set, &iter);
5554         WARN_ON_ONCE(iter.err);
5555
5556         nft_map_catchall_activate(ctx, set);
5557 }
5558
5559 void nf_tables_activate_set(const struct nft_ctx *ctx, struct nft_set *set)
5560 {
5561         if (nft_set_is_anonymous(set)) {
5562                 if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
5563                         nft_map_activate(ctx, set);
5564
5565                 nft_clear(ctx->net, set);
5566         }
5567
5568         nft_use_inc_restore(&set->use);
5569 }
5570 EXPORT_SYMBOL_GPL(nf_tables_activate_set);
5571
5572 void nf_tables_deactivate_set(const struct nft_ctx *ctx, struct nft_set *set,
5573                               struct nft_set_binding *binding,
5574                               enum nft_trans_phase phase)
5575 {
5576         switch (phase) {
5577         case NFT_TRANS_PREPARE_ERROR:
5578                 nft_set_trans_unbind(ctx, set);
5579                 if (nft_set_is_anonymous(set))
5580                         nft_deactivate_next(ctx->net, set);
5581                 else
5582                         list_del_rcu(&binding->list);
5583
5584                 nft_use_dec(&set->use);
5585                 break;
5586         case NFT_TRANS_PREPARE:
5587                 if (nft_set_is_anonymous(set)) {
5588                         if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
5589                                 nft_map_deactivate(ctx, set);
5590
5591                         nft_deactivate_next(ctx->net, set);
5592                 }
5593                 nft_use_dec(&set->use);
5594                 return;
5595         case NFT_TRANS_ABORT:
5596         case NFT_TRANS_RELEASE:
5597                 if (nft_set_is_anonymous(set) &&
5598                     set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
5599                         nft_map_deactivate(ctx, set);
5600
5601                 nft_use_dec(&set->use);
5602                 fallthrough;
5603         default:
5604                 nf_tables_unbind_set(ctx, set, binding,
5605                                      phase == NFT_TRANS_COMMIT);
5606         }
5607 }
5608 EXPORT_SYMBOL_GPL(nf_tables_deactivate_set);
5609
5610 void nf_tables_destroy_set(const struct nft_ctx *ctx, struct nft_set *set)
5611 {
5612         if (list_empty(&set->bindings) && nft_set_is_anonymous(set))
5613                 nft_set_destroy(ctx, set);
5614 }
5615 EXPORT_SYMBOL_GPL(nf_tables_destroy_set);
5616
5617 const struct nft_set_ext_type nft_set_ext_types[] = {
5618         [NFT_SET_EXT_KEY]               = {
5619                 .align  = __alignof__(u32),
5620         },
5621         [NFT_SET_EXT_DATA]              = {
5622                 .align  = __alignof__(u32),
5623         },
5624         [NFT_SET_EXT_EXPRESSIONS]       = {
5625                 .align  = __alignof__(struct nft_set_elem_expr),
5626         },
5627         [NFT_SET_EXT_OBJREF]            = {
5628                 .len    = sizeof(struct nft_object *),
5629                 .align  = __alignof__(struct nft_object *),
5630         },
5631         [NFT_SET_EXT_FLAGS]             = {
5632                 .len    = sizeof(u8),
5633                 .align  = __alignof__(u8),
5634         },
5635         [NFT_SET_EXT_TIMEOUT]           = {
5636                 .len    = sizeof(u64),
5637                 .align  = __alignof__(u64),
5638         },
5639         [NFT_SET_EXT_EXPIRATION]        = {
5640                 .len    = sizeof(u64),
5641                 .align  = __alignof__(u64),
5642         },
5643         [NFT_SET_EXT_USERDATA]          = {
5644                 .len    = sizeof(struct nft_userdata),
5645                 .align  = __alignof__(struct nft_userdata),
5646         },
5647         [NFT_SET_EXT_KEY_END]           = {
5648                 .align  = __alignof__(u32),
5649         },
5650 };
5651
5652 /*
5653  * Set elements
5654  */
5655
5656 static const struct nla_policy nft_set_elem_policy[NFTA_SET_ELEM_MAX + 1] = {
5657         [NFTA_SET_ELEM_KEY]             = { .type = NLA_NESTED },
5658         [NFTA_SET_ELEM_DATA]            = { .type = NLA_NESTED },
5659         [NFTA_SET_ELEM_FLAGS]           = { .type = NLA_U32 },
5660         [NFTA_SET_ELEM_TIMEOUT]         = { .type = NLA_U64 },
5661         [NFTA_SET_ELEM_EXPIRATION]      = { .type = NLA_U64 },
5662         [NFTA_SET_ELEM_USERDATA]        = { .type = NLA_BINARY,
5663                                             .len = NFT_USERDATA_MAXLEN },
5664         [NFTA_SET_ELEM_EXPR]            = { .type = NLA_NESTED },
5665         [NFTA_SET_ELEM_OBJREF]          = { .type = NLA_STRING,
5666                                             .len = NFT_OBJ_MAXNAMELEN - 1 },
5667         [NFTA_SET_ELEM_KEY_END]         = { .type = NLA_NESTED },
5668         [NFTA_SET_ELEM_EXPRESSIONS]     = NLA_POLICY_NESTED_ARRAY(nft_expr_policy),
5669 };
5670
5671 static const struct nla_policy nft_set_elem_list_policy[NFTA_SET_ELEM_LIST_MAX + 1] = {
5672         [NFTA_SET_ELEM_LIST_TABLE]      = { .type = NLA_STRING,
5673                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
5674         [NFTA_SET_ELEM_LIST_SET]        = { .type = NLA_STRING,
5675                                             .len = NFT_SET_MAXNAMELEN - 1 },
5676         [NFTA_SET_ELEM_LIST_ELEMENTS]   = NLA_POLICY_NESTED_ARRAY(nft_set_elem_policy),
5677         [NFTA_SET_ELEM_LIST_SET_ID]     = { .type = NLA_U32 },
5678 };
5679
5680 static int nft_set_elem_expr_dump(struct sk_buff *skb,
5681                                   const struct nft_set *set,
5682                                   const struct nft_set_ext *ext,
5683                                   bool reset)
5684 {
5685         struct nft_set_elem_expr *elem_expr;
5686         u32 size, num_exprs = 0;
5687         struct nft_expr *expr;
5688         struct nlattr *nest;
5689
5690         elem_expr = nft_set_ext_expr(ext);
5691         nft_setelem_expr_foreach(expr, elem_expr, size)
5692                 num_exprs++;
5693
5694         if (num_exprs == 1) {
5695                 expr = nft_setelem_expr_at(elem_expr, 0);
5696                 if (nft_expr_dump(skb, NFTA_SET_ELEM_EXPR, expr, reset) < 0)
5697                         return -1;
5698
5699                 return 0;
5700         } else if (num_exprs > 1) {
5701                 nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_EXPRESSIONS);
5702                 if (nest == NULL)
5703                         goto nla_put_failure;
5704
5705                 nft_setelem_expr_foreach(expr, elem_expr, size) {
5706                         expr = nft_setelem_expr_at(elem_expr, size);
5707                         if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr, reset) < 0)
5708                                 goto nla_put_failure;
5709                 }
5710                 nla_nest_end(skb, nest);
5711         }
5712         return 0;
5713
5714 nla_put_failure:
5715         return -1;
5716 }
5717
5718 static int nf_tables_fill_setelem(struct sk_buff *skb,
5719                                   const struct nft_set *set,
5720                                   const struct nft_elem_priv *elem_priv,
5721                                   bool reset)
5722 {
5723         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
5724         unsigned char *b = skb_tail_pointer(skb);
5725         struct nlattr *nest;
5726
5727         nest = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
5728         if (nest == NULL)
5729                 goto nla_put_failure;
5730
5731         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
5732             nft_data_dump(skb, NFTA_SET_ELEM_KEY, nft_set_ext_key(ext),
5733                           NFT_DATA_VALUE, set->klen) < 0)
5734                 goto nla_put_failure;
5735
5736         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
5737             nft_data_dump(skb, NFTA_SET_ELEM_KEY_END, nft_set_ext_key_end(ext),
5738                           NFT_DATA_VALUE, set->klen) < 0)
5739                 goto nla_put_failure;
5740
5741         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
5742             nft_data_dump(skb, NFTA_SET_ELEM_DATA, nft_set_ext_data(ext),
5743                           set->dtype == NFT_DATA_VERDICT ? NFT_DATA_VERDICT : NFT_DATA_VALUE,
5744                           set->dlen) < 0)
5745                 goto nla_put_failure;
5746
5747         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS) &&
5748             nft_set_elem_expr_dump(skb, set, ext, reset))
5749                 goto nla_put_failure;
5750
5751         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
5752             nla_put_string(skb, NFTA_SET_ELEM_OBJREF,
5753                            (*nft_set_ext_obj(ext))->key.name) < 0)
5754                 goto nla_put_failure;
5755
5756         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
5757             nla_put_be32(skb, NFTA_SET_ELEM_FLAGS,
5758                          htonl(*nft_set_ext_flags(ext))))
5759                 goto nla_put_failure;
5760
5761         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT) &&
5762             nla_put_be64(skb, NFTA_SET_ELEM_TIMEOUT,
5763                          nf_jiffies64_to_msecs(*nft_set_ext_timeout(ext)),
5764                          NFTA_SET_ELEM_PAD))
5765                 goto nla_put_failure;
5766
5767         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
5768                 u64 expires, now = get_jiffies_64();
5769
5770                 expires = *nft_set_ext_expiration(ext);
5771                 if (time_before64(now, expires))
5772                         expires -= now;
5773                 else
5774                         expires = 0;
5775
5776                 if (nla_put_be64(skb, NFTA_SET_ELEM_EXPIRATION,
5777                                  nf_jiffies64_to_msecs(expires),
5778                                  NFTA_SET_ELEM_PAD))
5779                         goto nla_put_failure;
5780         }
5781
5782         if (nft_set_ext_exists(ext, NFT_SET_EXT_USERDATA)) {
5783                 struct nft_userdata *udata;
5784
5785                 udata = nft_set_ext_userdata(ext);
5786                 if (nla_put(skb, NFTA_SET_ELEM_USERDATA,
5787                             udata->len + 1, udata->data))
5788                         goto nla_put_failure;
5789         }
5790
5791         nla_nest_end(skb, nest);
5792         return 0;
5793
5794 nla_put_failure:
5795         nlmsg_trim(skb, b);
5796         return -EMSGSIZE;
5797 }
5798
5799 struct nft_set_dump_args {
5800         const struct netlink_callback   *cb;
5801         struct nft_set_iter             iter;
5802         struct sk_buff                  *skb;
5803         bool                            reset;
5804 };
5805
5806 static int nf_tables_dump_setelem(const struct nft_ctx *ctx,
5807                                   struct nft_set *set,
5808                                   const struct nft_set_iter *iter,
5809                                   struct nft_elem_priv *elem_priv)
5810 {
5811         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
5812         struct nft_set_dump_args *args;
5813
5814         if (!nft_set_elem_active(ext, iter->genmask))
5815                 return 0;
5816
5817         if (nft_set_elem_expired(ext) || nft_set_elem_is_dead(ext))
5818                 return 0;
5819
5820         args = container_of(iter, struct nft_set_dump_args, iter);
5821         return nf_tables_fill_setelem(args->skb, set, elem_priv, args->reset);
5822 }
5823
5824 static void audit_log_nft_set_reset(const struct nft_table *table,
5825                                     unsigned int base_seq,
5826                                     unsigned int nentries)
5827 {
5828         char *buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, base_seq);
5829
5830         audit_log_nfcfg(buf, table->family, nentries,
5831                         AUDIT_NFT_OP_SETELEM_RESET, GFP_ATOMIC);
5832         kfree(buf);
5833 }
5834
5835 struct nft_set_dump_ctx {
5836         const struct nft_set    *set;
5837         struct nft_ctx          ctx;
5838         bool                    reset;
5839 };
5840
5841 static int nft_set_catchall_dump(struct net *net, struct sk_buff *skb,
5842                                  const struct nft_set *set, bool reset,
5843                                  unsigned int base_seq)
5844 {
5845         struct nft_set_elem_catchall *catchall;
5846         u8 genmask = nft_genmask_cur(net);
5847         struct nft_set_ext *ext;
5848         int ret = 0;
5849
5850         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5851                 ext = nft_set_elem_ext(set, catchall->elem);
5852                 if (!nft_set_elem_active(ext, genmask) ||
5853                     nft_set_elem_expired(ext))
5854                         continue;
5855
5856                 ret = nf_tables_fill_setelem(skb, set, catchall->elem, reset);
5857                 if (reset && !ret)
5858                         audit_log_nft_set_reset(set->table, base_seq, 1);
5859                 break;
5860         }
5861
5862         return ret;
5863 }
5864
5865 static int nf_tables_dump_set(struct sk_buff *skb, struct netlink_callback *cb)
5866 {
5867         struct nft_set_dump_ctx *dump_ctx = cb->data;
5868         struct net *net = sock_net(skb->sk);
5869         struct nftables_pernet *nft_net;
5870         struct nft_table *table;
5871         struct nft_set *set;
5872         struct nft_set_dump_args args;
5873         bool set_found = false;
5874         struct nlmsghdr *nlh;
5875         struct nlattr *nest;
5876         u32 portid, seq;
5877         int event;
5878
5879         rcu_read_lock();
5880         nft_net = nft_pernet(net);
5881         cb->seq = READ_ONCE(nft_net->base_seq);
5882
5883         list_for_each_entry_rcu(table, &nft_net->tables, list) {
5884                 if (dump_ctx->ctx.family != NFPROTO_UNSPEC &&
5885                     dump_ctx->ctx.family != table->family)
5886                         continue;
5887
5888                 if (table != dump_ctx->ctx.table)
5889                         continue;
5890
5891                 list_for_each_entry_rcu(set, &table->sets, list) {
5892                         if (set == dump_ctx->set) {
5893                                 set_found = true;
5894                                 break;
5895                         }
5896                 }
5897                 break;
5898         }
5899
5900         if (!set_found) {
5901                 rcu_read_unlock();
5902                 return -ENOENT;
5903         }
5904
5905         event  = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWSETELEM);
5906         portid = NETLINK_CB(cb->skb).portid;
5907         seq    = cb->nlh->nlmsg_seq;
5908
5909         nlh = nfnl_msg_put(skb, portid, seq, event, NLM_F_MULTI,
5910                            table->family, NFNETLINK_V0, nft_base_seq(net));
5911         if (!nlh)
5912                 goto nla_put_failure;
5913
5914         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_TABLE, table->name))
5915                 goto nla_put_failure;
5916         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_SET, set->name))
5917                 goto nla_put_failure;
5918
5919         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
5920         if (nest == NULL)
5921                 goto nla_put_failure;
5922
5923         args.cb                 = cb;
5924         args.skb                = skb;
5925         args.reset              = dump_ctx->reset;
5926         args.iter.genmask       = nft_genmask_cur(net);
5927         args.iter.type          = NFT_ITER_READ;
5928         args.iter.skip          = cb->args[0];
5929         args.iter.count         = 0;
5930         args.iter.err           = 0;
5931         args.iter.fn            = nf_tables_dump_setelem;
5932         set->ops->walk(&dump_ctx->ctx, set, &args.iter);
5933
5934         if (!args.iter.err && args.iter.count == cb->args[0])
5935                 args.iter.err = nft_set_catchall_dump(net, skb, set,
5936                                                       dump_ctx->reset, cb->seq);
5937         nla_nest_end(skb, nest);
5938         nlmsg_end(skb, nlh);
5939
5940         rcu_read_unlock();
5941
5942         if (args.iter.err && args.iter.err != -EMSGSIZE)
5943                 return args.iter.err;
5944         if (args.iter.count == cb->args[0])
5945                 return 0;
5946
5947         cb->args[0] = args.iter.count;
5948         return skb->len;
5949
5950 nla_put_failure:
5951         rcu_read_unlock();
5952         return -ENOSPC;
5953 }
5954
5955 static int nf_tables_dumpreset_set(struct sk_buff *skb,
5956                                    struct netlink_callback *cb)
5957 {
5958         struct nftables_pernet *nft_net = nft_pernet(sock_net(skb->sk));
5959         struct nft_set_dump_ctx *dump_ctx = cb->data;
5960         int ret, skip = cb->args[0];
5961
5962         mutex_lock(&nft_net->commit_mutex);
5963
5964         ret = nf_tables_dump_set(skb, cb);
5965
5966         if (cb->args[0] > skip)
5967                 audit_log_nft_set_reset(dump_ctx->ctx.table, cb->seq,
5968                                         cb->args[0] - skip);
5969
5970         mutex_unlock(&nft_net->commit_mutex);
5971
5972         return ret;
5973 }
5974
5975 static int nf_tables_dump_set_start(struct netlink_callback *cb)
5976 {
5977         struct nft_set_dump_ctx *dump_ctx = cb->data;
5978
5979         cb->data = kmemdup(dump_ctx, sizeof(*dump_ctx), GFP_ATOMIC);
5980
5981         return cb->data ? 0 : -ENOMEM;
5982 }
5983
5984 static int nf_tables_dump_set_done(struct netlink_callback *cb)
5985 {
5986         kfree(cb->data);
5987         return 0;
5988 }
5989
5990 static int nf_tables_fill_setelem_info(struct sk_buff *skb,
5991                                        const struct nft_ctx *ctx, u32 seq,
5992                                        u32 portid, int event, u16 flags,
5993                                        const struct nft_set *set,
5994                                        const struct nft_elem_priv *elem_priv,
5995                                        bool reset)
5996 {
5997         struct nlmsghdr *nlh;
5998         struct nlattr *nest;
5999         int err;
6000
6001         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
6002         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
6003                            NFNETLINK_V0, nft_base_seq(ctx->net));
6004         if (!nlh)
6005                 goto nla_put_failure;
6006
6007         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
6008                 goto nla_put_failure;
6009         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
6010                 goto nla_put_failure;
6011
6012         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
6013         if (nest == NULL)
6014                 goto nla_put_failure;
6015
6016         err = nf_tables_fill_setelem(skb, set, elem_priv, reset);
6017         if (err < 0)
6018                 goto nla_put_failure;
6019
6020         nla_nest_end(skb, nest);
6021
6022         nlmsg_end(skb, nlh);
6023         return 0;
6024
6025 nla_put_failure:
6026         nlmsg_trim(skb, nlh);
6027         return -1;
6028 }
6029
6030 static int nft_setelem_parse_flags(const struct nft_set *set,
6031                                    const struct nlattr *attr, u32 *flags)
6032 {
6033         if (attr == NULL)
6034                 return 0;
6035
6036         *flags = ntohl(nla_get_be32(attr));
6037         if (*flags & ~(NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
6038                 return -EOPNOTSUPP;
6039         if (!(set->flags & NFT_SET_INTERVAL) &&
6040             *flags & NFT_SET_ELEM_INTERVAL_END)
6041                 return -EINVAL;
6042         if ((*flags & (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL)) ==
6043             (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
6044                 return -EINVAL;
6045
6046         return 0;
6047 }
6048
6049 static int nft_setelem_parse_key(struct nft_ctx *ctx, const struct nft_set *set,
6050                                  struct nft_data *key, struct nlattr *attr)
6051 {
6052         struct nft_data_desc desc = {
6053                 .type   = NFT_DATA_VALUE,
6054                 .size   = NFT_DATA_VALUE_MAXLEN,
6055                 .len    = set->klen,
6056         };
6057
6058         return nft_data_init(ctx, key, &desc, attr);
6059 }
6060
6061 static int nft_setelem_parse_data(struct nft_ctx *ctx, struct nft_set *set,
6062                                   struct nft_data_desc *desc,
6063                                   struct nft_data *data,
6064                                   struct nlattr *attr)
6065 {
6066         u32 dtype;
6067
6068         if (set->dtype == NFT_DATA_VERDICT)
6069                 dtype = NFT_DATA_VERDICT;
6070         else
6071                 dtype = NFT_DATA_VALUE;
6072
6073         desc->type = dtype;
6074         desc->size = NFT_DATA_VALUE_MAXLEN;
6075         desc->len = set->dlen;
6076         desc->flags = NFT_DATA_DESC_SETELEM;
6077
6078         return nft_data_init(ctx, data, desc, attr);
6079 }
6080
6081 static void *nft_setelem_catchall_get(const struct net *net,
6082                                       const struct nft_set *set)
6083 {
6084         struct nft_set_elem_catchall *catchall;
6085         u8 genmask = nft_genmask_cur(net);
6086         struct nft_set_ext *ext;
6087         void *priv = NULL;
6088
6089         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
6090                 ext = nft_set_elem_ext(set, catchall->elem);
6091                 if (!nft_set_elem_active(ext, genmask) ||
6092                     nft_set_elem_expired(ext))
6093                         continue;
6094
6095                 priv = catchall->elem;
6096                 break;
6097         }
6098
6099         return priv;
6100 }
6101
6102 static int nft_setelem_get(struct nft_ctx *ctx, const struct nft_set *set,
6103                            struct nft_set_elem *elem, u32 flags)
6104 {
6105         void *priv;
6106
6107         if (!(flags & NFT_SET_ELEM_CATCHALL)) {
6108                 priv = set->ops->get(ctx->net, set, elem, flags);
6109                 if (IS_ERR(priv))
6110                         return PTR_ERR(priv);
6111         } else {
6112                 priv = nft_setelem_catchall_get(ctx->net, set);
6113                 if (!priv)
6114                         return -ENOENT;
6115         }
6116         elem->priv = priv;
6117
6118         return 0;
6119 }
6120
6121 static int nft_get_set_elem(struct nft_ctx *ctx, const struct nft_set *set,
6122                             const struct nlattr *attr, bool reset)
6123 {
6124         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
6125         struct nft_set_elem elem;
6126         struct sk_buff *skb;
6127         uint32_t flags = 0;
6128         int err;
6129
6130         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
6131                                           nft_set_elem_policy, NULL);
6132         if (err < 0)
6133                 return err;
6134
6135         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
6136         if (err < 0)
6137                 return err;
6138
6139         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
6140                 return -EINVAL;
6141
6142         if (nla[NFTA_SET_ELEM_KEY]) {
6143                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
6144                                             nla[NFTA_SET_ELEM_KEY]);
6145                 if (err < 0)
6146                         return err;
6147         }
6148
6149         if (nla[NFTA_SET_ELEM_KEY_END]) {
6150                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
6151                                             nla[NFTA_SET_ELEM_KEY_END]);
6152                 if (err < 0)
6153                         return err;
6154         }
6155
6156         err = nft_setelem_get(ctx, set, &elem, flags);
6157         if (err < 0)
6158                 return err;
6159
6160         err = -ENOMEM;
6161         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_ATOMIC);
6162         if (skb == NULL)
6163                 return err;
6164
6165         err = nf_tables_fill_setelem_info(skb, ctx, ctx->seq, ctx->portid,
6166                                           NFT_MSG_NEWSETELEM, 0, set, elem.priv,
6167                                           reset);
6168         if (err < 0)
6169                 goto err_fill_setelem;
6170
6171         return nfnetlink_unicast(skb, ctx->net, ctx->portid);
6172
6173 err_fill_setelem:
6174         kfree_skb(skb);
6175         return err;
6176 }
6177
6178 static int nft_set_dump_ctx_init(struct nft_set_dump_ctx *dump_ctx,
6179                                  const struct sk_buff *skb,
6180                                  const struct nfnl_info *info,
6181                                  const struct nlattr * const nla[],
6182                                  bool reset)
6183 {
6184         struct netlink_ext_ack *extack = info->extack;
6185         u8 genmask = nft_genmask_cur(info->net);
6186         u8 family = info->nfmsg->nfgen_family;
6187         struct net *net = info->net;
6188         struct nft_table *table;
6189         struct nft_set *set;
6190
6191         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
6192                                  genmask, 0);
6193         if (IS_ERR(table)) {
6194                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
6195                 return PTR_ERR(table);
6196         }
6197
6198         set = nft_set_lookup(table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
6199         if (IS_ERR(set)) {
6200                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_SET]);
6201                 return PTR_ERR(set);
6202         }
6203
6204         nft_ctx_init(&dump_ctx->ctx, net, skb,
6205                      info->nlh, family, table, NULL, nla);
6206         dump_ctx->set = set;
6207         dump_ctx->reset = reset;
6208         return 0;
6209 }
6210
6211 /* called with rcu_read_lock held */
6212 static int nf_tables_getsetelem(struct sk_buff *skb,
6213                                 const struct nfnl_info *info,
6214                                 const struct nlattr * const nla[])
6215 {
6216         struct netlink_ext_ack *extack = info->extack;
6217         struct nft_set_dump_ctx dump_ctx;
6218         struct nlattr *attr;
6219         int rem, err = 0;
6220
6221         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
6222                 struct netlink_dump_control c = {
6223                         .start = nf_tables_dump_set_start,
6224                         .dump = nf_tables_dump_set,
6225                         .done = nf_tables_dump_set_done,
6226                         .module = THIS_MODULE,
6227                 };
6228
6229                 err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, false);
6230                 if (err)
6231                         return err;
6232
6233                 c.data = &dump_ctx;
6234                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
6235         }
6236
6237         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
6238                 return -EINVAL;
6239
6240         err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, false);
6241         if (err)
6242                 return err;
6243
6244         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
6245                 err = nft_get_set_elem(&dump_ctx.ctx, dump_ctx.set, attr, false);
6246                 if (err < 0) {
6247                         NL_SET_BAD_ATTR(extack, attr);
6248                         break;
6249                 }
6250         }
6251
6252         return err;
6253 }
6254
6255 static int nf_tables_getsetelem_reset(struct sk_buff *skb,
6256                                       const struct nfnl_info *info,
6257                                       const struct nlattr * const nla[])
6258 {
6259         struct nftables_pernet *nft_net = nft_pernet(info->net);
6260         struct netlink_ext_ack *extack = info->extack;
6261         struct nft_set_dump_ctx dump_ctx;
6262         int rem, err = 0, nelems = 0;
6263         struct nlattr *attr;
6264
6265         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
6266                 struct netlink_dump_control c = {
6267                         .start = nf_tables_dump_set_start,
6268                         .dump = nf_tables_dumpreset_set,
6269                         .done = nf_tables_dump_set_done,
6270                         .module = THIS_MODULE,
6271                 };
6272
6273                 err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, true);
6274                 if (err)
6275                         return err;
6276
6277                 c.data = &dump_ctx;
6278                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
6279         }
6280
6281         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
6282                 return -EINVAL;
6283
6284         if (!try_module_get(THIS_MODULE))
6285                 return -EINVAL;
6286         rcu_read_unlock();
6287         mutex_lock(&nft_net->commit_mutex);
6288         rcu_read_lock();
6289
6290         err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, true);
6291         if (err)
6292                 goto out_unlock;
6293
6294         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
6295                 err = nft_get_set_elem(&dump_ctx.ctx, dump_ctx.set, attr, true);
6296                 if (err < 0) {
6297                         NL_SET_BAD_ATTR(extack, attr);
6298                         break;
6299                 }
6300                 nelems++;
6301         }
6302         audit_log_nft_set_reset(dump_ctx.ctx.table, nft_net->base_seq, nelems);
6303
6304 out_unlock:
6305         rcu_read_unlock();
6306         mutex_unlock(&nft_net->commit_mutex);
6307         rcu_read_lock();
6308         module_put(THIS_MODULE);
6309
6310         return err;
6311 }
6312
6313 static void nf_tables_setelem_notify(const struct nft_ctx *ctx,
6314                                      const struct nft_set *set,
6315                                      const struct nft_elem_priv *elem_priv,
6316                                      int event)
6317 {
6318         struct nftables_pernet *nft_net;
6319         struct net *net = ctx->net;
6320         u32 portid = ctx->portid;
6321         struct sk_buff *skb;
6322         u16 flags = 0;
6323         int err;
6324
6325         if (!ctx->report && !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
6326                 return;
6327
6328         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
6329         if (skb == NULL)
6330                 goto err;
6331
6332         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
6333                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
6334
6335         err = nf_tables_fill_setelem_info(skb, ctx, 0, portid, event, flags,
6336                                           set, elem_priv, false);
6337         if (err < 0) {
6338                 kfree_skb(skb);
6339                 goto err;
6340         }
6341
6342         nft_net = nft_pernet(net);
6343         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
6344         return;
6345 err:
6346         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
6347 }
6348
6349 static struct nft_trans *nft_trans_elem_alloc(struct nft_ctx *ctx,
6350                                               int msg_type,
6351                                               struct nft_set *set)
6352 {
6353         struct nft_trans *trans;
6354
6355         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_elem));
6356         if (trans == NULL)
6357                 return NULL;
6358
6359         nft_trans_elem_set(trans) = set;
6360         return trans;
6361 }
6362
6363 struct nft_expr *nft_set_elem_expr_alloc(const struct nft_ctx *ctx,
6364                                          const struct nft_set *set,
6365                                          const struct nlattr *attr)
6366 {
6367         struct nft_expr *expr;
6368         int err;
6369
6370         expr = nft_expr_init(ctx, attr);
6371         if (IS_ERR(expr))
6372                 return expr;
6373
6374         err = -EOPNOTSUPP;
6375         if (expr->ops->type->flags & NFT_EXPR_GC) {
6376                 if (set->flags & NFT_SET_TIMEOUT)
6377                         goto err_set_elem_expr;
6378                 if (!set->ops->gc_init)
6379                         goto err_set_elem_expr;
6380                 set->ops->gc_init(set);
6381         }
6382
6383         return expr;
6384
6385 err_set_elem_expr:
6386         nft_expr_destroy(ctx, expr);
6387         return ERR_PTR(err);
6388 }
6389
6390 static int nft_set_ext_check(const struct nft_set_ext_tmpl *tmpl, u8 id, u32 len)
6391 {
6392         len += nft_set_ext_types[id].len;
6393         if (len > tmpl->ext_len[id] ||
6394             len > U8_MAX)
6395                 return -1;
6396
6397         return 0;
6398 }
6399
6400 static int nft_set_ext_memcpy(const struct nft_set_ext_tmpl *tmpl, u8 id,
6401                               void *to, const void *from, u32 len)
6402 {
6403         if (nft_set_ext_check(tmpl, id, len) < 0)
6404                 return -1;
6405
6406         memcpy(to, from, len);
6407
6408         return 0;
6409 }
6410
6411 struct nft_elem_priv *nft_set_elem_init(const struct nft_set *set,
6412                                         const struct nft_set_ext_tmpl *tmpl,
6413                                         const u32 *key, const u32 *key_end,
6414                                         const u32 *data,
6415                                         u64 timeout, u64 expiration, gfp_t gfp)
6416 {
6417         struct nft_set_ext *ext;
6418         void *elem;
6419
6420         elem = kzalloc(set->ops->elemsize + tmpl->len, gfp);
6421         if (elem == NULL)
6422                 return ERR_PTR(-ENOMEM);
6423
6424         ext = nft_set_elem_ext(set, elem);
6425         nft_set_ext_init(ext, tmpl);
6426
6427         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
6428             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY,
6429                                nft_set_ext_key(ext), key, set->klen) < 0)
6430                 goto err_ext_check;
6431
6432         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
6433             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY_END,
6434                                nft_set_ext_key_end(ext), key_end, set->klen) < 0)
6435                 goto err_ext_check;
6436
6437         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
6438             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_DATA,
6439                                nft_set_ext_data(ext), data, set->dlen) < 0)
6440                 goto err_ext_check;
6441
6442         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
6443                 *nft_set_ext_expiration(ext) = get_jiffies_64() + expiration;
6444                 if (expiration == 0)
6445                         *nft_set_ext_expiration(ext) += timeout;
6446         }
6447         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT))
6448                 *nft_set_ext_timeout(ext) = timeout;
6449
6450         return elem;
6451
6452 err_ext_check:
6453         kfree(elem);
6454
6455         return ERR_PTR(-EINVAL);
6456 }
6457
6458 static void __nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
6459                                         struct nft_expr *expr)
6460 {
6461         if (expr->ops->destroy_clone) {
6462                 expr->ops->destroy_clone(ctx, expr);
6463                 module_put(expr->ops->type->owner);
6464         } else {
6465                 nf_tables_expr_destroy(ctx, expr);
6466         }
6467 }
6468
6469 static void nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
6470                                       struct nft_set_elem_expr *elem_expr)
6471 {
6472         struct nft_expr *expr;
6473         u32 size;
6474
6475         nft_setelem_expr_foreach(expr, elem_expr, size)
6476                 __nft_set_elem_expr_destroy(ctx, expr);
6477 }
6478
6479 /* Drop references and destroy. Called from gc, dynset and abort path. */
6480 void nft_set_elem_destroy(const struct nft_set *set,
6481                           const struct nft_elem_priv *elem_priv,
6482                           bool destroy_expr)
6483 {
6484         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
6485         struct nft_ctx ctx = {
6486                 .net    = read_pnet(&set->net),
6487                 .family = set->table->family,
6488         };
6489
6490         nft_data_release(nft_set_ext_key(ext), NFT_DATA_VALUE);
6491         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
6492                 nft_data_release(nft_set_ext_data(ext), set->dtype);
6493         if (destroy_expr && nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
6494                 nft_set_elem_expr_destroy(&ctx, nft_set_ext_expr(ext));
6495         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
6496                 nft_use_dec(&(*nft_set_ext_obj(ext))->use);
6497
6498         kfree(elem_priv);
6499 }
6500 EXPORT_SYMBOL_GPL(nft_set_elem_destroy);
6501
6502 /* Destroy element. References have been already dropped in the preparation
6503  * path via nft_setelem_data_deactivate().
6504  */
6505 void nf_tables_set_elem_destroy(const struct nft_ctx *ctx,
6506                                 const struct nft_set *set,
6507                                 const struct nft_elem_priv *elem_priv)
6508 {
6509         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
6510
6511         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
6512                 nft_set_elem_expr_destroy(ctx, nft_set_ext_expr(ext));
6513
6514         kfree(elem_priv);
6515 }
6516
6517 int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set,
6518                             struct nft_expr *expr_array[])
6519 {
6520         struct nft_expr *expr;
6521         int err, i, k;
6522
6523         for (i = 0; i < set->num_exprs; i++) {
6524                 expr = kzalloc(set->exprs[i]->ops->size, GFP_KERNEL_ACCOUNT);
6525                 if (!expr)
6526                         goto err_expr;
6527
6528                 err = nft_expr_clone(expr, set->exprs[i], GFP_KERNEL_ACCOUNT);
6529                 if (err < 0) {
6530                         kfree(expr);
6531                         goto err_expr;
6532                 }
6533                 expr_array[i] = expr;
6534         }
6535
6536         return 0;
6537
6538 err_expr:
6539         for (k = i - 1; k >= 0; k--)
6540                 nft_expr_destroy(ctx, expr_array[k]);
6541
6542         return -ENOMEM;
6543 }
6544
6545 static int nft_set_elem_expr_setup(struct nft_ctx *ctx,
6546                                    const struct nft_set_ext_tmpl *tmpl,
6547                                    const struct nft_set_ext *ext,
6548                                    struct nft_expr *expr_array[],
6549                                    u32 num_exprs)
6550 {
6551         struct nft_set_elem_expr *elem_expr = nft_set_ext_expr(ext);
6552         u32 len = sizeof(struct nft_set_elem_expr);
6553         struct nft_expr *expr;
6554         int i, err;
6555
6556         if (num_exprs == 0)
6557                 return 0;
6558
6559         for (i = 0; i < num_exprs; i++)
6560                 len += expr_array[i]->ops->size;
6561
6562         if (nft_set_ext_check(tmpl, NFT_SET_EXT_EXPRESSIONS, len) < 0)
6563                 return -EINVAL;
6564
6565         for (i = 0; i < num_exprs; i++) {
6566                 expr = nft_setelem_expr_at(elem_expr, elem_expr->size);
6567                 err = nft_expr_clone(expr, expr_array[i], GFP_KERNEL_ACCOUNT);
6568                 if (err < 0)
6569                         goto err_elem_expr_setup;
6570
6571                 elem_expr->size += expr_array[i]->ops->size;
6572                 nft_expr_destroy(ctx, expr_array[i]);
6573                 expr_array[i] = NULL;
6574         }
6575
6576         return 0;
6577
6578 err_elem_expr_setup:
6579         for (; i < num_exprs; i++) {
6580                 nft_expr_destroy(ctx, expr_array[i]);
6581                 expr_array[i] = NULL;
6582         }
6583
6584         return -ENOMEM;
6585 }
6586
6587 struct nft_set_ext *nft_set_catchall_lookup(const struct net *net,
6588                                             const struct nft_set *set)
6589 {
6590         struct nft_set_elem_catchall *catchall;
6591         u8 genmask = nft_genmask_cur(net);
6592         struct nft_set_ext *ext;
6593
6594         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
6595                 ext = nft_set_elem_ext(set, catchall->elem);
6596                 if (nft_set_elem_active(ext, genmask) &&
6597                     !nft_set_elem_expired(ext) &&
6598                     !nft_set_elem_is_dead(ext))
6599                         return ext;
6600         }
6601
6602         return NULL;
6603 }
6604 EXPORT_SYMBOL_GPL(nft_set_catchall_lookup);
6605
6606 static int nft_setelem_catchall_insert(const struct net *net,
6607                                        struct nft_set *set,
6608                                        const struct nft_set_elem *elem,
6609                                        struct nft_elem_priv **priv)
6610 {
6611         struct nft_set_elem_catchall *catchall;
6612         u8 genmask = nft_genmask_next(net);
6613         struct nft_set_ext *ext;
6614
6615         list_for_each_entry(catchall, &set->catchall_list, list) {
6616                 ext = nft_set_elem_ext(set, catchall->elem);
6617                 if (nft_set_elem_active(ext, genmask)) {
6618                         *priv = catchall->elem;
6619                         return -EEXIST;
6620                 }
6621         }
6622
6623         catchall = kmalloc(sizeof(*catchall), GFP_KERNEL);
6624         if (!catchall)
6625                 return -ENOMEM;
6626
6627         catchall->elem = elem->priv;
6628         list_add_tail_rcu(&catchall->list, &set->catchall_list);
6629
6630         return 0;
6631 }
6632
6633 static int nft_setelem_insert(const struct net *net,
6634                               struct nft_set *set,
6635                               const struct nft_set_elem *elem,
6636                               struct nft_elem_priv **elem_priv,
6637                               unsigned int flags)
6638 {
6639         int ret;
6640
6641         if (flags & NFT_SET_ELEM_CATCHALL)
6642                 ret = nft_setelem_catchall_insert(net, set, elem, elem_priv);
6643         else
6644                 ret = set->ops->insert(net, set, elem, elem_priv);
6645
6646         return ret;
6647 }
6648
6649 static bool nft_setelem_is_catchall(const struct nft_set *set,
6650                                     const struct nft_elem_priv *elem_priv)
6651 {
6652         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
6653
6654         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
6655             *nft_set_ext_flags(ext) & NFT_SET_ELEM_CATCHALL)
6656                 return true;
6657
6658         return false;
6659 }
6660
6661 static void nft_setelem_activate(struct net *net, struct nft_set *set,
6662                                  struct nft_elem_priv *elem_priv)
6663 {
6664         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
6665
6666         if (nft_setelem_is_catchall(set, elem_priv)) {
6667                 nft_clear(net, ext);
6668         } else {
6669                 set->ops->activate(net, set, elem_priv);
6670         }
6671 }
6672
6673 static int nft_setelem_catchall_deactivate(const struct net *net,
6674                                            struct nft_set *set,
6675                                            struct nft_set_elem *elem)
6676 {
6677         struct nft_set_elem_catchall *catchall;
6678         struct nft_set_ext *ext;
6679
6680         list_for_each_entry(catchall, &set->catchall_list, list) {
6681                 ext = nft_set_elem_ext(set, catchall->elem);
6682                 if (!nft_is_active_next(net, ext))
6683                         continue;
6684
6685                 kfree(elem->priv);
6686                 elem->priv = catchall->elem;
6687                 nft_set_elem_change_active(net, set, ext);
6688                 return 0;
6689         }
6690
6691         return -ENOENT;
6692 }
6693
6694 static int __nft_setelem_deactivate(const struct net *net,
6695                                     struct nft_set *set,
6696                                     struct nft_set_elem *elem)
6697 {
6698         void *priv;
6699
6700         priv = set->ops->deactivate(net, set, elem);
6701         if (!priv)
6702                 return -ENOENT;
6703
6704         kfree(elem->priv);
6705         elem->priv = priv;
6706         set->ndeact++;
6707
6708         return 0;
6709 }
6710
6711 static int nft_setelem_deactivate(const struct net *net,
6712                                   struct nft_set *set,
6713                                   struct nft_set_elem *elem, u32 flags)
6714 {
6715         int ret;
6716
6717         if (flags & NFT_SET_ELEM_CATCHALL)
6718                 ret = nft_setelem_catchall_deactivate(net, set, elem);
6719         else
6720                 ret = __nft_setelem_deactivate(net, set, elem);
6721
6722         return ret;
6723 }
6724
6725 static void nft_setelem_catchall_destroy(struct nft_set_elem_catchall *catchall)
6726 {
6727         list_del_rcu(&catchall->list);
6728         kfree_rcu(catchall, rcu);
6729 }
6730
6731 static void nft_setelem_catchall_remove(const struct net *net,
6732                                         const struct nft_set *set,
6733                                         struct nft_elem_priv *elem_priv)
6734 {
6735         struct nft_set_elem_catchall *catchall, *next;
6736
6737         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
6738                 if (catchall->elem == elem_priv) {
6739                         nft_setelem_catchall_destroy(catchall);
6740                         break;
6741                 }
6742         }
6743 }
6744
6745 static void nft_setelem_remove(const struct net *net,
6746                                const struct nft_set *set,
6747                                struct nft_elem_priv *elem_priv)
6748 {
6749         if (nft_setelem_is_catchall(set, elem_priv))
6750                 nft_setelem_catchall_remove(net, set, elem_priv);
6751         else
6752                 set->ops->remove(net, set, elem_priv);
6753 }
6754
6755 static bool nft_setelem_valid_key_end(const struct nft_set *set,
6756                                       struct nlattr **nla, u32 flags)
6757 {
6758         if ((set->flags & (NFT_SET_CONCAT | NFT_SET_INTERVAL)) ==
6759                           (NFT_SET_CONCAT | NFT_SET_INTERVAL)) {
6760                 if (flags & NFT_SET_ELEM_INTERVAL_END)
6761                         return false;
6762
6763                 if (nla[NFTA_SET_ELEM_KEY_END] &&
6764                     flags & NFT_SET_ELEM_CATCHALL)
6765                         return false;
6766         } else {
6767                 if (nla[NFTA_SET_ELEM_KEY_END])
6768                         return false;
6769         }
6770
6771         return true;
6772 }
6773
6774 static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set,
6775                             const struct nlattr *attr, u32 nlmsg_flags)
6776 {
6777         struct nft_expr *expr_array[NFT_SET_EXPR_MAX] = {};
6778         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
6779         u8 genmask = nft_genmask_next(ctx->net);
6780         u32 flags = 0, size = 0, num_exprs = 0;
6781         struct nft_set_ext_tmpl tmpl;
6782         struct nft_set_ext *ext, *ext2;
6783         struct nft_set_elem elem;
6784         struct nft_set_binding *binding;
6785         struct nft_elem_priv *elem_priv;
6786         struct nft_object *obj = NULL;
6787         struct nft_userdata *udata;
6788         struct nft_data_desc desc;
6789         enum nft_registers dreg;
6790         struct nft_trans *trans;
6791         u64 expiration;
6792         u64 timeout;
6793         int err, i;
6794         u8 ulen;
6795
6796         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
6797                                           nft_set_elem_policy, NULL);
6798         if (err < 0)
6799                 return err;
6800
6801         nft_set_ext_prepare(&tmpl);
6802
6803         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
6804         if (err < 0)
6805                 return err;
6806
6807         if (((flags & NFT_SET_ELEM_CATCHALL) && nla[NFTA_SET_ELEM_KEY]) ||
6808             (!(flags & NFT_SET_ELEM_CATCHALL) && !nla[NFTA_SET_ELEM_KEY]))
6809                 return -EINVAL;
6810
6811         if (flags != 0) {
6812                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
6813                 if (err < 0)
6814                         return err;
6815         }
6816
6817         if (set->flags & NFT_SET_MAP) {
6818                 if (nla[NFTA_SET_ELEM_DATA] == NULL &&
6819                     !(flags & NFT_SET_ELEM_INTERVAL_END))
6820                         return -EINVAL;
6821         } else {
6822                 if (nla[NFTA_SET_ELEM_DATA] != NULL)
6823                         return -EINVAL;
6824         }
6825
6826         if (set->flags & NFT_SET_OBJECT) {
6827                 if (!nla[NFTA_SET_ELEM_OBJREF] &&
6828                     !(flags & NFT_SET_ELEM_INTERVAL_END))
6829                         return -EINVAL;
6830         } else {
6831                 if (nla[NFTA_SET_ELEM_OBJREF])
6832                         return -EINVAL;
6833         }
6834
6835         if (!nft_setelem_valid_key_end(set, nla, flags))
6836                 return -EINVAL;
6837
6838         if ((flags & NFT_SET_ELEM_INTERVAL_END) &&
6839              (nla[NFTA_SET_ELEM_DATA] ||
6840               nla[NFTA_SET_ELEM_OBJREF] ||
6841               nla[NFTA_SET_ELEM_TIMEOUT] ||
6842               nla[NFTA_SET_ELEM_EXPIRATION] ||
6843               nla[NFTA_SET_ELEM_USERDATA] ||
6844               nla[NFTA_SET_ELEM_EXPR] ||
6845               nla[NFTA_SET_ELEM_KEY_END] ||
6846               nla[NFTA_SET_ELEM_EXPRESSIONS]))
6847                 return -EINVAL;
6848
6849         timeout = 0;
6850         if (nla[NFTA_SET_ELEM_TIMEOUT] != NULL) {
6851                 if (!(set->flags & NFT_SET_TIMEOUT))
6852                         return -EINVAL;
6853                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_TIMEOUT],
6854                                             &timeout);
6855                 if (err)
6856                         return err;
6857         } else if (set->flags & NFT_SET_TIMEOUT &&
6858                    !(flags & NFT_SET_ELEM_INTERVAL_END)) {
6859                 timeout = READ_ONCE(set->timeout);
6860         }
6861
6862         expiration = 0;
6863         if (nla[NFTA_SET_ELEM_EXPIRATION] != NULL) {
6864                 if (!(set->flags & NFT_SET_TIMEOUT))
6865                         return -EINVAL;
6866                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_EXPIRATION],
6867                                             &expiration);
6868                 if (err)
6869                         return err;
6870         }
6871
6872         if (nla[NFTA_SET_ELEM_EXPR]) {
6873                 struct nft_expr *expr;
6874
6875                 if (set->num_exprs && set->num_exprs != 1)
6876                         return -EOPNOTSUPP;
6877
6878                 expr = nft_set_elem_expr_alloc(ctx, set,
6879                                                nla[NFTA_SET_ELEM_EXPR]);
6880                 if (IS_ERR(expr))
6881                         return PTR_ERR(expr);
6882
6883                 expr_array[0] = expr;
6884                 num_exprs = 1;
6885
6886                 if (set->num_exprs && set->exprs[0]->ops != expr->ops) {
6887                         err = -EOPNOTSUPP;
6888                         goto err_set_elem_expr;
6889                 }
6890         } else if (nla[NFTA_SET_ELEM_EXPRESSIONS]) {
6891                 struct nft_expr *expr;
6892                 struct nlattr *tmp;
6893                 int left;
6894
6895                 i = 0;
6896                 nla_for_each_nested(tmp, nla[NFTA_SET_ELEM_EXPRESSIONS], left) {
6897                         if (i == NFT_SET_EXPR_MAX ||
6898                             (set->num_exprs && set->num_exprs == i)) {
6899                                 err = -E2BIG;
6900                                 goto err_set_elem_expr;
6901                         }
6902                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
6903                                 err = -EINVAL;
6904                                 goto err_set_elem_expr;
6905                         }
6906                         expr = nft_set_elem_expr_alloc(ctx, set, tmp);
6907                         if (IS_ERR(expr)) {
6908                                 err = PTR_ERR(expr);
6909                                 goto err_set_elem_expr;
6910                         }
6911                         expr_array[i] = expr;
6912                         num_exprs++;
6913
6914                         if (set->num_exprs && expr->ops != set->exprs[i]->ops) {
6915                                 err = -EOPNOTSUPP;
6916                                 goto err_set_elem_expr;
6917                         }
6918                         i++;
6919                 }
6920                 if (set->num_exprs && set->num_exprs != i) {
6921                         err = -EOPNOTSUPP;
6922                         goto err_set_elem_expr;
6923                 }
6924         } else if (set->num_exprs > 0 &&
6925                    !(flags & NFT_SET_ELEM_INTERVAL_END)) {
6926                 err = nft_set_elem_expr_clone(ctx, set, expr_array);
6927                 if (err < 0)
6928                         goto err_set_elem_expr_clone;
6929
6930                 num_exprs = set->num_exprs;
6931         }
6932
6933         if (nla[NFTA_SET_ELEM_KEY]) {
6934                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
6935                                             nla[NFTA_SET_ELEM_KEY]);
6936                 if (err < 0)
6937                         goto err_set_elem_expr;
6938
6939                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
6940                 if (err < 0)
6941                         goto err_parse_key;
6942         }
6943
6944         if (nla[NFTA_SET_ELEM_KEY_END]) {
6945                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
6946                                             nla[NFTA_SET_ELEM_KEY_END]);
6947                 if (err < 0)
6948                         goto err_parse_key;
6949
6950                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
6951                 if (err < 0)
6952                         goto err_parse_key_end;
6953         }
6954
6955         if (timeout > 0) {
6956                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_EXPIRATION);
6957                 if (err < 0)
6958                         goto err_parse_key_end;
6959
6960                 if (timeout != READ_ONCE(set->timeout)) {
6961                         err = nft_set_ext_add(&tmpl, NFT_SET_EXT_TIMEOUT);
6962                         if (err < 0)
6963                                 goto err_parse_key_end;
6964                 }
6965         }
6966
6967         if (num_exprs) {
6968                 for (i = 0; i < num_exprs; i++)
6969                         size += expr_array[i]->ops->size;
6970
6971                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_EXPRESSIONS,
6972                                              sizeof(struct nft_set_elem_expr) + size);
6973                 if (err < 0)
6974                         goto err_parse_key_end;
6975         }
6976
6977         if (nla[NFTA_SET_ELEM_OBJREF] != NULL) {
6978                 obj = nft_obj_lookup(ctx->net, ctx->table,
6979                                      nla[NFTA_SET_ELEM_OBJREF],
6980                                      set->objtype, genmask);
6981                 if (IS_ERR(obj)) {
6982                         err = PTR_ERR(obj);
6983                         obj = NULL;
6984                         goto err_parse_key_end;
6985                 }
6986
6987                 if (!nft_use_inc(&obj->use)) {
6988                         err = -EMFILE;
6989                         obj = NULL;
6990                         goto err_parse_key_end;
6991                 }
6992
6993                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_OBJREF);
6994                 if (err < 0)
6995                         goto err_parse_key_end;
6996         }
6997
6998         if (nla[NFTA_SET_ELEM_DATA] != NULL) {
6999                 err = nft_setelem_parse_data(ctx, set, &desc, &elem.data.val,
7000                                              nla[NFTA_SET_ELEM_DATA]);
7001                 if (err < 0)
7002                         goto err_parse_key_end;
7003
7004                 dreg = nft_type_to_reg(set->dtype);
7005                 list_for_each_entry(binding, &set->bindings, list) {
7006                         struct nft_ctx bind_ctx = {
7007                                 .net    = ctx->net,
7008                                 .family = ctx->family,
7009                                 .table  = ctx->table,
7010                                 .chain  = (struct nft_chain *)binding->chain,
7011                         };
7012
7013                         if (!(binding->flags & NFT_SET_MAP))
7014                                 continue;
7015
7016                         err = nft_validate_register_store(&bind_ctx, dreg,
7017                                                           &elem.data.val,
7018                                                           desc.type, desc.len);
7019                         if (err < 0)
7020                                 goto err_parse_data;
7021
7022                         if (desc.type == NFT_DATA_VERDICT &&
7023                             (elem.data.val.verdict.code == NFT_GOTO ||
7024                              elem.data.val.verdict.code == NFT_JUMP))
7025                                 nft_validate_state_update(ctx->table,
7026                                                           NFT_VALIDATE_NEED);
7027                 }
7028
7029                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_DATA, desc.len);
7030                 if (err < 0)
7031                         goto err_parse_data;
7032         }
7033
7034         /* The full maximum length of userdata can exceed the maximum
7035          * offset value (U8_MAX) for following extensions, therefor it
7036          * must be the last extension added.
7037          */
7038         ulen = 0;
7039         if (nla[NFTA_SET_ELEM_USERDATA] != NULL) {
7040                 ulen = nla_len(nla[NFTA_SET_ELEM_USERDATA]);
7041                 if (ulen > 0) {
7042                         err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_USERDATA,
7043                                                      ulen);
7044                         if (err < 0)
7045                                 goto err_parse_data;
7046                 }
7047         }
7048
7049         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
7050                                       elem.key_end.val.data, elem.data.val.data,
7051                                       timeout, expiration, GFP_KERNEL_ACCOUNT);
7052         if (IS_ERR(elem.priv)) {
7053                 err = PTR_ERR(elem.priv);
7054                 goto err_parse_data;
7055         }
7056
7057         ext = nft_set_elem_ext(set, elem.priv);
7058         if (flags)
7059                 *nft_set_ext_flags(ext) = flags;
7060
7061         if (obj)
7062                 *nft_set_ext_obj(ext) = obj;
7063
7064         if (ulen > 0) {
7065                 if (nft_set_ext_check(&tmpl, NFT_SET_EXT_USERDATA, ulen) < 0) {
7066                         err = -EINVAL;
7067                         goto err_elem_free;
7068                 }
7069                 udata = nft_set_ext_userdata(ext);
7070                 udata->len = ulen - 1;
7071                 nla_memcpy(&udata->data, nla[NFTA_SET_ELEM_USERDATA], ulen);
7072         }
7073         err = nft_set_elem_expr_setup(ctx, &tmpl, ext, expr_array, num_exprs);
7074         if (err < 0)
7075                 goto err_elem_free;
7076
7077         trans = nft_trans_elem_alloc(ctx, NFT_MSG_NEWSETELEM, set);
7078         if (trans == NULL) {
7079                 err = -ENOMEM;
7080                 goto err_elem_free;
7081         }
7082
7083         ext->genmask = nft_genmask_cur(ctx->net);
7084
7085         err = nft_setelem_insert(ctx->net, set, &elem, &elem_priv, flags);
7086         if (err) {
7087                 if (err == -EEXIST) {
7088                         ext2 = nft_set_elem_ext(set, elem_priv);
7089                         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) ^
7090                             nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) ||
7091                             nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) ^
7092                             nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF))
7093                                 goto err_element_clash;
7094                         if ((nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
7095                              nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) &&
7096                              memcmp(nft_set_ext_data(ext),
7097                                     nft_set_ext_data(ext2), set->dlen) != 0) ||
7098                             (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
7099                              nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF) &&
7100                              *nft_set_ext_obj(ext) != *nft_set_ext_obj(ext2)))
7101                                 goto err_element_clash;
7102                         else if (!(nlmsg_flags & NLM_F_EXCL))
7103                                 err = 0;
7104                 } else if (err == -ENOTEMPTY) {
7105                         /* ENOTEMPTY reports overlapping between this element
7106                          * and an existing one.
7107                          */
7108                         err = -EEXIST;
7109                 }
7110                 goto err_element_clash;
7111         }
7112
7113         if (!(flags & NFT_SET_ELEM_CATCHALL)) {
7114                 unsigned int max = set->size ? set->size + set->ndeact : UINT_MAX;
7115
7116                 if (!atomic_add_unless(&set->nelems, 1, max)) {
7117                         err = -ENFILE;
7118                         goto err_set_full;
7119                 }
7120         }
7121
7122         nft_trans_elem_priv(trans) = elem.priv;
7123         nft_trans_commit_list_add_tail(ctx->net, trans);
7124         return 0;
7125
7126 err_set_full:
7127         nft_setelem_remove(ctx->net, set, elem.priv);
7128 err_element_clash:
7129         kfree(trans);
7130 err_elem_free:
7131         nf_tables_set_elem_destroy(ctx, set, elem.priv);
7132 err_parse_data:
7133         if (nla[NFTA_SET_ELEM_DATA] != NULL)
7134                 nft_data_release(&elem.data.val, desc.type);
7135 err_parse_key_end:
7136         if (obj)
7137                 nft_use_dec_restore(&obj->use);
7138
7139         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
7140 err_parse_key:
7141         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
7142 err_set_elem_expr:
7143         for (i = 0; i < num_exprs && expr_array[i]; i++)
7144                 nft_expr_destroy(ctx, expr_array[i]);
7145 err_set_elem_expr_clone:
7146         return err;
7147 }
7148
7149 static int nf_tables_newsetelem(struct sk_buff *skb,
7150                                 const struct nfnl_info *info,
7151                                 const struct nlattr * const nla[])
7152 {
7153         struct netlink_ext_ack *extack = info->extack;
7154         u8 genmask = nft_genmask_next(info->net);
7155         u8 family = info->nfmsg->nfgen_family;
7156         struct net *net = info->net;
7157         const struct nlattr *attr;
7158         struct nft_table *table;
7159         struct nft_set *set;
7160         struct nft_ctx ctx;
7161         int rem, err;
7162
7163         if (nla[NFTA_SET_ELEM_LIST_ELEMENTS] == NULL)
7164                 return -EINVAL;
7165
7166         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
7167                                  genmask, NETLINK_CB(skb).portid);
7168         if (IS_ERR(table)) {
7169                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
7170                 return PTR_ERR(table);
7171         }
7172
7173         set = nft_set_lookup_global(net, table, nla[NFTA_SET_ELEM_LIST_SET],
7174                                     nla[NFTA_SET_ELEM_LIST_SET_ID], genmask);
7175         if (IS_ERR(set)) {
7176                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_SET]);
7177                 return PTR_ERR(set);
7178         }
7179
7180         if (!list_empty(&set->bindings) &&
7181             (set->flags & (NFT_SET_CONSTANT | NFT_SET_ANONYMOUS)))
7182                 return -EBUSY;
7183
7184         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7185
7186         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
7187                 err = nft_add_set_elem(&ctx, set, attr, info->nlh->nlmsg_flags);
7188                 if (err < 0) {
7189                         NL_SET_BAD_ATTR(extack, attr);
7190                         return err;
7191                 }
7192         }
7193
7194         if (table->validate_state == NFT_VALIDATE_DO)
7195                 return nft_table_validate(net, table);
7196
7197         return 0;
7198 }
7199
7200 /**
7201  *      nft_data_hold - hold a nft_data item
7202  *
7203  *      @data: struct nft_data to release
7204  *      @type: type of data
7205  *
7206  *      Hold a nft_data item. NFT_DATA_VALUE types can be silently discarded,
7207  *      NFT_DATA_VERDICT bumps the reference to chains in case of NFT_JUMP and
7208  *      NFT_GOTO verdicts. This function must be called on active data objects
7209  *      from the second phase of the commit protocol.
7210  */
7211 void nft_data_hold(const struct nft_data *data, enum nft_data_types type)
7212 {
7213         struct nft_chain *chain;
7214
7215         if (type == NFT_DATA_VERDICT) {
7216                 switch (data->verdict.code) {
7217                 case NFT_JUMP:
7218                 case NFT_GOTO:
7219                         chain = data->verdict.chain;
7220                         nft_use_inc_restore(&chain->use);
7221                         break;
7222                 }
7223         }
7224 }
7225
7226 static int nft_setelem_active_next(const struct net *net,
7227                                    const struct nft_set *set,
7228                                    struct nft_elem_priv *elem_priv)
7229 {
7230         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
7231         u8 genmask = nft_genmask_next(net);
7232
7233         return nft_set_elem_active(ext, genmask);
7234 }
7235
7236 static void nft_setelem_data_activate(const struct net *net,
7237                                       const struct nft_set *set,
7238                                       struct nft_elem_priv *elem_priv)
7239 {
7240         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
7241
7242         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
7243                 nft_data_hold(nft_set_ext_data(ext), set->dtype);
7244         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
7245                 nft_use_inc_restore(&(*nft_set_ext_obj(ext))->use);
7246 }
7247
7248 void nft_setelem_data_deactivate(const struct net *net,
7249                                  const struct nft_set *set,
7250                                  struct nft_elem_priv *elem_priv)
7251 {
7252         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
7253
7254         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
7255                 nft_data_release(nft_set_ext_data(ext), set->dtype);
7256         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
7257                 nft_use_dec(&(*nft_set_ext_obj(ext))->use);
7258 }
7259
7260 static int nft_del_setelem(struct nft_ctx *ctx, struct nft_set *set,
7261                            const struct nlattr *attr)
7262 {
7263         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
7264         struct nft_set_ext_tmpl tmpl;
7265         struct nft_set_elem elem;
7266         struct nft_set_ext *ext;
7267         struct nft_trans *trans;
7268         u32 flags = 0;
7269         int err;
7270
7271         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
7272                                           nft_set_elem_policy, NULL);
7273         if (err < 0)
7274                 return err;
7275
7276         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
7277         if (err < 0)
7278                 return err;
7279
7280         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
7281                 return -EINVAL;
7282
7283         if (!nft_setelem_valid_key_end(set, nla, flags))
7284                 return -EINVAL;
7285
7286         nft_set_ext_prepare(&tmpl);
7287
7288         if (flags != 0) {
7289                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
7290                 if (err < 0)
7291                         return err;
7292         }
7293
7294         if (nla[NFTA_SET_ELEM_KEY]) {
7295                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
7296                                             nla[NFTA_SET_ELEM_KEY]);
7297                 if (err < 0)
7298                         return err;
7299
7300                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
7301                 if (err < 0)
7302                         goto fail_elem;
7303         }
7304
7305         if (nla[NFTA_SET_ELEM_KEY_END]) {
7306                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
7307                                             nla[NFTA_SET_ELEM_KEY_END]);
7308                 if (err < 0)
7309                         goto fail_elem;
7310
7311                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
7312                 if (err < 0)
7313                         goto fail_elem_key_end;
7314         }
7315
7316         err = -ENOMEM;
7317         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
7318                                       elem.key_end.val.data, NULL, 0, 0,
7319                                       GFP_KERNEL_ACCOUNT);
7320         if (IS_ERR(elem.priv)) {
7321                 err = PTR_ERR(elem.priv);
7322                 goto fail_elem_key_end;
7323         }
7324
7325         ext = nft_set_elem_ext(set, elem.priv);
7326         if (flags)
7327                 *nft_set_ext_flags(ext) = flags;
7328
7329         trans = nft_trans_elem_alloc(ctx, NFT_MSG_DELSETELEM, set);
7330         if (trans == NULL)
7331                 goto fail_trans;
7332
7333         err = nft_setelem_deactivate(ctx->net, set, &elem, flags);
7334         if (err < 0)
7335                 goto fail_ops;
7336
7337         nft_setelem_data_deactivate(ctx->net, set, elem.priv);
7338
7339         nft_trans_elem_priv(trans) = elem.priv;
7340         nft_trans_commit_list_add_tail(ctx->net, trans);
7341         return 0;
7342
7343 fail_ops:
7344         kfree(trans);
7345 fail_trans:
7346         kfree(elem.priv);
7347 fail_elem_key_end:
7348         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
7349 fail_elem:
7350         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
7351         return err;
7352 }
7353
7354 static int nft_setelem_flush(const struct nft_ctx *ctx,
7355                              struct nft_set *set,
7356                              const struct nft_set_iter *iter,
7357                              struct nft_elem_priv *elem_priv)
7358 {
7359         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
7360         struct nft_trans *trans;
7361
7362         if (!nft_set_elem_active(ext, iter->genmask))
7363                 return 0;
7364
7365         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
7366                                     sizeof(struct nft_trans_elem), GFP_ATOMIC);
7367         if (!trans)
7368                 return -ENOMEM;
7369
7370         set->ops->flush(ctx->net, set, elem_priv);
7371         set->ndeact++;
7372
7373         nft_setelem_data_deactivate(ctx->net, set, elem_priv);
7374         nft_trans_elem_set(trans) = set;
7375         nft_trans_elem_priv(trans) = elem_priv;
7376         nft_trans_commit_list_add_tail(ctx->net, trans);
7377
7378         return 0;
7379 }
7380
7381 static int __nft_set_catchall_flush(const struct nft_ctx *ctx,
7382                                     struct nft_set *set,
7383                                     struct nft_elem_priv *elem_priv)
7384 {
7385         struct nft_trans *trans;
7386
7387         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
7388                                     sizeof(struct nft_trans_elem), GFP_KERNEL);
7389         if (!trans)
7390                 return -ENOMEM;
7391
7392         nft_setelem_data_deactivate(ctx->net, set, elem_priv);
7393         nft_trans_elem_set(trans) = set;
7394         nft_trans_elem_priv(trans) = elem_priv;
7395         nft_trans_commit_list_add_tail(ctx->net, trans);
7396
7397         return 0;
7398 }
7399
7400 static int nft_set_catchall_flush(const struct nft_ctx *ctx,
7401                                   struct nft_set *set)
7402 {
7403         u8 genmask = nft_genmask_next(ctx->net);
7404         struct nft_set_elem_catchall *catchall;
7405         struct nft_set_ext *ext;
7406         int ret = 0;
7407
7408         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
7409                 ext = nft_set_elem_ext(set, catchall->elem);
7410                 if (!nft_set_elem_active(ext, genmask))
7411                         continue;
7412
7413                 ret = __nft_set_catchall_flush(ctx, set, catchall->elem);
7414                 if (ret < 0)
7415                         break;
7416                 nft_set_elem_change_active(ctx->net, set, ext);
7417         }
7418
7419         return ret;
7420 }
7421
7422 static int nft_set_flush(struct nft_ctx *ctx, struct nft_set *set, u8 genmask)
7423 {
7424         struct nft_set_iter iter = {
7425                 .genmask        = genmask,
7426                 .type           = NFT_ITER_UPDATE,
7427                 .fn             = nft_setelem_flush,
7428         };
7429
7430         set->ops->walk(ctx, set, &iter);
7431         if (!iter.err)
7432                 iter.err = nft_set_catchall_flush(ctx, set);
7433
7434         return iter.err;
7435 }
7436
7437 static int nf_tables_delsetelem(struct sk_buff *skb,
7438                                 const struct nfnl_info *info,
7439                                 const struct nlattr * const nla[])
7440 {
7441         struct netlink_ext_ack *extack = info->extack;
7442         u8 genmask = nft_genmask_next(info->net);
7443         u8 family = info->nfmsg->nfgen_family;
7444         struct net *net = info->net;
7445         const struct nlattr *attr;
7446         struct nft_table *table;
7447         struct nft_set *set;
7448         struct nft_ctx ctx;
7449         int rem, err = 0;
7450
7451         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
7452                                  genmask, NETLINK_CB(skb).portid);
7453         if (IS_ERR(table)) {
7454                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
7455                 return PTR_ERR(table);
7456         }
7457
7458         set = nft_set_lookup(table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
7459         if (IS_ERR(set)) {
7460                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_SET]);
7461                 return PTR_ERR(set);
7462         }
7463
7464         if (nft_set_is_anonymous(set))
7465                 return -EOPNOTSUPP;
7466
7467         if (!list_empty(&set->bindings) && (set->flags & NFT_SET_CONSTANT))
7468                 return -EBUSY;
7469
7470         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7471
7472         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
7473                 return nft_set_flush(&ctx, set, genmask);
7474
7475         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
7476                 err = nft_del_setelem(&ctx, set, attr);
7477                 if (err == -ENOENT &&
7478                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYSETELEM)
7479                         continue;
7480
7481                 if (err < 0) {
7482                         NL_SET_BAD_ATTR(extack, attr);
7483                         return err;
7484                 }
7485         }
7486
7487         return 0;
7488 }
7489
7490 /*
7491  * Stateful objects
7492  */
7493
7494 /**
7495  *      nft_register_obj- register nf_tables stateful object type
7496  *      @obj_type: object type
7497  *
7498  *      Registers the object type for use with nf_tables. Returns zero on
7499  *      success or a negative errno code otherwise.
7500  */
7501 int nft_register_obj(struct nft_object_type *obj_type)
7502 {
7503         if (obj_type->type == NFT_OBJECT_UNSPEC)
7504                 return -EINVAL;
7505
7506         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7507         list_add_rcu(&obj_type->list, &nf_tables_objects);
7508         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7509         return 0;
7510 }
7511 EXPORT_SYMBOL_GPL(nft_register_obj);
7512
7513 /**
7514  *      nft_unregister_obj - unregister nf_tables object type
7515  *      @obj_type: object type
7516  *
7517  *      Unregisters the object type for use with nf_tables.
7518  */
7519 void nft_unregister_obj(struct nft_object_type *obj_type)
7520 {
7521         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7522         list_del_rcu(&obj_type->list);
7523         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7524 }
7525 EXPORT_SYMBOL_GPL(nft_unregister_obj);
7526
7527 struct nft_object *nft_obj_lookup(const struct net *net,
7528                                   const struct nft_table *table,
7529                                   const struct nlattr *nla, u32 objtype,
7530                                   u8 genmask)
7531 {
7532         struct nft_object_hash_key k = { .table = table };
7533         char search[NFT_OBJ_MAXNAMELEN];
7534         struct rhlist_head *tmp, *list;
7535         struct nft_object *obj;
7536
7537         nla_strscpy(search, nla, sizeof(search));
7538         k.name = search;
7539
7540         WARN_ON_ONCE(!rcu_read_lock_held() &&
7541                      !lockdep_commit_lock_is_held(net));
7542
7543         rcu_read_lock();
7544         list = rhltable_lookup(&nft_objname_ht, &k, nft_objname_ht_params);
7545         if (!list)
7546                 goto out;
7547
7548         rhl_for_each_entry_rcu(obj, tmp, list, rhlhead) {
7549                 if (objtype == obj->ops->type->type &&
7550                     nft_active_genmask(obj, genmask)) {
7551                         rcu_read_unlock();
7552                         return obj;
7553                 }
7554         }
7555 out:
7556         rcu_read_unlock();
7557         return ERR_PTR(-ENOENT);
7558 }
7559 EXPORT_SYMBOL_GPL(nft_obj_lookup);
7560
7561 static struct nft_object *nft_obj_lookup_byhandle(const struct nft_table *table,
7562                                                   const struct nlattr *nla,
7563                                                   u32 objtype, u8 genmask)
7564 {
7565         struct nft_object *obj;
7566
7567         list_for_each_entry(obj, &table->objects, list) {
7568                 if (be64_to_cpu(nla_get_be64(nla)) == obj->handle &&
7569                     objtype == obj->ops->type->type &&
7570                     nft_active_genmask(obj, genmask))
7571                         return obj;
7572         }
7573         return ERR_PTR(-ENOENT);
7574 }
7575
7576 static const struct nla_policy nft_obj_policy[NFTA_OBJ_MAX + 1] = {
7577         [NFTA_OBJ_TABLE]        = { .type = NLA_STRING,
7578                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
7579         [NFTA_OBJ_NAME]         = { .type = NLA_STRING,
7580                                     .len = NFT_OBJ_MAXNAMELEN - 1 },
7581         [NFTA_OBJ_TYPE]         = { .type = NLA_U32 },
7582         [NFTA_OBJ_DATA]         = { .type = NLA_NESTED },
7583         [NFTA_OBJ_HANDLE]       = { .type = NLA_U64},
7584         [NFTA_OBJ_USERDATA]     = { .type = NLA_BINARY,
7585                                     .len = NFT_USERDATA_MAXLEN },
7586 };
7587
7588 static struct nft_object *nft_obj_init(const struct nft_ctx *ctx,
7589                                        const struct nft_object_type *type,
7590                                        const struct nlattr *attr)
7591 {
7592         struct nlattr **tb;
7593         const struct nft_object_ops *ops;
7594         struct nft_object *obj;
7595         int err = -ENOMEM;
7596
7597         tb = kmalloc_array(type->maxattr + 1, sizeof(*tb), GFP_KERNEL);
7598         if (!tb)
7599                 goto err1;
7600
7601         if (attr) {
7602                 err = nla_parse_nested_deprecated(tb, type->maxattr, attr,
7603                                                   type->policy, NULL);
7604                 if (err < 0)
7605                         goto err2;
7606         } else {
7607                 memset(tb, 0, sizeof(tb[0]) * (type->maxattr + 1));
7608         }
7609
7610         if (type->select_ops) {
7611                 ops = type->select_ops(ctx, (const struct nlattr * const *)tb);
7612                 if (IS_ERR(ops)) {
7613                         err = PTR_ERR(ops);
7614                         goto err2;
7615                 }
7616         } else {
7617                 ops = type->ops;
7618         }
7619
7620         err = -ENOMEM;
7621         obj = kzalloc(sizeof(*obj) + ops->size, GFP_KERNEL_ACCOUNT);
7622         if (!obj)
7623                 goto err2;
7624
7625         err = ops->init(ctx, (const struct nlattr * const *)tb, obj);
7626         if (err < 0)
7627                 goto err3;
7628
7629         obj->ops = ops;
7630
7631         kfree(tb);
7632         return obj;
7633 err3:
7634         kfree(obj);
7635 err2:
7636         kfree(tb);
7637 err1:
7638         return ERR_PTR(err);
7639 }
7640
7641 static int nft_object_dump(struct sk_buff *skb, unsigned int attr,
7642                            struct nft_object *obj, bool reset)
7643 {
7644         struct nlattr *nest;
7645
7646         nest = nla_nest_start_noflag(skb, attr);
7647         if (!nest)
7648                 goto nla_put_failure;
7649         if (obj->ops->dump(skb, obj, reset) < 0)
7650                 goto nla_put_failure;
7651         nla_nest_end(skb, nest);
7652         return 0;
7653
7654 nla_put_failure:
7655         return -1;
7656 }
7657
7658 static const struct nft_object_type *__nft_obj_type_get(u32 objtype, u8 family)
7659 {
7660         const struct nft_object_type *type;
7661
7662         list_for_each_entry_rcu(type, &nf_tables_objects, list) {
7663                 if (type->family != NFPROTO_UNSPEC &&
7664                     type->family != family)
7665                         continue;
7666
7667                 if (objtype == type->type)
7668                         return type;
7669         }
7670         return NULL;
7671 }
7672
7673 static const struct nft_object_type *
7674 nft_obj_type_get(struct net *net, u32 objtype, u8 family)
7675 {
7676         const struct nft_object_type *type;
7677
7678         rcu_read_lock();
7679         type = __nft_obj_type_get(objtype, family);
7680         if (type != NULL && try_module_get(type->owner)) {
7681                 rcu_read_unlock();
7682                 return type;
7683         }
7684         rcu_read_unlock();
7685
7686         lockdep_nfnl_nft_mutex_not_held();
7687 #ifdef CONFIG_MODULES
7688         if (type == NULL) {
7689                 if (nft_request_module(net, "nft-obj-%u", objtype) == -EAGAIN)
7690                         return ERR_PTR(-EAGAIN);
7691         }
7692 #endif
7693         return ERR_PTR(-ENOENT);
7694 }
7695
7696 static int nf_tables_updobj(const struct nft_ctx *ctx,
7697                             const struct nft_object_type *type,
7698                             const struct nlattr *attr,
7699                             struct nft_object *obj)
7700 {
7701         struct nft_object *newobj;
7702         struct nft_trans *trans;
7703         int err = -ENOMEM;
7704
7705         if (!try_module_get(type->owner))
7706                 return -ENOENT;
7707
7708         trans = nft_trans_alloc(ctx, NFT_MSG_NEWOBJ,
7709                                 sizeof(struct nft_trans_obj));
7710         if (!trans)
7711                 goto err_trans;
7712
7713         newobj = nft_obj_init(ctx, type, attr);
7714         if (IS_ERR(newobj)) {
7715                 err = PTR_ERR(newobj);
7716                 goto err_free_trans;
7717         }
7718
7719         nft_trans_obj(trans) = obj;
7720         nft_trans_obj_update(trans) = true;
7721         nft_trans_obj_newobj(trans) = newobj;
7722         nft_trans_commit_list_add_tail(ctx->net, trans);
7723
7724         return 0;
7725
7726 err_free_trans:
7727         kfree(trans);
7728 err_trans:
7729         module_put(type->owner);
7730         return err;
7731 }
7732
7733 static int nf_tables_newobj(struct sk_buff *skb, const struct nfnl_info *info,
7734                             const struct nlattr * const nla[])
7735 {
7736         struct netlink_ext_ack *extack = info->extack;
7737         u8 genmask = nft_genmask_next(info->net);
7738         u8 family = info->nfmsg->nfgen_family;
7739         const struct nft_object_type *type;
7740         struct net *net = info->net;
7741         struct nft_table *table;
7742         struct nft_object *obj;
7743         struct nft_ctx ctx;
7744         u32 objtype;
7745         int err;
7746
7747         if (!nla[NFTA_OBJ_TYPE] ||
7748             !nla[NFTA_OBJ_NAME] ||
7749             !nla[NFTA_OBJ_DATA])
7750                 return -EINVAL;
7751
7752         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
7753                                  NETLINK_CB(skb).portid);
7754         if (IS_ERR(table)) {
7755                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7756                 return PTR_ERR(table);
7757         }
7758
7759         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7760         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
7761         if (IS_ERR(obj)) {
7762                 err = PTR_ERR(obj);
7763                 if (err != -ENOENT) {
7764                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7765                         return err;
7766                 }
7767         } else {
7768                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
7769                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7770                         return -EEXIST;
7771                 }
7772                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
7773                         return -EOPNOTSUPP;
7774
7775                 type = __nft_obj_type_get(objtype, family);
7776                 if (WARN_ON_ONCE(!type))
7777                         return -ENOENT;
7778
7779                 if (!obj->ops->update)
7780                         return 0;
7781
7782                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7783
7784                 return nf_tables_updobj(&ctx, type, nla[NFTA_OBJ_DATA], obj);
7785         }
7786
7787         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7788
7789         if (!nft_use_inc(&table->use))
7790                 return -EMFILE;
7791
7792         type = nft_obj_type_get(net, objtype, family);
7793         if (IS_ERR(type)) {
7794                 err = PTR_ERR(type);
7795                 goto err_type;
7796         }
7797
7798         obj = nft_obj_init(&ctx, type, nla[NFTA_OBJ_DATA]);
7799         if (IS_ERR(obj)) {
7800                 err = PTR_ERR(obj);
7801                 goto err_init;
7802         }
7803         obj->key.table = table;
7804         obj->handle = nf_tables_alloc_handle(table);
7805
7806         obj->key.name = nla_strdup(nla[NFTA_OBJ_NAME], GFP_KERNEL_ACCOUNT);
7807         if (!obj->key.name) {
7808                 err = -ENOMEM;
7809                 goto err_strdup;
7810         }
7811
7812         if (nla[NFTA_OBJ_USERDATA]) {
7813                 obj->udata = nla_memdup(nla[NFTA_OBJ_USERDATA], GFP_KERNEL_ACCOUNT);
7814                 if (obj->udata == NULL)
7815                         goto err_userdata;
7816
7817                 obj->udlen = nla_len(nla[NFTA_OBJ_USERDATA]);
7818         }
7819
7820         err = nft_trans_obj_add(&ctx, NFT_MSG_NEWOBJ, obj);
7821         if (err < 0)
7822                 goto err_trans;
7823
7824         err = rhltable_insert(&nft_objname_ht, &obj->rhlhead,
7825                               nft_objname_ht_params);
7826         if (err < 0)
7827                 goto err_obj_ht;
7828
7829         list_add_tail_rcu(&obj->list, &table->objects);
7830
7831         return 0;
7832 err_obj_ht:
7833         /* queued in transaction log */
7834         INIT_LIST_HEAD(&obj->list);
7835         return err;
7836 err_trans:
7837         kfree(obj->udata);
7838 err_userdata:
7839         kfree(obj->key.name);
7840 err_strdup:
7841         if (obj->ops->destroy)
7842                 obj->ops->destroy(&ctx, obj);
7843         kfree(obj);
7844 err_init:
7845         module_put(type->owner);
7846 err_type:
7847         nft_use_dec_restore(&table->use);
7848
7849         return err;
7850 }
7851
7852 static int nf_tables_fill_obj_info(struct sk_buff *skb, struct net *net,
7853                                    u32 portid, u32 seq, int event, u32 flags,
7854                                    int family, const struct nft_table *table,
7855                                    struct nft_object *obj, bool reset)
7856 {
7857         struct nlmsghdr *nlh;
7858
7859         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
7860         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
7861                            NFNETLINK_V0, nft_base_seq(net));
7862         if (!nlh)
7863                 goto nla_put_failure;
7864
7865         if (nla_put_string(skb, NFTA_OBJ_TABLE, table->name) ||
7866             nla_put_string(skb, NFTA_OBJ_NAME, obj->key.name) ||
7867             nla_put_be64(skb, NFTA_OBJ_HANDLE, cpu_to_be64(obj->handle),
7868                          NFTA_OBJ_PAD))
7869                 goto nla_put_failure;
7870
7871         if (event == NFT_MSG_DELOBJ) {
7872                 nlmsg_end(skb, nlh);
7873                 return 0;
7874         }
7875
7876         if (nla_put_be32(skb, NFTA_OBJ_TYPE, htonl(obj->ops->type->type)) ||
7877             nla_put_be32(skb, NFTA_OBJ_USE, htonl(obj->use)) ||
7878             nft_object_dump(skb, NFTA_OBJ_DATA, obj, reset))
7879                 goto nla_put_failure;
7880
7881         if (obj->udata &&
7882             nla_put(skb, NFTA_OBJ_USERDATA, obj->udlen, obj->udata))
7883                 goto nla_put_failure;
7884
7885         nlmsg_end(skb, nlh);
7886         return 0;
7887
7888 nla_put_failure:
7889         nlmsg_trim(skb, nlh);
7890         return -1;
7891 }
7892
7893 static void audit_log_obj_reset(const struct nft_table *table,
7894                                 unsigned int base_seq, unsigned int nentries)
7895 {
7896         char *buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, base_seq);
7897
7898         audit_log_nfcfg(buf, table->family, nentries,
7899                         AUDIT_NFT_OP_OBJ_RESET, GFP_ATOMIC);
7900         kfree(buf);
7901 }
7902
7903 struct nft_obj_dump_ctx {
7904         unsigned int    s_idx;
7905         char            *table;
7906         u32             type;
7907         bool            reset;
7908 };
7909
7910 static int nf_tables_dump_obj(struct sk_buff *skb, struct netlink_callback *cb)
7911 {
7912         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
7913         struct nft_obj_dump_ctx *ctx = (void *)cb->ctx;
7914         struct net *net = sock_net(skb->sk);
7915         int family = nfmsg->nfgen_family;
7916         struct nftables_pernet *nft_net;
7917         const struct nft_table *table;
7918         unsigned int entries = 0;
7919         struct nft_object *obj;
7920         unsigned int idx = 0;
7921         int rc = 0;
7922
7923         rcu_read_lock();
7924         nft_net = nft_pernet(net);
7925         cb->seq = READ_ONCE(nft_net->base_seq);
7926
7927         list_for_each_entry_rcu(table, &nft_net->tables, list) {
7928                 if (family != NFPROTO_UNSPEC && family != table->family)
7929                         continue;
7930
7931                 entries = 0;
7932                 list_for_each_entry_rcu(obj, &table->objects, list) {
7933                         if (!nft_is_active(net, obj))
7934                                 goto cont;
7935                         if (idx < ctx->s_idx)
7936                                 goto cont;
7937                         if (ctx->table && strcmp(ctx->table, table->name))
7938                                 goto cont;
7939                         if (ctx->type != NFT_OBJECT_UNSPEC &&
7940                             obj->ops->type->type != ctx->type)
7941                                 goto cont;
7942
7943                         rc = nf_tables_fill_obj_info(skb, net,
7944                                                      NETLINK_CB(cb->skb).portid,
7945                                                      cb->nlh->nlmsg_seq,
7946                                                      NFT_MSG_NEWOBJ,
7947                                                      NLM_F_MULTI | NLM_F_APPEND,
7948                                                      table->family, table,
7949                                                      obj, ctx->reset);
7950                         if (rc < 0)
7951                                 break;
7952
7953                         entries++;
7954                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
7955 cont:
7956                         idx++;
7957                 }
7958                 if (ctx->reset && entries)
7959                         audit_log_obj_reset(table, nft_net->base_seq, entries);
7960                 if (rc < 0)
7961                         break;
7962         }
7963         rcu_read_unlock();
7964
7965         ctx->s_idx = idx;
7966         return skb->len;
7967 }
7968
7969 static int nf_tables_dump_obj_start(struct netlink_callback *cb)
7970 {
7971         struct nft_obj_dump_ctx *ctx = (void *)cb->ctx;
7972         const struct nlattr * const *nla = cb->data;
7973
7974         BUILD_BUG_ON(sizeof(*ctx) > sizeof(cb->ctx));
7975
7976         if (nla[NFTA_OBJ_TABLE]) {
7977                 ctx->table = nla_strdup(nla[NFTA_OBJ_TABLE], GFP_ATOMIC);
7978                 if (!ctx->table)
7979                         return -ENOMEM;
7980         }
7981
7982         if (nla[NFTA_OBJ_TYPE])
7983                 ctx->type = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7984
7985         if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
7986                 ctx->reset = true;
7987
7988         return 0;
7989 }
7990
7991 static int nf_tables_dump_obj_done(struct netlink_callback *cb)
7992 {
7993         struct nft_obj_dump_ctx *ctx = (void *)cb->ctx;
7994
7995         kfree(ctx->table);
7996
7997         return 0;
7998 }
7999
8000 /* called with rcu_read_lock held */
8001 static int nf_tables_getobj(struct sk_buff *skb, const struct nfnl_info *info,
8002                             const struct nlattr * const nla[])
8003 {
8004         struct netlink_ext_ack *extack = info->extack;
8005         u8 genmask = nft_genmask_cur(info->net);
8006         u8 family = info->nfmsg->nfgen_family;
8007         const struct nft_table *table;
8008         struct net *net = info->net;
8009         struct nft_object *obj;
8010         struct sk_buff *skb2;
8011         bool reset = false;
8012         u32 objtype;
8013         int err;
8014
8015         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
8016                 struct netlink_dump_control c = {
8017                         .start = nf_tables_dump_obj_start,
8018                         .dump = nf_tables_dump_obj,
8019                         .done = nf_tables_dump_obj_done,
8020                         .module = THIS_MODULE,
8021                         .data = (void *)nla,
8022                 };
8023
8024                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
8025         }
8026
8027         if (!nla[NFTA_OBJ_NAME] ||
8028             !nla[NFTA_OBJ_TYPE])
8029                 return -EINVAL;
8030
8031         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask, 0);
8032         if (IS_ERR(table)) {
8033                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
8034                 return PTR_ERR(table);
8035         }
8036
8037         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
8038         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
8039         if (IS_ERR(obj)) {
8040                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
8041                 return PTR_ERR(obj);
8042         }
8043
8044         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
8045         if (!skb2)
8046                 return -ENOMEM;
8047
8048         if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
8049                 reset = true;
8050
8051         if (reset) {
8052                 const struct nftables_pernet *nft_net;
8053                 char *buf;
8054
8055                 nft_net = nft_pernet(net);
8056                 buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, nft_net->base_seq);
8057
8058                 audit_log_nfcfg(buf,
8059                                 family,
8060                                 1,
8061                                 AUDIT_NFT_OP_OBJ_RESET,
8062                                 GFP_ATOMIC);
8063                 kfree(buf);
8064         }
8065
8066         err = nf_tables_fill_obj_info(skb2, net, NETLINK_CB(skb).portid,
8067                                       info->nlh->nlmsg_seq, NFT_MSG_NEWOBJ, 0,
8068                                       family, table, obj, reset);
8069         if (err < 0)
8070                 goto err_fill_obj_info;
8071
8072         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
8073
8074 err_fill_obj_info:
8075         kfree_skb(skb2);
8076         return err;
8077 }
8078
8079 static void nft_obj_destroy(const struct nft_ctx *ctx, struct nft_object *obj)
8080 {
8081         if (obj->ops->destroy)
8082                 obj->ops->destroy(ctx, obj);
8083
8084         module_put(obj->ops->type->owner);
8085         kfree(obj->key.name);
8086         kfree(obj->udata);
8087         kfree(obj);
8088 }
8089
8090 static int nf_tables_delobj(struct sk_buff *skb, const struct nfnl_info *info,
8091                             const struct nlattr * const nla[])
8092 {
8093         struct netlink_ext_ack *extack = info->extack;
8094         u8 genmask = nft_genmask_next(info->net);
8095         u8 family = info->nfmsg->nfgen_family;
8096         struct net *net = info->net;
8097         const struct nlattr *attr;
8098         struct nft_table *table;
8099         struct nft_object *obj;
8100         struct nft_ctx ctx;
8101         u32 objtype;
8102
8103         if (!nla[NFTA_OBJ_TYPE] ||
8104             (!nla[NFTA_OBJ_NAME] && !nla[NFTA_OBJ_HANDLE]))
8105                 return -EINVAL;
8106
8107         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
8108                                  NETLINK_CB(skb).portid);
8109         if (IS_ERR(table)) {
8110                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
8111                 return PTR_ERR(table);
8112         }
8113
8114         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
8115         if (nla[NFTA_OBJ_HANDLE]) {
8116                 attr = nla[NFTA_OBJ_HANDLE];
8117                 obj = nft_obj_lookup_byhandle(table, attr, objtype, genmask);
8118         } else {
8119                 attr = nla[NFTA_OBJ_NAME];
8120                 obj = nft_obj_lookup(net, table, attr, objtype, genmask);
8121         }
8122
8123         if (IS_ERR(obj)) {
8124                 if (PTR_ERR(obj) == -ENOENT &&
8125                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYOBJ)
8126                         return 0;
8127
8128                 NL_SET_BAD_ATTR(extack, attr);
8129                 return PTR_ERR(obj);
8130         }
8131         if (obj->use > 0) {
8132                 NL_SET_BAD_ATTR(extack, attr);
8133                 return -EBUSY;
8134         }
8135
8136         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8137
8138         return nft_delobj(&ctx, obj);
8139 }
8140
8141 static void
8142 __nft_obj_notify(struct net *net, const struct nft_table *table,
8143                  struct nft_object *obj, u32 portid, u32 seq, int event,
8144                  u16 flags, int family, int report, gfp_t gfp)
8145 {
8146         struct nftables_pernet *nft_net = nft_pernet(net);
8147         struct sk_buff *skb;
8148         int err;
8149
8150         if (!report &&
8151             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
8152                 return;
8153
8154         skb = nlmsg_new(NLMSG_GOODSIZE, gfp);
8155         if (skb == NULL)
8156                 goto err;
8157
8158         err = nf_tables_fill_obj_info(skb, net, portid, seq, event,
8159                                       flags & (NLM_F_CREATE | NLM_F_EXCL),
8160                                       family, table, obj, false);
8161         if (err < 0) {
8162                 kfree_skb(skb);
8163                 goto err;
8164         }
8165
8166         nft_notify_enqueue(skb, report, &nft_net->notify_list);
8167         return;
8168 err:
8169         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
8170 }
8171
8172 void nft_obj_notify(struct net *net, const struct nft_table *table,
8173                     struct nft_object *obj, u32 portid, u32 seq, int event,
8174                     u16 flags, int family, int report, gfp_t gfp)
8175 {
8176         struct nftables_pernet *nft_net = nft_pernet(net);
8177         char *buf = kasprintf(gfp, "%s:%u",
8178                               table->name, nft_net->base_seq);
8179
8180         audit_log_nfcfg(buf,
8181                         family,
8182                         obj->handle,
8183                         event == NFT_MSG_NEWOBJ ?
8184                                  AUDIT_NFT_OP_OBJ_REGISTER :
8185                                  AUDIT_NFT_OP_OBJ_UNREGISTER,
8186                         gfp);
8187         kfree(buf);
8188
8189         __nft_obj_notify(net, table, obj, portid, seq, event,
8190                          flags, family, report, gfp);
8191 }
8192 EXPORT_SYMBOL_GPL(nft_obj_notify);
8193
8194 static void nf_tables_obj_notify(const struct nft_ctx *ctx,
8195                                  struct nft_object *obj, int event)
8196 {
8197         __nft_obj_notify(ctx->net, ctx->table, obj, ctx->portid,
8198                          ctx->seq, event, ctx->flags, ctx->family,
8199                          ctx->report, GFP_KERNEL);
8200 }
8201
8202 /*
8203  * Flow tables
8204  */
8205 void nft_register_flowtable_type(struct nf_flowtable_type *type)
8206 {
8207         nfnl_lock(NFNL_SUBSYS_NFTABLES);
8208         list_add_tail_rcu(&type->list, &nf_tables_flowtables);
8209         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
8210 }
8211 EXPORT_SYMBOL_GPL(nft_register_flowtable_type);
8212
8213 void nft_unregister_flowtable_type(struct nf_flowtable_type *type)
8214 {
8215         nfnl_lock(NFNL_SUBSYS_NFTABLES);
8216         list_del_rcu(&type->list);
8217         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
8218 }
8219 EXPORT_SYMBOL_GPL(nft_unregister_flowtable_type);
8220
8221 static const struct nla_policy nft_flowtable_policy[NFTA_FLOWTABLE_MAX + 1] = {
8222         [NFTA_FLOWTABLE_TABLE]          = { .type = NLA_STRING,
8223                                             .len = NFT_NAME_MAXLEN - 1 },
8224         [NFTA_FLOWTABLE_NAME]           = { .type = NLA_STRING,
8225                                             .len = NFT_NAME_MAXLEN - 1 },
8226         [NFTA_FLOWTABLE_HOOK]           = { .type = NLA_NESTED },
8227         [NFTA_FLOWTABLE_HANDLE]         = { .type = NLA_U64 },
8228         [NFTA_FLOWTABLE_FLAGS]          = { .type = NLA_U32 },
8229 };
8230
8231 struct nft_flowtable *nft_flowtable_lookup(const struct nft_table *table,
8232                                            const struct nlattr *nla, u8 genmask)
8233 {
8234         struct nft_flowtable *flowtable;
8235
8236         list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
8237                 if (!nla_strcmp(nla, flowtable->name) &&
8238                     nft_active_genmask(flowtable, genmask))
8239                         return flowtable;
8240         }
8241         return ERR_PTR(-ENOENT);
8242 }
8243 EXPORT_SYMBOL_GPL(nft_flowtable_lookup);
8244
8245 void nf_tables_deactivate_flowtable(const struct nft_ctx *ctx,
8246                                     struct nft_flowtable *flowtable,
8247                                     enum nft_trans_phase phase)
8248 {
8249         switch (phase) {
8250         case NFT_TRANS_PREPARE_ERROR:
8251         case NFT_TRANS_PREPARE:
8252         case NFT_TRANS_ABORT:
8253         case NFT_TRANS_RELEASE:
8254                 nft_use_dec(&flowtable->use);
8255                 fallthrough;
8256         default:
8257                 return;
8258         }
8259 }
8260 EXPORT_SYMBOL_GPL(nf_tables_deactivate_flowtable);
8261
8262 static struct nft_flowtable *
8263 nft_flowtable_lookup_byhandle(const struct nft_table *table,
8264                               const struct nlattr *nla, u8 genmask)
8265 {
8266        struct nft_flowtable *flowtable;
8267
8268        list_for_each_entry(flowtable, &table->flowtables, list) {
8269                if (be64_to_cpu(nla_get_be64(nla)) == flowtable->handle &&
8270                    nft_active_genmask(flowtable, genmask))
8271                        return flowtable;
8272        }
8273        return ERR_PTR(-ENOENT);
8274 }
8275
8276 struct nft_flowtable_hook {
8277         u32                     num;
8278         int                     priority;
8279         struct list_head        list;
8280 };
8281
8282 static const struct nla_policy nft_flowtable_hook_policy[NFTA_FLOWTABLE_HOOK_MAX + 1] = {
8283         [NFTA_FLOWTABLE_HOOK_NUM]       = { .type = NLA_U32 },
8284         [NFTA_FLOWTABLE_HOOK_PRIORITY]  = { .type = NLA_U32 },
8285         [NFTA_FLOWTABLE_HOOK_DEVS]      = { .type = NLA_NESTED },
8286 };
8287
8288 static int nft_flowtable_parse_hook(const struct nft_ctx *ctx,
8289                                     const struct nlattr * const nla[],
8290                                     struct nft_flowtable_hook *flowtable_hook,
8291                                     struct nft_flowtable *flowtable,
8292                                     struct netlink_ext_ack *extack, bool add)
8293 {
8294         struct nlattr *tb[NFTA_FLOWTABLE_HOOK_MAX + 1];
8295         struct nft_hook *hook;
8296         int hooknum, priority;
8297         int err;
8298
8299         INIT_LIST_HEAD(&flowtable_hook->list);
8300
8301         err = nla_parse_nested_deprecated(tb, NFTA_FLOWTABLE_HOOK_MAX,
8302                                           nla[NFTA_FLOWTABLE_HOOK],
8303                                           nft_flowtable_hook_policy, NULL);
8304         if (err < 0)
8305                 return err;
8306
8307         if (add) {
8308                 if (!tb[NFTA_FLOWTABLE_HOOK_NUM] ||
8309                     !tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
8310                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8311                         return -ENOENT;
8312                 }
8313
8314                 hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
8315                 if (hooknum != NF_NETDEV_INGRESS)
8316                         return -EOPNOTSUPP;
8317
8318                 priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
8319
8320                 flowtable_hook->priority        = priority;
8321                 flowtable_hook->num             = hooknum;
8322         } else {
8323                 if (tb[NFTA_FLOWTABLE_HOOK_NUM]) {
8324                         hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
8325                         if (hooknum != flowtable->hooknum)
8326                                 return -EOPNOTSUPP;
8327                 }
8328
8329                 if (tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
8330                         priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
8331                         if (priority != flowtable->data.priority)
8332                                 return -EOPNOTSUPP;
8333                 }
8334
8335                 flowtable_hook->priority        = flowtable->data.priority;
8336                 flowtable_hook->num             = flowtable->hooknum;
8337         }
8338
8339         if (tb[NFTA_FLOWTABLE_HOOK_DEVS]) {
8340                 err = nf_tables_parse_netdev_hooks(ctx->net,
8341                                                    tb[NFTA_FLOWTABLE_HOOK_DEVS],
8342                                                    &flowtable_hook->list,
8343                                                    extack);
8344                 if (err < 0)
8345                         return err;
8346         }
8347
8348         list_for_each_entry(hook, &flowtable_hook->list, list) {
8349                 hook->ops.pf            = NFPROTO_NETDEV;
8350                 hook->ops.hooknum       = flowtable_hook->num;
8351                 hook->ops.priority      = flowtable_hook->priority;
8352                 hook->ops.priv          = &flowtable->data;
8353                 hook->ops.hook          = flowtable->data.type->hook;
8354         }
8355
8356         return err;
8357 }
8358
8359 /* call under rcu_read_lock */
8360 static const struct nf_flowtable_type *__nft_flowtable_type_get(u8 family)
8361 {
8362         const struct nf_flowtable_type *type;
8363
8364         list_for_each_entry_rcu(type, &nf_tables_flowtables, list) {
8365                 if (family == type->family)
8366                         return type;
8367         }
8368         return NULL;
8369 }
8370
8371 static const struct nf_flowtable_type *
8372 nft_flowtable_type_get(struct net *net, u8 family)
8373 {
8374         const struct nf_flowtable_type *type;
8375
8376         rcu_read_lock();
8377         type = __nft_flowtable_type_get(family);
8378         if (type != NULL && try_module_get(type->owner)) {
8379                 rcu_read_unlock();
8380                 return type;
8381         }
8382         rcu_read_unlock();
8383
8384         lockdep_nfnl_nft_mutex_not_held();
8385 #ifdef CONFIG_MODULES
8386         if (type == NULL) {
8387                 if (nft_request_module(net, "nf-flowtable-%u", family) == -EAGAIN)
8388                         return ERR_PTR(-EAGAIN);
8389         }
8390 #endif
8391         return ERR_PTR(-ENOENT);
8392 }
8393
8394 /* Only called from error and netdev event paths. */
8395 static void nft_unregister_flowtable_hook(struct net *net,
8396                                           struct nft_flowtable *flowtable,
8397                                           struct nft_hook *hook)
8398 {
8399         nf_unregister_net_hook(net, &hook->ops);
8400         flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
8401                                     FLOW_BLOCK_UNBIND);
8402 }
8403
8404 static void __nft_unregister_flowtable_net_hooks(struct net *net,
8405                                                  struct list_head *hook_list,
8406                                                  bool release_netdev)
8407 {
8408         struct nft_hook *hook, *next;
8409
8410         list_for_each_entry_safe(hook, next, hook_list, list) {
8411                 nf_unregister_net_hook(net, &hook->ops);
8412                 if (release_netdev) {
8413                         list_del(&hook->list);
8414                         kfree_rcu(hook, rcu);
8415                 }
8416         }
8417 }
8418
8419 static void nft_unregister_flowtable_net_hooks(struct net *net,
8420                                                struct list_head *hook_list)
8421 {
8422         __nft_unregister_flowtable_net_hooks(net, hook_list, false);
8423 }
8424
8425 static int nft_register_flowtable_net_hooks(struct net *net,
8426                                             struct nft_table *table,
8427                                             struct list_head *hook_list,
8428                                             struct nft_flowtable *flowtable)
8429 {
8430         struct nft_hook *hook, *hook2, *next;
8431         struct nft_flowtable *ft;
8432         int err, i = 0;
8433
8434         list_for_each_entry(hook, hook_list, list) {
8435                 list_for_each_entry(ft, &table->flowtables, list) {
8436                         if (!nft_is_active_next(net, ft))
8437                                 continue;
8438
8439                         list_for_each_entry(hook2, &ft->hook_list, list) {
8440                                 if (hook->ops.dev == hook2->ops.dev &&
8441                                     hook->ops.pf == hook2->ops.pf) {
8442                                         err = -EEXIST;
8443                                         goto err_unregister_net_hooks;
8444                                 }
8445                         }
8446                 }
8447
8448                 err = flowtable->data.type->setup(&flowtable->data,
8449                                                   hook->ops.dev,
8450                                                   FLOW_BLOCK_BIND);
8451                 if (err < 0)
8452                         goto err_unregister_net_hooks;
8453
8454                 err = nf_register_net_hook(net, &hook->ops);
8455                 if (err < 0) {
8456                         flowtable->data.type->setup(&flowtable->data,
8457                                                     hook->ops.dev,
8458                                                     FLOW_BLOCK_UNBIND);
8459                         goto err_unregister_net_hooks;
8460                 }
8461
8462                 i++;
8463         }
8464
8465         return 0;
8466
8467 err_unregister_net_hooks:
8468         list_for_each_entry_safe(hook, next, hook_list, list) {
8469                 if (i-- <= 0)
8470                         break;
8471
8472                 nft_unregister_flowtable_hook(net, flowtable, hook);
8473                 list_del_rcu(&hook->list);
8474                 kfree_rcu(hook, rcu);
8475         }
8476
8477         return err;
8478 }
8479
8480 static void nft_hooks_destroy(struct list_head *hook_list)
8481 {
8482         struct nft_hook *hook, *next;
8483
8484         list_for_each_entry_safe(hook, next, hook_list, list) {
8485                 list_del_rcu(&hook->list);
8486                 kfree_rcu(hook, rcu);
8487         }
8488 }
8489
8490 static int nft_flowtable_update(struct nft_ctx *ctx, const struct nlmsghdr *nlh,
8491                                 struct nft_flowtable *flowtable,
8492                                 struct netlink_ext_ack *extack)
8493 {
8494         const struct nlattr * const *nla = ctx->nla;
8495         struct nft_flowtable_hook flowtable_hook;
8496         struct nft_hook *hook, *next;
8497         struct nft_trans *trans;
8498         bool unregister = false;
8499         u32 flags;
8500         int err;
8501
8502         err = nft_flowtable_parse_hook(ctx, nla, &flowtable_hook, flowtable,
8503                                        extack, false);
8504         if (err < 0)
8505                 return err;
8506
8507         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
8508                 if (nft_hook_list_find(&flowtable->hook_list, hook)) {
8509                         list_del(&hook->list);
8510                         kfree(hook);
8511                 }
8512         }
8513
8514         if (nla[NFTA_FLOWTABLE_FLAGS]) {
8515                 flags = ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
8516                 if (flags & ~NFT_FLOWTABLE_MASK) {
8517                         err = -EOPNOTSUPP;
8518                         goto err_flowtable_update_hook;
8519                 }
8520                 if ((flowtable->data.flags & NFT_FLOWTABLE_HW_OFFLOAD) ^
8521                     (flags & NFT_FLOWTABLE_HW_OFFLOAD)) {
8522                         err = -EOPNOTSUPP;
8523                         goto err_flowtable_update_hook;
8524                 }
8525         } else {
8526                 flags = flowtable->data.flags;
8527         }
8528
8529         err = nft_register_flowtable_net_hooks(ctx->net, ctx->table,
8530                                                &flowtable_hook.list, flowtable);
8531         if (err < 0)
8532                 goto err_flowtable_update_hook;
8533
8534         trans = nft_trans_alloc(ctx, NFT_MSG_NEWFLOWTABLE,
8535                                 sizeof(struct nft_trans_flowtable));
8536         if (!trans) {
8537                 unregister = true;
8538                 err = -ENOMEM;
8539                 goto err_flowtable_update_hook;
8540         }
8541
8542         nft_trans_flowtable_flags(trans) = flags;
8543         nft_trans_flowtable(trans) = flowtable;
8544         nft_trans_flowtable_update(trans) = true;
8545         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
8546         list_splice(&flowtable_hook.list, &nft_trans_flowtable_hooks(trans));
8547
8548         nft_trans_commit_list_add_tail(ctx->net, trans);
8549
8550         return 0;
8551
8552 err_flowtable_update_hook:
8553         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
8554                 if (unregister)
8555                         nft_unregister_flowtable_hook(ctx->net, flowtable, hook);
8556                 list_del_rcu(&hook->list);
8557                 kfree_rcu(hook, rcu);
8558         }
8559
8560         return err;
8561
8562 }
8563
8564 static int nf_tables_newflowtable(struct sk_buff *skb,
8565                                   const struct nfnl_info *info,
8566                                   const struct nlattr * const nla[])
8567 {
8568         struct netlink_ext_ack *extack = info->extack;
8569         struct nft_flowtable_hook flowtable_hook;
8570         u8 genmask = nft_genmask_next(info->net);
8571         u8 family = info->nfmsg->nfgen_family;
8572         const struct nf_flowtable_type *type;
8573         struct nft_flowtable *flowtable;
8574         struct net *net = info->net;
8575         struct nft_table *table;
8576         struct nft_trans *trans;
8577         struct nft_ctx ctx;
8578         int err;
8579
8580         if (!nla[NFTA_FLOWTABLE_TABLE] ||
8581             !nla[NFTA_FLOWTABLE_NAME] ||
8582             !nla[NFTA_FLOWTABLE_HOOK])
8583                 return -EINVAL;
8584
8585         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8586                                  genmask, NETLINK_CB(skb).portid);
8587         if (IS_ERR(table)) {
8588                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
8589                 return PTR_ERR(table);
8590         }
8591
8592         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
8593                                          genmask);
8594         if (IS_ERR(flowtable)) {
8595                 err = PTR_ERR(flowtable);
8596                 if (err != -ENOENT) {
8597                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8598                         return err;
8599                 }
8600         } else {
8601                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
8602                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8603                         return -EEXIST;
8604                 }
8605
8606                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8607
8608                 return nft_flowtable_update(&ctx, info->nlh, flowtable, extack);
8609         }
8610
8611         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8612
8613         if (!nft_use_inc(&table->use))
8614                 return -EMFILE;
8615
8616         flowtable = kzalloc(sizeof(*flowtable), GFP_KERNEL_ACCOUNT);
8617         if (!flowtable) {
8618                 err = -ENOMEM;
8619                 goto flowtable_alloc;
8620         }
8621
8622         flowtable->table = table;
8623         flowtable->handle = nf_tables_alloc_handle(table);
8624         INIT_LIST_HEAD(&flowtable->hook_list);
8625
8626         flowtable->name = nla_strdup(nla[NFTA_FLOWTABLE_NAME], GFP_KERNEL_ACCOUNT);
8627         if (!flowtable->name) {
8628                 err = -ENOMEM;
8629                 goto err1;
8630         }
8631
8632         type = nft_flowtable_type_get(net, family);
8633         if (IS_ERR(type)) {
8634                 err = PTR_ERR(type);
8635                 goto err2;
8636         }
8637
8638         if (nla[NFTA_FLOWTABLE_FLAGS]) {
8639                 flowtable->data.flags =
8640                         ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
8641                 if (flowtable->data.flags & ~NFT_FLOWTABLE_MASK) {
8642                         err = -EOPNOTSUPP;
8643                         goto err3;
8644                 }
8645         }
8646
8647         write_pnet(&flowtable->data.net, net);
8648         flowtable->data.type = type;
8649         err = type->init(&flowtable->data);
8650         if (err < 0)
8651                 goto err3;
8652
8653         err = nft_flowtable_parse_hook(&ctx, nla, &flowtable_hook, flowtable,
8654                                        extack, true);
8655         if (err < 0)
8656                 goto err_flowtable_parse_hooks;
8657
8658         list_splice(&flowtable_hook.list, &flowtable->hook_list);
8659         flowtable->data.priority = flowtable_hook.priority;
8660         flowtable->hooknum = flowtable_hook.num;
8661
8662         trans = nft_trans_flowtable_add(&ctx, NFT_MSG_NEWFLOWTABLE, flowtable);
8663         if (IS_ERR(trans)) {
8664                 err = PTR_ERR(trans);
8665                 goto err_flowtable_trans;
8666         }
8667
8668         /* This must be LAST to ensure no packets are walking over this flowtable. */
8669         err = nft_register_flowtable_net_hooks(ctx.net, table,
8670                                                &flowtable->hook_list,
8671                                                flowtable);
8672         if (err < 0)
8673                 goto err_flowtable_hooks;
8674
8675         list_add_tail_rcu(&flowtable->list, &table->flowtables);
8676
8677         return 0;
8678
8679 err_flowtable_hooks:
8680         nft_trans_destroy(trans);
8681 err_flowtable_trans:
8682         nft_hooks_destroy(&flowtable->hook_list);
8683 err_flowtable_parse_hooks:
8684         flowtable->data.type->free(&flowtable->data);
8685 err3:
8686         module_put(type->owner);
8687 err2:
8688         kfree(flowtable->name);
8689 err1:
8690         kfree(flowtable);
8691 flowtable_alloc:
8692         nft_use_dec_restore(&table->use);
8693
8694         return err;
8695 }
8696
8697 static void nft_flowtable_hook_release(struct nft_flowtable_hook *flowtable_hook)
8698 {
8699         struct nft_hook *this, *next;
8700
8701         list_for_each_entry_safe(this, next, &flowtable_hook->list, list) {
8702                 list_del(&this->list);
8703                 kfree(this);
8704         }
8705 }
8706
8707 static int nft_delflowtable_hook(struct nft_ctx *ctx,
8708                                  struct nft_flowtable *flowtable,
8709                                  struct netlink_ext_ack *extack)
8710 {
8711         const struct nlattr * const *nla = ctx->nla;
8712         struct nft_flowtable_hook flowtable_hook;
8713         LIST_HEAD(flowtable_del_list);
8714         struct nft_hook *this, *hook;
8715         struct nft_trans *trans;
8716         int err;
8717
8718         err = nft_flowtable_parse_hook(ctx, nla, &flowtable_hook, flowtable,
8719                                        extack, false);
8720         if (err < 0)
8721                 return err;
8722
8723         list_for_each_entry(this, &flowtable_hook.list, list) {
8724                 hook = nft_hook_list_find(&flowtable->hook_list, this);
8725                 if (!hook) {
8726                         err = -ENOENT;
8727                         goto err_flowtable_del_hook;
8728                 }
8729                 list_move(&hook->list, &flowtable_del_list);
8730         }
8731
8732         trans = nft_trans_alloc(ctx, NFT_MSG_DELFLOWTABLE,
8733                                 sizeof(struct nft_trans_flowtable));
8734         if (!trans) {
8735                 err = -ENOMEM;
8736                 goto err_flowtable_del_hook;
8737         }
8738
8739         nft_trans_flowtable(trans) = flowtable;
8740         nft_trans_flowtable_update(trans) = true;
8741         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
8742         list_splice(&flowtable_del_list, &nft_trans_flowtable_hooks(trans));
8743         nft_flowtable_hook_release(&flowtable_hook);
8744
8745         nft_trans_commit_list_add_tail(ctx->net, trans);
8746
8747         return 0;
8748
8749 err_flowtable_del_hook:
8750         list_splice(&flowtable_del_list, &flowtable->hook_list);
8751         nft_flowtable_hook_release(&flowtable_hook);
8752
8753         return err;
8754 }
8755
8756 static int nf_tables_delflowtable(struct sk_buff *skb,
8757                                   const struct nfnl_info *info,
8758                                   const struct nlattr * const nla[])
8759 {
8760         struct netlink_ext_ack *extack = info->extack;
8761         u8 genmask = nft_genmask_next(info->net);
8762         u8 family = info->nfmsg->nfgen_family;
8763         struct nft_flowtable *flowtable;
8764         struct net *net = info->net;
8765         const struct nlattr *attr;
8766         struct nft_table *table;
8767         struct nft_ctx ctx;
8768
8769         if (!nla[NFTA_FLOWTABLE_TABLE] ||
8770             (!nla[NFTA_FLOWTABLE_NAME] &&
8771              !nla[NFTA_FLOWTABLE_HANDLE]))
8772                 return -EINVAL;
8773
8774         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8775                                  genmask, NETLINK_CB(skb).portid);
8776         if (IS_ERR(table)) {
8777                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
8778                 return PTR_ERR(table);
8779         }
8780
8781         if (nla[NFTA_FLOWTABLE_HANDLE]) {
8782                 attr = nla[NFTA_FLOWTABLE_HANDLE];
8783                 flowtable = nft_flowtable_lookup_byhandle(table, attr, genmask);
8784         } else {
8785                 attr = nla[NFTA_FLOWTABLE_NAME];
8786                 flowtable = nft_flowtable_lookup(table, attr, genmask);
8787         }
8788
8789         if (IS_ERR(flowtable)) {
8790                 if (PTR_ERR(flowtable) == -ENOENT &&
8791                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYFLOWTABLE)
8792                         return 0;
8793
8794                 NL_SET_BAD_ATTR(extack, attr);
8795                 return PTR_ERR(flowtable);
8796         }
8797
8798         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8799
8800         if (nla[NFTA_FLOWTABLE_HOOK])
8801                 return nft_delflowtable_hook(&ctx, flowtable, extack);
8802
8803         if (flowtable->use > 0) {
8804                 NL_SET_BAD_ATTR(extack, attr);
8805                 return -EBUSY;
8806         }
8807
8808         return nft_delflowtable(&ctx, flowtable);
8809 }
8810
8811 static int nf_tables_fill_flowtable_info(struct sk_buff *skb, struct net *net,
8812                                          u32 portid, u32 seq, int event,
8813                                          u32 flags, int family,
8814                                          struct nft_flowtable *flowtable,
8815                                          struct list_head *hook_list)
8816 {
8817         struct nlattr *nest, *nest_devs;
8818         struct nft_hook *hook;
8819         struct nlmsghdr *nlh;
8820
8821         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
8822         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
8823                            NFNETLINK_V0, nft_base_seq(net));
8824         if (!nlh)
8825                 goto nla_put_failure;
8826
8827         if (nla_put_string(skb, NFTA_FLOWTABLE_TABLE, flowtable->table->name) ||
8828             nla_put_string(skb, NFTA_FLOWTABLE_NAME, flowtable->name) ||
8829             nla_put_be64(skb, NFTA_FLOWTABLE_HANDLE, cpu_to_be64(flowtable->handle),
8830                          NFTA_FLOWTABLE_PAD))
8831                 goto nla_put_failure;
8832
8833         if (event == NFT_MSG_DELFLOWTABLE && !hook_list) {
8834                 nlmsg_end(skb, nlh);
8835                 return 0;
8836         }
8837
8838         if (nla_put_be32(skb, NFTA_FLOWTABLE_USE, htonl(flowtable->use)) ||
8839             nla_put_be32(skb, NFTA_FLOWTABLE_FLAGS, htonl(flowtable->data.flags)))
8840                 goto nla_put_failure;
8841
8842         nest = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK);
8843         if (!nest)
8844                 goto nla_put_failure;
8845         if (nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_NUM, htonl(flowtable->hooknum)) ||
8846             nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_PRIORITY, htonl(flowtable->data.priority)))
8847                 goto nla_put_failure;
8848
8849         nest_devs = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK_DEVS);
8850         if (!nest_devs)
8851                 goto nla_put_failure;
8852
8853         if (!hook_list)
8854                 hook_list = &flowtable->hook_list;
8855
8856         list_for_each_entry_rcu(hook, hook_list, list) {
8857                 if (nla_put_string(skb, NFTA_DEVICE_NAME, hook->ops.dev->name))
8858                         goto nla_put_failure;
8859         }
8860         nla_nest_end(skb, nest_devs);
8861         nla_nest_end(skb, nest);
8862
8863         nlmsg_end(skb, nlh);
8864         return 0;
8865
8866 nla_put_failure:
8867         nlmsg_trim(skb, nlh);
8868         return -1;
8869 }
8870
8871 struct nft_flowtable_filter {
8872         char            *table;
8873 };
8874
8875 static int nf_tables_dump_flowtable(struct sk_buff *skb,
8876                                     struct netlink_callback *cb)
8877 {
8878         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
8879         struct nft_flowtable_filter *filter = cb->data;
8880         unsigned int idx = 0, s_idx = cb->args[0];
8881         struct net *net = sock_net(skb->sk);
8882         int family = nfmsg->nfgen_family;
8883         struct nft_flowtable *flowtable;
8884         struct nftables_pernet *nft_net;
8885         const struct nft_table *table;
8886
8887         rcu_read_lock();
8888         nft_net = nft_pernet(net);
8889         cb->seq = READ_ONCE(nft_net->base_seq);
8890
8891         list_for_each_entry_rcu(table, &nft_net->tables, list) {
8892                 if (family != NFPROTO_UNSPEC && family != table->family)
8893                         continue;
8894
8895                 list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
8896                         if (!nft_is_active(net, flowtable))
8897                                 goto cont;
8898                         if (idx < s_idx)
8899                                 goto cont;
8900                         if (idx > s_idx)
8901                                 memset(&cb->args[1], 0,
8902                                        sizeof(cb->args) - sizeof(cb->args[0]));
8903                         if (filter && filter->table &&
8904                             strcmp(filter->table, table->name))
8905                                 goto cont;
8906
8907                         if (nf_tables_fill_flowtable_info(skb, net, NETLINK_CB(cb->skb).portid,
8908                                                           cb->nlh->nlmsg_seq,
8909                                                           NFT_MSG_NEWFLOWTABLE,
8910                                                           NLM_F_MULTI | NLM_F_APPEND,
8911                                                           table->family,
8912                                                           flowtable, NULL) < 0)
8913                                 goto done;
8914
8915                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
8916 cont:
8917                         idx++;
8918                 }
8919         }
8920 done:
8921         rcu_read_unlock();
8922
8923         cb->args[0] = idx;
8924         return skb->len;
8925 }
8926
8927 static int nf_tables_dump_flowtable_start(struct netlink_callback *cb)
8928 {
8929         const struct nlattr * const *nla = cb->data;
8930         struct nft_flowtable_filter *filter = NULL;
8931
8932         if (nla[NFTA_FLOWTABLE_TABLE]) {
8933                 filter = kzalloc(sizeof(*filter), GFP_ATOMIC);
8934                 if (!filter)
8935                         return -ENOMEM;
8936
8937                 filter->table = nla_strdup(nla[NFTA_FLOWTABLE_TABLE],
8938                                            GFP_ATOMIC);
8939                 if (!filter->table) {
8940                         kfree(filter);
8941                         return -ENOMEM;
8942                 }
8943         }
8944
8945         cb->data = filter;
8946         return 0;
8947 }
8948
8949 static int nf_tables_dump_flowtable_done(struct netlink_callback *cb)
8950 {
8951         struct nft_flowtable_filter *filter = cb->data;
8952
8953         if (!filter)
8954                 return 0;
8955
8956         kfree(filter->table);
8957         kfree(filter);
8958
8959         return 0;
8960 }
8961
8962 /* called with rcu_read_lock held */
8963 static int nf_tables_getflowtable(struct sk_buff *skb,
8964                                   const struct nfnl_info *info,
8965                                   const struct nlattr * const nla[])
8966 {
8967         struct netlink_ext_ack *extack = info->extack;
8968         u8 genmask = nft_genmask_cur(info->net);
8969         u8 family = info->nfmsg->nfgen_family;
8970         struct nft_flowtable *flowtable;
8971         const struct nft_table *table;
8972         struct net *net = info->net;
8973         struct sk_buff *skb2;
8974         int err;
8975
8976         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
8977                 struct netlink_dump_control c = {
8978                         .start = nf_tables_dump_flowtable_start,
8979                         .dump = nf_tables_dump_flowtable,
8980                         .done = nf_tables_dump_flowtable_done,
8981                         .module = THIS_MODULE,
8982                         .data = (void *)nla,
8983                 };
8984
8985                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
8986         }
8987
8988         if (!nla[NFTA_FLOWTABLE_NAME])
8989                 return -EINVAL;
8990
8991         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8992                                  genmask, 0);
8993         if (IS_ERR(table)) {
8994                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
8995                 return PTR_ERR(table);
8996         }
8997
8998         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
8999                                          genmask);
9000         if (IS_ERR(flowtable)) {
9001                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
9002                 return PTR_ERR(flowtable);
9003         }
9004
9005         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
9006         if (!skb2)
9007                 return -ENOMEM;
9008
9009         err = nf_tables_fill_flowtable_info(skb2, net, NETLINK_CB(skb).portid,
9010                                             info->nlh->nlmsg_seq,
9011                                             NFT_MSG_NEWFLOWTABLE, 0, family,
9012                                             flowtable, NULL);
9013         if (err < 0)
9014                 goto err_fill_flowtable_info;
9015
9016         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
9017
9018 err_fill_flowtable_info:
9019         kfree_skb(skb2);
9020         return err;
9021 }
9022
9023 static void nf_tables_flowtable_notify(struct nft_ctx *ctx,
9024                                        struct nft_flowtable *flowtable,
9025                                        struct list_head *hook_list, int event)
9026 {
9027         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
9028         struct sk_buff *skb;
9029         u16 flags = 0;
9030         int err;
9031
9032         if (!ctx->report &&
9033             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
9034                 return;
9035
9036         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
9037         if (skb == NULL)
9038                 goto err;
9039
9040         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
9041                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
9042
9043         err = nf_tables_fill_flowtable_info(skb, ctx->net, ctx->portid,
9044                                             ctx->seq, event, flags,
9045                                             ctx->family, flowtable, hook_list);
9046         if (err < 0) {
9047                 kfree_skb(skb);
9048                 goto err;
9049         }
9050
9051         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
9052         return;
9053 err:
9054         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
9055 }
9056
9057 static void nf_tables_flowtable_destroy(struct nft_flowtable *flowtable)
9058 {
9059         struct nft_hook *hook, *next;
9060
9061         flowtable->data.type->free(&flowtable->data);
9062         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
9063                 flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
9064                                             FLOW_BLOCK_UNBIND);
9065                 list_del_rcu(&hook->list);
9066                 kfree(hook);
9067         }
9068         kfree(flowtable->name);
9069         module_put(flowtable->data.type->owner);
9070         kfree(flowtable);
9071 }
9072
9073 static int nf_tables_fill_gen_info(struct sk_buff *skb, struct net *net,
9074                                    u32 portid, u32 seq)
9075 {
9076         struct nftables_pernet *nft_net = nft_pernet(net);
9077         struct nlmsghdr *nlh;
9078         char buf[TASK_COMM_LEN];
9079         int event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWGEN);
9080
9081         nlh = nfnl_msg_put(skb, portid, seq, event, 0, AF_UNSPEC,
9082                            NFNETLINK_V0, nft_base_seq(net));
9083         if (!nlh)
9084                 goto nla_put_failure;
9085
9086         if (nla_put_be32(skb, NFTA_GEN_ID, htonl(nft_net->base_seq)) ||
9087             nla_put_be32(skb, NFTA_GEN_PROC_PID, htonl(task_pid_nr(current))) ||
9088             nla_put_string(skb, NFTA_GEN_PROC_NAME, get_task_comm(buf, current)))
9089                 goto nla_put_failure;
9090
9091         nlmsg_end(skb, nlh);
9092         return 0;
9093
9094 nla_put_failure:
9095         nlmsg_trim(skb, nlh);
9096         return -EMSGSIZE;
9097 }
9098
9099 static void nft_flowtable_event(unsigned long event, struct net_device *dev,
9100                                 struct nft_flowtable *flowtable)
9101 {
9102         struct nft_hook *hook;
9103
9104         list_for_each_entry(hook, &flowtable->hook_list, list) {
9105                 if (hook->ops.dev != dev)
9106                         continue;
9107
9108                 /* flow_offload_netdev_event() cleans up entries for us. */
9109                 nft_unregister_flowtable_hook(dev_net(dev), flowtable, hook);
9110                 list_del_rcu(&hook->list);
9111                 kfree_rcu(hook, rcu);
9112                 break;
9113         }
9114 }
9115
9116 static int nf_tables_flowtable_event(struct notifier_block *this,
9117                                      unsigned long event, void *ptr)
9118 {
9119         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
9120         struct nft_flowtable *flowtable;
9121         struct nftables_pernet *nft_net;
9122         struct nft_table *table;
9123         struct net *net;
9124
9125         if (event != NETDEV_UNREGISTER)
9126                 return 0;
9127
9128         net = dev_net(dev);
9129         nft_net = nft_pernet(net);
9130         mutex_lock(&nft_net->commit_mutex);
9131         list_for_each_entry(table, &nft_net->tables, list) {
9132                 list_for_each_entry(flowtable, &table->flowtables, list) {
9133                         nft_flowtable_event(event, dev, flowtable);
9134                 }
9135         }
9136         mutex_unlock(&nft_net->commit_mutex);
9137
9138         return NOTIFY_DONE;
9139 }
9140
9141 static struct notifier_block nf_tables_flowtable_notifier = {
9142         .notifier_call  = nf_tables_flowtable_event,
9143 };
9144
9145 static void nf_tables_gen_notify(struct net *net, struct sk_buff *skb,
9146                                  int event)
9147 {
9148         struct nlmsghdr *nlh = nlmsg_hdr(skb);
9149         struct sk_buff *skb2;
9150         int err;
9151
9152         if (!nlmsg_report(nlh) &&
9153             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
9154                 return;
9155
9156         skb2 = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
9157         if (skb2 == NULL)
9158                 goto err;
9159
9160         err = nf_tables_fill_gen_info(skb2, net, NETLINK_CB(skb).portid,
9161                                       nlh->nlmsg_seq);
9162         if (err < 0) {
9163                 kfree_skb(skb2);
9164                 goto err;
9165         }
9166
9167         nfnetlink_send(skb2, net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
9168                        nlmsg_report(nlh), GFP_KERNEL);
9169         return;
9170 err:
9171         nfnetlink_set_err(net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
9172                           -ENOBUFS);
9173 }
9174
9175 static int nf_tables_getgen(struct sk_buff *skb, const struct nfnl_info *info,
9176                             const struct nlattr * const nla[])
9177 {
9178         struct sk_buff *skb2;
9179         int err;
9180
9181         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
9182         if (skb2 == NULL)
9183                 return -ENOMEM;
9184
9185         err = nf_tables_fill_gen_info(skb2, info->net, NETLINK_CB(skb).portid,
9186                                       info->nlh->nlmsg_seq);
9187         if (err < 0)
9188                 goto err_fill_gen_info;
9189
9190         return nfnetlink_unicast(skb2, info->net, NETLINK_CB(skb).portid);
9191
9192 err_fill_gen_info:
9193         kfree_skb(skb2);
9194         return err;
9195 }
9196
9197 static const struct nfnl_callback nf_tables_cb[NFT_MSG_MAX] = {
9198         [NFT_MSG_NEWTABLE] = {
9199                 .call           = nf_tables_newtable,
9200                 .type           = NFNL_CB_BATCH,
9201                 .attr_count     = NFTA_TABLE_MAX,
9202                 .policy         = nft_table_policy,
9203         },
9204         [NFT_MSG_GETTABLE] = {
9205                 .call           = nf_tables_gettable,
9206                 .type           = NFNL_CB_RCU,
9207                 .attr_count     = NFTA_TABLE_MAX,
9208                 .policy         = nft_table_policy,
9209         },
9210         [NFT_MSG_DELTABLE] = {
9211                 .call           = nf_tables_deltable,
9212                 .type           = NFNL_CB_BATCH,
9213                 .attr_count     = NFTA_TABLE_MAX,
9214                 .policy         = nft_table_policy,
9215         },
9216         [NFT_MSG_DESTROYTABLE] = {
9217                 .call           = nf_tables_deltable,
9218                 .type           = NFNL_CB_BATCH,
9219                 .attr_count     = NFTA_TABLE_MAX,
9220                 .policy         = nft_table_policy,
9221         },
9222         [NFT_MSG_NEWCHAIN] = {
9223                 .call           = nf_tables_newchain,
9224                 .type           = NFNL_CB_BATCH,
9225                 .attr_count     = NFTA_CHAIN_MAX,
9226                 .policy         = nft_chain_policy,
9227         },
9228         [NFT_MSG_GETCHAIN] = {
9229                 .call           = nf_tables_getchain,
9230                 .type           = NFNL_CB_RCU,
9231                 .attr_count     = NFTA_CHAIN_MAX,
9232                 .policy         = nft_chain_policy,
9233         },
9234         [NFT_MSG_DELCHAIN] = {
9235                 .call           = nf_tables_delchain,
9236                 .type           = NFNL_CB_BATCH,
9237                 .attr_count     = NFTA_CHAIN_MAX,
9238                 .policy         = nft_chain_policy,
9239         },
9240         [NFT_MSG_DESTROYCHAIN] = {
9241                 .call           = nf_tables_delchain,
9242                 .type           = NFNL_CB_BATCH,
9243                 .attr_count     = NFTA_CHAIN_MAX,
9244                 .policy         = nft_chain_policy,
9245         },
9246         [NFT_MSG_NEWRULE] = {
9247                 .call           = nf_tables_newrule,
9248                 .type           = NFNL_CB_BATCH,
9249                 .attr_count     = NFTA_RULE_MAX,
9250                 .policy         = nft_rule_policy,
9251         },
9252         [NFT_MSG_GETRULE] = {
9253                 .call           = nf_tables_getrule,
9254                 .type           = NFNL_CB_RCU,
9255                 .attr_count     = NFTA_RULE_MAX,
9256                 .policy         = nft_rule_policy,
9257         },
9258         [NFT_MSG_GETRULE_RESET] = {
9259                 .call           = nf_tables_getrule_reset,
9260                 .type           = NFNL_CB_RCU,
9261                 .attr_count     = NFTA_RULE_MAX,
9262                 .policy         = nft_rule_policy,
9263         },
9264         [NFT_MSG_DELRULE] = {
9265                 .call           = nf_tables_delrule,
9266                 .type           = NFNL_CB_BATCH,
9267                 .attr_count     = NFTA_RULE_MAX,
9268                 .policy         = nft_rule_policy,
9269         },
9270         [NFT_MSG_DESTROYRULE] = {
9271                 .call           = nf_tables_delrule,
9272                 .type           = NFNL_CB_BATCH,
9273                 .attr_count     = NFTA_RULE_MAX,
9274                 .policy         = nft_rule_policy,
9275         },
9276         [NFT_MSG_NEWSET] = {
9277                 .call           = nf_tables_newset,
9278                 .type           = NFNL_CB_BATCH,
9279                 .attr_count     = NFTA_SET_MAX,
9280                 .policy         = nft_set_policy,
9281         },
9282         [NFT_MSG_GETSET] = {
9283                 .call           = nf_tables_getset,
9284                 .type           = NFNL_CB_RCU,
9285                 .attr_count     = NFTA_SET_MAX,
9286                 .policy         = nft_set_policy,
9287         },
9288         [NFT_MSG_DELSET] = {
9289                 .call           = nf_tables_delset,
9290                 .type           = NFNL_CB_BATCH,
9291                 .attr_count     = NFTA_SET_MAX,
9292                 .policy         = nft_set_policy,
9293         },
9294         [NFT_MSG_DESTROYSET] = {
9295                 .call           = nf_tables_delset,
9296                 .type           = NFNL_CB_BATCH,
9297                 .attr_count     = NFTA_SET_MAX,
9298                 .policy         = nft_set_policy,
9299         },
9300         [NFT_MSG_NEWSETELEM] = {
9301                 .call           = nf_tables_newsetelem,
9302                 .type           = NFNL_CB_BATCH,
9303                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9304                 .policy         = nft_set_elem_list_policy,
9305         },
9306         [NFT_MSG_GETSETELEM] = {
9307                 .call           = nf_tables_getsetelem,
9308                 .type           = NFNL_CB_RCU,
9309                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9310                 .policy         = nft_set_elem_list_policy,
9311         },
9312         [NFT_MSG_GETSETELEM_RESET] = {
9313                 .call           = nf_tables_getsetelem_reset,
9314                 .type           = NFNL_CB_RCU,
9315                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9316                 .policy         = nft_set_elem_list_policy,
9317         },
9318         [NFT_MSG_DELSETELEM] = {
9319                 .call           = nf_tables_delsetelem,
9320                 .type           = NFNL_CB_BATCH,
9321                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9322                 .policy         = nft_set_elem_list_policy,
9323         },
9324         [NFT_MSG_DESTROYSETELEM] = {
9325                 .call           = nf_tables_delsetelem,
9326                 .type           = NFNL_CB_BATCH,
9327                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9328                 .policy         = nft_set_elem_list_policy,
9329         },
9330         [NFT_MSG_GETGEN] = {
9331                 .call           = nf_tables_getgen,
9332                 .type           = NFNL_CB_RCU,
9333         },
9334         [NFT_MSG_NEWOBJ] = {
9335                 .call           = nf_tables_newobj,
9336                 .type           = NFNL_CB_BATCH,
9337                 .attr_count     = NFTA_OBJ_MAX,
9338                 .policy         = nft_obj_policy,
9339         },
9340         [NFT_MSG_GETOBJ] = {
9341                 .call           = nf_tables_getobj,
9342                 .type           = NFNL_CB_RCU,
9343                 .attr_count     = NFTA_OBJ_MAX,
9344                 .policy         = nft_obj_policy,
9345         },
9346         [NFT_MSG_DELOBJ] = {
9347                 .call           = nf_tables_delobj,
9348                 .type           = NFNL_CB_BATCH,
9349                 .attr_count     = NFTA_OBJ_MAX,
9350                 .policy         = nft_obj_policy,
9351         },
9352         [NFT_MSG_DESTROYOBJ] = {
9353                 .call           = nf_tables_delobj,
9354                 .type           = NFNL_CB_BATCH,
9355                 .attr_count     = NFTA_OBJ_MAX,
9356                 .policy         = nft_obj_policy,
9357         },
9358         [NFT_MSG_GETOBJ_RESET] = {
9359                 .call           = nf_tables_getobj,
9360                 .type           = NFNL_CB_RCU,
9361                 .attr_count     = NFTA_OBJ_MAX,
9362                 .policy         = nft_obj_policy,
9363         },
9364         [NFT_MSG_NEWFLOWTABLE] = {
9365                 .call           = nf_tables_newflowtable,
9366                 .type           = NFNL_CB_BATCH,
9367                 .attr_count     = NFTA_FLOWTABLE_MAX,
9368                 .policy         = nft_flowtable_policy,
9369         },
9370         [NFT_MSG_GETFLOWTABLE] = {
9371                 .call           = nf_tables_getflowtable,
9372                 .type           = NFNL_CB_RCU,
9373                 .attr_count     = NFTA_FLOWTABLE_MAX,
9374                 .policy         = nft_flowtable_policy,
9375         },
9376         [NFT_MSG_DELFLOWTABLE] = {
9377                 .call           = nf_tables_delflowtable,
9378                 .type           = NFNL_CB_BATCH,
9379                 .attr_count     = NFTA_FLOWTABLE_MAX,
9380                 .policy         = nft_flowtable_policy,
9381         },
9382         [NFT_MSG_DESTROYFLOWTABLE] = {
9383                 .call           = nf_tables_delflowtable,
9384                 .type           = NFNL_CB_BATCH,
9385                 .attr_count     = NFTA_FLOWTABLE_MAX,
9386                 .policy         = nft_flowtable_policy,
9387         },
9388 };
9389
9390 static int nf_tables_validate(struct net *net)
9391 {
9392         struct nftables_pernet *nft_net = nft_pernet(net);
9393         struct nft_table *table;
9394
9395         list_for_each_entry(table, &nft_net->tables, list) {
9396                 switch (table->validate_state) {
9397                 case NFT_VALIDATE_SKIP:
9398                         continue;
9399                 case NFT_VALIDATE_NEED:
9400                         nft_validate_state_update(table, NFT_VALIDATE_DO);
9401                         fallthrough;
9402                 case NFT_VALIDATE_DO:
9403                         if (nft_table_validate(net, table) < 0)
9404                                 return -EAGAIN;
9405
9406                         nft_validate_state_update(table, NFT_VALIDATE_SKIP);
9407                         break;
9408                 }
9409         }
9410
9411         return 0;
9412 }
9413
9414 /* a drop policy has to be deferred until all rules have been activated,
9415  * otherwise a large ruleset that contains a drop-policy base chain will
9416  * cause all packets to get dropped until the full transaction has been
9417  * processed.
9418  *
9419  * We defer the drop policy until the transaction has been finalized.
9420  */
9421 static void nft_chain_commit_drop_policy(struct nft_trans *trans)
9422 {
9423         struct nft_base_chain *basechain;
9424
9425         if (nft_trans_chain_policy(trans) != NF_DROP)
9426                 return;
9427
9428         if (!nft_is_base_chain(trans->ctx.chain))
9429                 return;
9430
9431         basechain = nft_base_chain(trans->ctx.chain);
9432         basechain->policy = NF_DROP;
9433 }
9434
9435 static void nft_chain_commit_update(struct nft_trans *trans)
9436 {
9437         struct nft_base_chain *basechain;
9438
9439         if (nft_trans_chain_name(trans)) {
9440                 rhltable_remove(&trans->ctx.table->chains_ht,
9441                                 &trans->ctx.chain->rhlhead,
9442                                 nft_chain_ht_params);
9443                 swap(trans->ctx.chain->name, nft_trans_chain_name(trans));
9444                 rhltable_insert_key(&trans->ctx.table->chains_ht,
9445                                     trans->ctx.chain->name,
9446                                     &trans->ctx.chain->rhlhead,
9447                                     nft_chain_ht_params);
9448         }
9449
9450         if (!nft_is_base_chain(trans->ctx.chain))
9451                 return;
9452
9453         nft_chain_stats_replace(trans);
9454
9455         basechain = nft_base_chain(trans->ctx.chain);
9456
9457         switch (nft_trans_chain_policy(trans)) {
9458         case NF_DROP:
9459         case NF_ACCEPT:
9460                 basechain->policy = nft_trans_chain_policy(trans);
9461                 break;
9462         }
9463 }
9464
9465 static void nft_obj_commit_update(struct nft_trans *trans)
9466 {
9467         struct nft_object *newobj;
9468         struct nft_object *obj;
9469
9470         obj = nft_trans_obj(trans);
9471         newobj = nft_trans_obj_newobj(trans);
9472
9473         if (WARN_ON_ONCE(!obj->ops->update))
9474                 return;
9475
9476         obj->ops->update(obj, newobj);
9477         nft_obj_destroy(&trans->ctx, newobj);
9478 }
9479
9480 static void nft_commit_release(struct nft_trans *trans)
9481 {
9482         switch (trans->msg_type) {
9483         case NFT_MSG_DELTABLE:
9484         case NFT_MSG_DESTROYTABLE:
9485                 nf_tables_table_destroy(&trans->ctx);
9486                 break;
9487         case NFT_MSG_NEWCHAIN:
9488                 free_percpu(nft_trans_chain_stats(trans));
9489                 kfree(nft_trans_chain_name(trans));
9490                 break;
9491         case NFT_MSG_DELCHAIN:
9492         case NFT_MSG_DESTROYCHAIN:
9493                 if (nft_trans_chain_update(trans))
9494                         nft_hooks_destroy(&nft_trans_chain_hooks(trans));
9495                 else
9496                         nf_tables_chain_destroy(&trans->ctx);
9497                 break;
9498         case NFT_MSG_DELRULE:
9499         case NFT_MSG_DESTROYRULE:
9500                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
9501                 break;
9502         case NFT_MSG_DELSET:
9503         case NFT_MSG_DESTROYSET:
9504                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
9505                 break;
9506         case NFT_MSG_DELSETELEM:
9507         case NFT_MSG_DESTROYSETELEM:
9508                 nf_tables_set_elem_destroy(&trans->ctx,
9509                                            nft_trans_elem_set(trans),
9510                                            nft_trans_elem_priv(trans));
9511                 break;
9512         case NFT_MSG_DELOBJ:
9513         case NFT_MSG_DESTROYOBJ:
9514                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
9515                 break;
9516         case NFT_MSG_DELFLOWTABLE:
9517         case NFT_MSG_DESTROYFLOWTABLE:
9518                 if (nft_trans_flowtable_update(trans))
9519                         nft_hooks_destroy(&nft_trans_flowtable_hooks(trans));
9520                 else
9521                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
9522                 break;
9523         }
9524
9525         if (trans->put_net)
9526                 put_net(trans->ctx.net);
9527
9528         kfree(trans);
9529 }
9530
9531 static void nf_tables_trans_destroy_work(struct work_struct *w)
9532 {
9533         struct nft_trans *trans, *next;
9534         LIST_HEAD(head);
9535
9536         spin_lock(&nf_tables_destroy_list_lock);
9537         list_splice_init(&nf_tables_destroy_list, &head);
9538         spin_unlock(&nf_tables_destroy_list_lock);
9539
9540         if (list_empty(&head))
9541                 return;
9542
9543         synchronize_rcu();
9544
9545         list_for_each_entry_safe(trans, next, &head, list) {
9546                 nft_trans_list_del(trans);
9547                 nft_commit_release(trans);
9548         }
9549 }
9550
9551 void nf_tables_trans_destroy_flush_work(void)
9552 {
9553         flush_work(&trans_destroy_work);
9554 }
9555 EXPORT_SYMBOL_GPL(nf_tables_trans_destroy_flush_work);
9556
9557 static bool nft_expr_reduce(struct nft_regs_track *track,
9558                             const struct nft_expr *expr)
9559 {
9560         return false;
9561 }
9562
9563 static int nf_tables_commit_chain_prepare(struct net *net, struct nft_chain *chain)
9564 {
9565         const struct nft_expr *expr, *last;
9566         struct nft_regs_track track = {};
9567         unsigned int size, data_size;
9568         void *data, *data_boundary;
9569         struct nft_rule_dp *prule;
9570         struct nft_rule *rule;
9571
9572         /* already handled or inactive chain? */
9573         if (chain->blob_next || !nft_is_active_next(net, chain))
9574                 return 0;
9575
9576         data_size = 0;
9577         list_for_each_entry(rule, &chain->rules, list) {
9578                 if (nft_is_active_next(net, rule)) {
9579                         data_size += sizeof(*prule) + rule->dlen;
9580                         if (data_size > INT_MAX)
9581                                 return -ENOMEM;
9582                 }
9583         }
9584
9585         chain->blob_next = nf_tables_chain_alloc_rules(chain, data_size);
9586         if (!chain->blob_next)
9587                 return -ENOMEM;
9588
9589         data = (void *)chain->blob_next->data;
9590         data_boundary = data + data_size;
9591         size = 0;
9592
9593         list_for_each_entry(rule, &chain->rules, list) {
9594                 if (!nft_is_active_next(net, rule))
9595                         continue;
9596
9597                 prule = (struct nft_rule_dp *)data;
9598                 data += offsetof(struct nft_rule_dp, data);
9599                 if (WARN_ON_ONCE(data > data_boundary))
9600                         return -ENOMEM;
9601
9602                 size = 0;
9603                 track.last = nft_expr_last(rule);
9604                 nft_rule_for_each_expr(expr, last, rule) {
9605                         track.cur = expr;
9606
9607                         if (nft_expr_reduce(&track, expr)) {
9608                                 expr = track.cur;
9609                                 continue;
9610                         }
9611
9612                         if (WARN_ON_ONCE(data + size + expr->ops->size > data_boundary))
9613                                 return -ENOMEM;
9614
9615                         memcpy(data + size, expr, expr->ops->size);
9616                         size += expr->ops->size;
9617                 }
9618                 if (WARN_ON_ONCE(size >= 1 << 12))
9619                         return -ENOMEM;
9620
9621                 prule->handle = rule->handle;
9622                 prule->dlen = size;
9623                 prule->is_last = 0;
9624
9625                 data += size;
9626                 size = 0;
9627                 chain->blob_next->size += (unsigned long)(data - (void *)prule);
9628         }
9629
9630         if (WARN_ON_ONCE(data > data_boundary))
9631                 return -ENOMEM;
9632
9633         prule = (struct nft_rule_dp *)data;
9634         nft_last_rule(chain, prule);
9635
9636         return 0;
9637 }
9638
9639 static void nf_tables_commit_chain_prepare_cancel(struct net *net)
9640 {
9641         struct nftables_pernet *nft_net = nft_pernet(net);
9642         struct nft_trans *trans, *next;
9643
9644         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
9645                 struct nft_chain *chain = trans->ctx.chain;
9646
9647                 if (trans->msg_type == NFT_MSG_NEWRULE ||
9648                     trans->msg_type == NFT_MSG_DELRULE) {
9649                         kvfree(chain->blob_next);
9650                         chain->blob_next = NULL;
9651                 }
9652         }
9653 }
9654
9655 static void __nf_tables_commit_chain_free_rules(struct rcu_head *h)
9656 {
9657         struct nft_rule_dp_last *l = container_of(h, struct nft_rule_dp_last, h);
9658
9659         kvfree(l->blob);
9660 }
9661
9662 static void nf_tables_commit_chain_free_rules_old(struct nft_rule_blob *blob)
9663 {
9664         struct nft_rule_dp_last *last;
9665
9666         /* last rule trailer is after end marker */
9667         last = (void *)blob + sizeof(*blob) + blob->size;
9668         last->blob = blob;
9669
9670         call_rcu(&last->h, __nf_tables_commit_chain_free_rules);
9671 }
9672
9673 static void nf_tables_commit_chain(struct net *net, struct nft_chain *chain)
9674 {
9675         struct nft_rule_blob *g0, *g1;
9676         bool next_genbit;
9677
9678         next_genbit = nft_gencursor_next(net);
9679
9680         g0 = rcu_dereference_protected(chain->blob_gen_0,
9681                                        lockdep_commit_lock_is_held(net));
9682         g1 = rcu_dereference_protected(chain->blob_gen_1,
9683                                        lockdep_commit_lock_is_held(net));
9684
9685         /* No changes to this chain? */
9686         if (chain->blob_next == NULL) {
9687                 /* chain had no change in last or next generation */
9688                 if (g0 == g1)
9689                         return;
9690                 /*
9691                  * chain had no change in this generation; make sure next
9692                  * one uses same rules as current generation.
9693                  */
9694                 if (next_genbit) {
9695                         rcu_assign_pointer(chain->blob_gen_1, g0);
9696                         nf_tables_commit_chain_free_rules_old(g1);
9697                 } else {
9698                         rcu_assign_pointer(chain->blob_gen_0, g1);
9699                         nf_tables_commit_chain_free_rules_old(g0);
9700                 }
9701
9702                 return;
9703         }
9704
9705         if (next_genbit)
9706                 rcu_assign_pointer(chain->blob_gen_1, chain->blob_next);
9707         else
9708                 rcu_assign_pointer(chain->blob_gen_0, chain->blob_next);
9709
9710         chain->blob_next = NULL;
9711
9712         if (g0 == g1)
9713                 return;
9714
9715         if (next_genbit)
9716                 nf_tables_commit_chain_free_rules_old(g1);
9717         else
9718                 nf_tables_commit_chain_free_rules_old(g0);
9719 }
9720
9721 static void nft_obj_del(struct nft_object *obj)
9722 {
9723         rhltable_remove(&nft_objname_ht, &obj->rhlhead, nft_objname_ht_params);
9724         list_del_rcu(&obj->list);
9725 }
9726
9727 void nft_chain_del(struct nft_chain *chain)
9728 {
9729         struct nft_table *table = chain->table;
9730
9731         WARN_ON_ONCE(rhltable_remove(&table->chains_ht, &chain->rhlhead,
9732                                      nft_chain_ht_params));
9733         list_del_rcu(&chain->list);
9734 }
9735
9736 static void nft_trans_gc_setelem_remove(struct nft_ctx *ctx,
9737                                         struct nft_trans_gc *trans)
9738 {
9739         struct nft_elem_priv **priv = trans->priv;
9740         unsigned int i;
9741
9742         for (i = 0; i < trans->count; i++) {
9743                 nft_setelem_data_deactivate(ctx->net, trans->set, priv[i]);
9744                 nft_setelem_remove(ctx->net, trans->set, priv[i]);
9745         }
9746 }
9747
9748 void nft_trans_gc_destroy(struct nft_trans_gc *trans)
9749 {
9750         nft_set_put(trans->set);
9751         put_net(trans->net);
9752         kfree(trans);
9753 }
9754
9755 static void nft_trans_gc_trans_free(struct rcu_head *rcu)
9756 {
9757         struct nft_elem_priv *elem_priv;
9758         struct nft_trans_gc *trans;
9759         struct nft_ctx ctx = {};
9760         unsigned int i;
9761
9762         trans = container_of(rcu, struct nft_trans_gc, rcu);
9763         ctx.net = read_pnet(&trans->set->net);
9764
9765         for (i = 0; i < trans->count; i++) {
9766                 elem_priv = trans->priv[i];
9767                 if (!nft_setelem_is_catchall(trans->set, elem_priv))
9768                         atomic_dec(&trans->set->nelems);
9769
9770                 nf_tables_set_elem_destroy(&ctx, trans->set, elem_priv);
9771         }
9772
9773         nft_trans_gc_destroy(trans);
9774 }
9775
9776 static bool nft_trans_gc_work_done(struct nft_trans_gc *trans)
9777 {
9778         struct nftables_pernet *nft_net;
9779         struct nft_ctx ctx = {};
9780
9781         nft_net = nft_pernet(trans->net);
9782
9783         mutex_lock(&nft_net->commit_mutex);
9784
9785         /* Check for race with transaction, otherwise this batch refers to
9786          * stale objects that might not be there anymore. Skip transaction if
9787          * set has been destroyed from control plane transaction in case gc
9788          * worker loses race.
9789          */
9790         if (READ_ONCE(nft_net->gc_seq) != trans->seq || trans->set->dead) {
9791                 mutex_unlock(&nft_net->commit_mutex);
9792                 return false;
9793         }
9794
9795         ctx.net = trans->net;
9796         ctx.table = trans->set->table;
9797
9798         nft_trans_gc_setelem_remove(&ctx, trans);
9799         mutex_unlock(&nft_net->commit_mutex);
9800
9801         return true;
9802 }
9803
9804 static void nft_trans_gc_work(struct work_struct *work)
9805 {
9806         struct nft_trans_gc *trans, *next;
9807         LIST_HEAD(trans_gc_list);
9808
9809         spin_lock(&nf_tables_gc_list_lock);
9810         list_splice_init(&nf_tables_gc_list, &trans_gc_list);
9811         spin_unlock(&nf_tables_gc_list_lock);
9812
9813         list_for_each_entry_safe(trans, next, &trans_gc_list, list) {
9814                 list_del(&trans->list);
9815                 if (!nft_trans_gc_work_done(trans)) {
9816                         nft_trans_gc_destroy(trans);
9817                         continue;
9818                 }
9819                 call_rcu(&trans->rcu, nft_trans_gc_trans_free);
9820         }
9821 }
9822
9823 struct nft_trans_gc *nft_trans_gc_alloc(struct nft_set *set,
9824                                         unsigned int gc_seq, gfp_t gfp)
9825 {
9826         struct net *net = read_pnet(&set->net);
9827         struct nft_trans_gc *trans;
9828
9829         trans = kzalloc(sizeof(*trans), gfp);
9830         if (!trans)
9831                 return NULL;
9832
9833         trans->net = maybe_get_net(net);
9834         if (!trans->net) {
9835                 kfree(trans);
9836                 return NULL;
9837         }
9838
9839         refcount_inc(&set->refs);
9840         trans->set = set;
9841         trans->seq = gc_seq;
9842
9843         return trans;
9844 }
9845
9846 void nft_trans_gc_elem_add(struct nft_trans_gc *trans, void *priv)
9847 {
9848         trans->priv[trans->count++] = priv;
9849 }
9850
9851 static void nft_trans_gc_queue_work(struct nft_trans_gc *trans)
9852 {
9853         spin_lock(&nf_tables_gc_list_lock);
9854         list_add_tail(&trans->list, &nf_tables_gc_list);
9855         spin_unlock(&nf_tables_gc_list_lock);
9856
9857         schedule_work(&trans_gc_work);
9858 }
9859
9860 static int nft_trans_gc_space(struct nft_trans_gc *trans)
9861 {
9862         return NFT_TRANS_GC_BATCHCOUNT - trans->count;
9863 }
9864
9865 struct nft_trans_gc *nft_trans_gc_queue_async(struct nft_trans_gc *gc,
9866                                               unsigned int gc_seq, gfp_t gfp)
9867 {
9868         struct nft_set *set;
9869
9870         if (nft_trans_gc_space(gc))
9871                 return gc;
9872
9873         set = gc->set;
9874         nft_trans_gc_queue_work(gc);
9875
9876         return nft_trans_gc_alloc(set, gc_seq, gfp);
9877 }
9878
9879 void nft_trans_gc_queue_async_done(struct nft_trans_gc *trans)
9880 {
9881         if (trans->count == 0) {
9882                 nft_trans_gc_destroy(trans);
9883                 return;
9884         }
9885
9886         nft_trans_gc_queue_work(trans);
9887 }
9888
9889 struct nft_trans_gc *nft_trans_gc_queue_sync(struct nft_trans_gc *gc, gfp_t gfp)
9890 {
9891         struct nft_set *set;
9892
9893         if (WARN_ON_ONCE(!lockdep_commit_lock_is_held(gc->net)))
9894                 return NULL;
9895
9896         if (nft_trans_gc_space(gc))
9897                 return gc;
9898
9899         set = gc->set;
9900         call_rcu(&gc->rcu, nft_trans_gc_trans_free);
9901
9902         return nft_trans_gc_alloc(set, 0, gfp);
9903 }
9904
9905 void nft_trans_gc_queue_sync_done(struct nft_trans_gc *trans)
9906 {
9907         WARN_ON_ONCE(!lockdep_commit_lock_is_held(trans->net));
9908
9909         if (trans->count == 0) {
9910                 nft_trans_gc_destroy(trans);
9911                 return;
9912         }
9913
9914         call_rcu(&trans->rcu, nft_trans_gc_trans_free);
9915 }
9916
9917 struct nft_trans_gc *nft_trans_gc_catchall_async(struct nft_trans_gc *gc,
9918                                                  unsigned int gc_seq)
9919 {
9920         struct nft_set_elem_catchall *catchall;
9921         const struct nft_set *set = gc->set;
9922         struct nft_set_ext *ext;
9923
9924         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
9925                 ext = nft_set_elem_ext(set, catchall->elem);
9926
9927                 if (!nft_set_elem_expired(ext))
9928                         continue;
9929                 if (nft_set_elem_is_dead(ext))
9930                         goto dead_elem;
9931
9932                 nft_set_elem_dead(ext);
9933 dead_elem:
9934                 gc = nft_trans_gc_queue_async(gc, gc_seq, GFP_ATOMIC);
9935                 if (!gc)
9936                         return NULL;
9937
9938                 nft_trans_gc_elem_add(gc, catchall->elem);
9939         }
9940
9941         return gc;
9942 }
9943
9944 struct nft_trans_gc *nft_trans_gc_catchall_sync(struct nft_trans_gc *gc)
9945 {
9946         struct nft_set_elem_catchall *catchall, *next;
9947         u64 tstamp = nft_net_tstamp(gc->net);
9948         const struct nft_set *set = gc->set;
9949         struct nft_elem_priv *elem_priv;
9950         struct nft_set_ext *ext;
9951
9952         WARN_ON_ONCE(!lockdep_commit_lock_is_held(gc->net));
9953
9954         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
9955                 ext = nft_set_elem_ext(set, catchall->elem);
9956
9957                 if (!__nft_set_elem_expired(ext, tstamp))
9958                         continue;
9959
9960                 gc = nft_trans_gc_queue_sync(gc, GFP_KERNEL);
9961                 if (!gc)
9962                         return NULL;
9963
9964                 elem_priv = catchall->elem;
9965                 nft_setelem_data_deactivate(gc->net, gc->set, elem_priv);
9966                 nft_setelem_catchall_destroy(catchall);
9967                 nft_trans_gc_elem_add(gc, elem_priv);
9968         }
9969
9970         return gc;
9971 }
9972
9973 static void nf_tables_module_autoload_cleanup(struct net *net)
9974 {
9975         struct nftables_pernet *nft_net = nft_pernet(net);
9976         struct nft_module_request *req, *next;
9977
9978         WARN_ON_ONCE(!list_empty(&nft_net->commit_list));
9979         list_for_each_entry_safe(req, next, &nft_net->module_list, list) {
9980                 WARN_ON_ONCE(!req->done);
9981                 list_del(&req->list);
9982                 kfree(req);
9983         }
9984 }
9985
9986 static void nf_tables_commit_release(struct net *net)
9987 {
9988         struct nftables_pernet *nft_net = nft_pernet(net);
9989         struct nft_trans *trans;
9990
9991         /* all side effects have to be made visible.
9992          * For example, if a chain named 'foo' has been deleted, a
9993          * new transaction must not find it anymore.
9994          *
9995          * Memory reclaim happens asynchronously from work queue
9996          * to prevent expensive synchronize_rcu() in commit phase.
9997          */
9998         if (list_empty(&nft_net->commit_list)) {
9999                 nf_tables_module_autoload_cleanup(net);
10000                 mutex_unlock(&nft_net->commit_mutex);
10001                 return;
10002         }
10003
10004         trans = list_last_entry(&nft_net->commit_list,
10005                                 struct nft_trans, list);
10006         get_net(trans->ctx.net);
10007         WARN_ON_ONCE(trans->put_net);
10008
10009         trans->put_net = true;
10010         spin_lock(&nf_tables_destroy_list_lock);
10011         list_splice_tail_init(&nft_net->commit_list, &nf_tables_destroy_list);
10012         spin_unlock(&nf_tables_destroy_list_lock);
10013
10014         nf_tables_module_autoload_cleanup(net);
10015         schedule_work(&trans_destroy_work);
10016
10017         mutex_unlock(&nft_net->commit_mutex);
10018 }
10019
10020 static void nft_commit_notify(struct net *net, u32 portid)
10021 {
10022         struct nftables_pernet *nft_net = nft_pernet(net);
10023         struct sk_buff *batch_skb = NULL, *nskb, *skb;
10024         unsigned char *data;
10025         int len;
10026
10027         list_for_each_entry_safe(skb, nskb, &nft_net->notify_list, list) {
10028                 if (!batch_skb) {
10029 new_batch:
10030                         batch_skb = skb;
10031                         len = NLMSG_GOODSIZE - skb->len;
10032                         list_del(&skb->list);
10033                         continue;
10034                 }
10035                 len -= skb->len;
10036                 if (len > 0 && NFT_CB(skb).report == NFT_CB(batch_skb).report) {
10037                         data = skb_put(batch_skb, skb->len);
10038                         memcpy(data, skb->data, skb->len);
10039                         list_del(&skb->list);
10040                         kfree_skb(skb);
10041                         continue;
10042                 }
10043                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
10044                                NFT_CB(batch_skb).report, GFP_KERNEL);
10045                 goto new_batch;
10046         }
10047
10048         if (batch_skb) {
10049                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
10050                                NFT_CB(batch_skb).report, GFP_KERNEL);
10051         }
10052
10053         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
10054 }
10055
10056 static int nf_tables_commit_audit_alloc(struct list_head *adl,
10057                                         struct nft_table *table)
10058 {
10059         struct nft_audit_data *adp;
10060
10061         list_for_each_entry(adp, adl, list) {
10062                 if (adp->table == table)
10063                         return 0;
10064         }
10065         adp = kzalloc(sizeof(*adp), GFP_KERNEL);
10066         if (!adp)
10067                 return -ENOMEM;
10068         adp->table = table;
10069         list_add(&adp->list, adl);
10070         return 0;
10071 }
10072
10073 static void nf_tables_commit_audit_free(struct list_head *adl)
10074 {
10075         struct nft_audit_data *adp, *adn;
10076
10077         list_for_each_entry_safe(adp, adn, adl, list) {
10078                 list_del(&adp->list);
10079                 kfree(adp);
10080         }
10081 }
10082
10083 static void nf_tables_commit_audit_collect(struct list_head *adl,
10084                                            struct nft_table *table, u32 op)
10085 {
10086         struct nft_audit_data *adp;
10087
10088         list_for_each_entry(adp, adl, list) {
10089                 if (adp->table == table)
10090                         goto found;
10091         }
10092         WARN_ONCE(1, "table=%s not expected in commit list", table->name);
10093         return;
10094 found:
10095         adp->entries++;
10096         if (!adp->op || adp->op > op)
10097                 adp->op = op;
10098 }
10099
10100 #define AUNFTABLENAMELEN (NFT_TABLE_MAXNAMELEN + 22)
10101
10102 static void nf_tables_commit_audit_log(struct list_head *adl, u32 generation)
10103 {
10104         struct nft_audit_data *adp, *adn;
10105         char aubuf[AUNFTABLENAMELEN];
10106
10107         list_for_each_entry_safe(adp, adn, adl, list) {
10108                 snprintf(aubuf, AUNFTABLENAMELEN, "%s:%u", adp->table->name,
10109                          generation);
10110                 audit_log_nfcfg(aubuf, adp->table->family, adp->entries,
10111                                 nft2audit_op[adp->op], GFP_KERNEL);
10112                 list_del(&adp->list);
10113                 kfree(adp);
10114         }
10115 }
10116
10117 static void nft_set_commit_update(struct list_head *set_update_list)
10118 {
10119         struct nft_set *set, *next;
10120
10121         list_for_each_entry_safe(set, next, set_update_list, pending_update) {
10122                 list_del_init(&set->pending_update);
10123
10124                 if (!set->ops->commit || set->dead)
10125                         continue;
10126
10127                 set->ops->commit(set);
10128         }
10129 }
10130
10131 static unsigned int nft_gc_seq_begin(struct nftables_pernet *nft_net)
10132 {
10133         unsigned int gc_seq;
10134
10135         /* Bump gc counter, it becomes odd, this is the busy mark. */
10136         gc_seq = READ_ONCE(nft_net->gc_seq);
10137         WRITE_ONCE(nft_net->gc_seq, ++gc_seq);
10138
10139         return gc_seq;
10140 }
10141
10142 static void nft_gc_seq_end(struct nftables_pernet *nft_net, unsigned int gc_seq)
10143 {
10144         WRITE_ONCE(nft_net->gc_seq, ++gc_seq);
10145 }
10146
10147 static int nf_tables_commit(struct net *net, struct sk_buff *skb)
10148 {
10149         struct nftables_pernet *nft_net = nft_pernet(net);
10150         struct nft_trans *trans, *next;
10151         unsigned int base_seq, gc_seq;
10152         LIST_HEAD(set_update_list);
10153         struct nft_trans_elem *te;
10154         struct nft_chain *chain;
10155         struct nft_table *table;
10156         LIST_HEAD(adl);
10157         int err;
10158
10159         if (list_empty(&nft_net->commit_list)) {
10160                 mutex_unlock(&nft_net->commit_mutex);
10161                 return 0;
10162         }
10163
10164         list_for_each_entry(trans, &nft_net->binding_list, binding_list) {
10165                 switch (trans->msg_type) {
10166                 case NFT_MSG_NEWSET:
10167                         if (!nft_trans_set_update(trans) &&
10168                             nft_set_is_anonymous(nft_trans_set(trans)) &&
10169                             !nft_trans_set_bound(trans)) {
10170                                 pr_warn_once("nftables ruleset with unbound set\n");
10171                                 return -EINVAL;
10172                         }
10173                         break;
10174                 case NFT_MSG_NEWCHAIN:
10175                         if (!nft_trans_chain_update(trans) &&
10176                             nft_chain_binding(nft_trans_chain(trans)) &&
10177                             !nft_trans_chain_bound(trans)) {
10178                                 pr_warn_once("nftables ruleset with unbound chain\n");
10179                                 return -EINVAL;
10180                         }
10181                         break;
10182                 }
10183         }
10184
10185         /* 0. Validate ruleset, otherwise roll back for error reporting. */
10186         if (nf_tables_validate(net) < 0) {
10187                 nft_net->validate_state = NFT_VALIDATE_DO;
10188                 return -EAGAIN;
10189         }
10190
10191         err = nft_flow_rule_offload_commit(net);
10192         if (err < 0)
10193                 return err;
10194
10195         /* 1.  Allocate space for next generation rules_gen_X[] */
10196         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
10197                 int ret;
10198
10199                 ret = nf_tables_commit_audit_alloc(&adl, trans->ctx.table);
10200                 if (ret) {
10201                         nf_tables_commit_chain_prepare_cancel(net);
10202                         nf_tables_commit_audit_free(&adl);
10203                         return ret;
10204                 }
10205                 if (trans->msg_type == NFT_MSG_NEWRULE ||
10206                     trans->msg_type == NFT_MSG_DELRULE) {
10207                         chain = trans->ctx.chain;
10208
10209                         ret = nf_tables_commit_chain_prepare(net, chain);
10210                         if (ret < 0) {
10211                                 nf_tables_commit_chain_prepare_cancel(net);
10212                                 nf_tables_commit_audit_free(&adl);
10213                                 return ret;
10214                         }
10215                 }
10216         }
10217
10218         /* step 2.  Make rules_gen_X visible to packet path */
10219         list_for_each_entry(table, &nft_net->tables, list) {
10220                 list_for_each_entry(chain, &table->chains, list)
10221                         nf_tables_commit_chain(net, chain);
10222         }
10223
10224         /*
10225          * Bump generation counter, invalidate any dump in progress.
10226          * Cannot fail after this point.
10227          */
10228         base_seq = READ_ONCE(nft_net->base_seq);
10229         while (++base_seq == 0)
10230                 ;
10231
10232         WRITE_ONCE(nft_net->base_seq, base_seq);
10233
10234         gc_seq = nft_gc_seq_begin(nft_net);
10235
10236         /* step 3. Start new generation, rules_gen_X now in use. */
10237         net->nft.gencursor = nft_gencursor_next(net);
10238
10239         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
10240                 nf_tables_commit_audit_collect(&adl, trans->ctx.table,
10241                                                trans->msg_type);
10242                 switch (trans->msg_type) {
10243                 case NFT_MSG_NEWTABLE:
10244                         if (nft_trans_table_update(trans)) {
10245                                 if (!(trans->ctx.table->flags & __NFT_TABLE_F_UPDATE)) {
10246                                         nft_trans_destroy(trans);
10247                                         break;
10248                                 }
10249                                 if (trans->ctx.table->flags & NFT_TABLE_F_DORMANT)
10250                                         nf_tables_table_disable(net, trans->ctx.table);
10251
10252                                 trans->ctx.table->flags &= ~__NFT_TABLE_F_UPDATE;
10253                         } else {
10254                                 nft_clear(net, trans->ctx.table);
10255                         }
10256                         nf_tables_table_notify(&trans->ctx, NFT_MSG_NEWTABLE);
10257                         nft_trans_destroy(trans);
10258                         break;
10259                 case NFT_MSG_DELTABLE:
10260                 case NFT_MSG_DESTROYTABLE:
10261                         list_del_rcu(&trans->ctx.table->list);
10262                         nf_tables_table_notify(&trans->ctx, trans->msg_type);
10263                         break;
10264                 case NFT_MSG_NEWCHAIN:
10265                         if (nft_trans_chain_update(trans)) {
10266                                 nft_chain_commit_update(trans);
10267                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN,
10268                                                        &nft_trans_chain_hooks(trans));
10269                                 list_splice(&nft_trans_chain_hooks(trans),
10270                                             &nft_trans_basechain(trans)->hook_list);
10271                                 /* trans destroyed after rcu grace period */
10272                         } else {
10273                                 nft_chain_commit_drop_policy(trans);
10274                                 nft_clear(net, trans->ctx.chain);
10275                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN, NULL);
10276                                 nft_trans_destroy(trans);
10277                         }
10278                         break;
10279                 case NFT_MSG_DELCHAIN:
10280                 case NFT_MSG_DESTROYCHAIN:
10281                         if (nft_trans_chain_update(trans)) {
10282                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_DELCHAIN,
10283                                                        &nft_trans_chain_hooks(trans));
10284                                 if (!(trans->ctx.table->flags & NFT_TABLE_F_DORMANT)) {
10285                                         nft_netdev_unregister_hooks(net,
10286                                                                     &nft_trans_chain_hooks(trans),
10287                                                                     true);
10288                                 }
10289                         } else {
10290                                 nft_chain_del(trans->ctx.chain);
10291                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_DELCHAIN,
10292                                                        NULL);
10293                                 nf_tables_unregister_hook(trans->ctx.net,
10294                                                           trans->ctx.table,
10295                                                           trans->ctx.chain);
10296                         }
10297                         break;
10298                 case NFT_MSG_NEWRULE:
10299                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
10300                         nf_tables_rule_notify(&trans->ctx,
10301                                               nft_trans_rule(trans),
10302                                               NFT_MSG_NEWRULE);
10303                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
10304                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10305
10306                         nft_trans_destroy(trans);
10307                         break;
10308                 case NFT_MSG_DELRULE:
10309                 case NFT_MSG_DESTROYRULE:
10310                         list_del_rcu(&nft_trans_rule(trans)->list);
10311                         nf_tables_rule_notify(&trans->ctx,
10312                                               nft_trans_rule(trans),
10313                                               trans->msg_type);
10314                         nft_rule_expr_deactivate(&trans->ctx,
10315                                                  nft_trans_rule(trans),
10316                                                  NFT_TRANS_COMMIT);
10317
10318                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
10319                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10320                         break;
10321                 case NFT_MSG_NEWSET:
10322                         if (nft_trans_set_update(trans)) {
10323                                 struct nft_set *set = nft_trans_set(trans);
10324
10325                                 WRITE_ONCE(set->timeout, nft_trans_set_timeout(trans));
10326                                 WRITE_ONCE(set->gc_int, nft_trans_set_gc_int(trans));
10327
10328                                 if (nft_trans_set_size(trans))
10329                                         WRITE_ONCE(set->size, nft_trans_set_size(trans));
10330                         } else {
10331                                 nft_clear(net, nft_trans_set(trans));
10332                                 /* This avoids hitting -EBUSY when deleting the table
10333                                  * from the transaction.
10334                                  */
10335                                 if (nft_set_is_anonymous(nft_trans_set(trans)) &&
10336                                     !list_empty(&nft_trans_set(trans)->bindings))
10337                                         nft_use_dec(&trans->ctx.table->use);
10338                         }
10339                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
10340                                              NFT_MSG_NEWSET, GFP_KERNEL);
10341                         nft_trans_destroy(trans);
10342                         break;
10343                 case NFT_MSG_DELSET:
10344                 case NFT_MSG_DESTROYSET:
10345                         nft_trans_set(trans)->dead = 1;
10346                         list_del_rcu(&nft_trans_set(trans)->list);
10347                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
10348                                              trans->msg_type, GFP_KERNEL);
10349                         break;
10350                 case NFT_MSG_NEWSETELEM:
10351                         te = (struct nft_trans_elem *)trans->data;
10352
10353                         nft_setelem_activate(net, te->set, te->elem_priv);
10354                         nf_tables_setelem_notify(&trans->ctx, te->set,
10355                                                  te->elem_priv,
10356                                                  NFT_MSG_NEWSETELEM);
10357                         if (te->set->ops->commit &&
10358                             list_empty(&te->set->pending_update)) {
10359                                 list_add_tail(&te->set->pending_update,
10360                                               &set_update_list);
10361                         }
10362                         nft_trans_destroy(trans);
10363                         break;
10364                 case NFT_MSG_DELSETELEM:
10365                 case NFT_MSG_DESTROYSETELEM:
10366                         te = (struct nft_trans_elem *)trans->data;
10367
10368                         nf_tables_setelem_notify(&trans->ctx, te->set,
10369                                                  te->elem_priv,
10370                                                  trans->msg_type);
10371                         nft_setelem_remove(net, te->set, te->elem_priv);
10372                         if (!nft_setelem_is_catchall(te->set, te->elem_priv)) {
10373                                 atomic_dec(&te->set->nelems);
10374                                 te->set->ndeact--;
10375                         }
10376                         if (te->set->ops->commit &&
10377                             list_empty(&te->set->pending_update)) {
10378                                 list_add_tail(&te->set->pending_update,
10379                                               &set_update_list);
10380                         }
10381                         break;
10382                 case NFT_MSG_NEWOBJ:
10383                         if (nft_trans_obj_update(trans)) {
10384                                 nft_obj_commit_update(trans);
10385                                 nf_tables_obj_notify(&trans->ctx,
10386                                                      nft_trans_obj(trans),
10387                                                      NFT_MSG_NEWOBJ);
10388                         } else {
10389                                 nft_clear(net, nft_trans_obj(trans));
10390                                 nf_tables_obj_notify(&trans->ctx,
10391                                                      nft_trans_obj(trans),
10392                                                      NFT_MSG_NEWOBJ);
10393                                 nft_trans_destroy(trans);
10394                         }
10395                         break;
10396                 case NFT_MSG_DELOBJ:
10397                 case NFT_MSG_DESTROYOBJ:
10398                         nft_obj_del(nft_trans_obj(trans));
10399                         nf_tables_obj_notify(&trans->ctx, nft_trans_obj(trans),
10400                                              trans->msg_type);
10401                         break;
10402                 case NFT_MSG_NEWFLOWTABLE:
10403                         if (nft_trans_flowtable_update(trans)) {
10404                                 nft_trans_flowtable(trans)->data.flags =
10405                                         nft_trans_flowtable_flags(trans);
10406                                 nf_tables_flowtable_notify(&trans->ctx,
10407                                                            nft_trans_flowtable(trans),
10408                                                            &nft_trans_flowtable_hooks(trans),
10409                                                            NFT_MSG_NEWFLOWTABLE);
10410                                 list_splice(&nft_trans_flowtable_hooks(trans),
10411                                             &nft_trans_flowtable(trans)->hook_list);
10412                         } else {
10413                                 nft_clear(net, nft_trans_flowtable(trans));
10414                                 nf_tables_flowtable_notify(&trans->ctx,
10415                                                            nft_trans_flowtable(trans),
10416                                                            NULL,
10417                                                            NFT_MSG_NEWFLOWTABLE);
10418                         }
10419                         nft_trans_destroy(trans);
10420                         break;
10421                 case NFT_MSG_DELFLOWTABLE:
10422                 case NFT_MSG_DESTROYFLOWTABLE:
10423                         if (nft_trans_flowtable_update(trans)) {
10424                                 nf_tables_flowtable_notify(&trans->ctx,
10425                                                            nft_trans_flowtable(trans),
10426                                                            &nft_trans_flowtable_hooks(trans),
10427                                                            trans->msg_type);
10428                                 nft_unregister_flowtable_net_hooks(net,
10429                                                                    &nft_trans_flowtable_hooks(trans));
10430                         } else {
10431                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
10432                                 nf_tables_flowtable_notify(&trans->ctx,
10433                                                            nft_trans_flowtable(trans),
10434                                                            NULL,
10435                                                            trans->msg_type);
10436                                 nft_unregister_flowtable_net_hooks(net,
10437                                                 &nft_trans_flowtable(trans)->hook_list);
10438                         }
10439                         break;
10440                 }
10441         }
10442
10443         nft_set_commit_update(&set_update_list);
10444
10445         nft_commit_notify(net, NETLINK_CB(skb).portid);
10446         nf_tables_gen_notify(net, skb, NFT_MSG_NEWGEN);
10447         nf_tables_commit_audit_log(&adl, nft_net->base_seq);
10448
10449         nft_gc_seq_end(nft_net, gc_seq);
10450         nft_net->validate_state = NFT_VALIDATE_SKIP;
10451         nf_tables_commit_release(net);
10452
10453         return 0;
10454 }
10455
10456 static void nf_tables_module_autoload(struct net *net)
10457 {
10458         struct nftables_pernet *nft_net = nft_pernet(net);
10459         struct nft_module_request *req, *next;
10460         LIST_HEAD(module_list);
10461
10462         list_splice_init(&nft_net->module_list, &module_list);
10463         mutex_unlock(&nft_net->commit_mutex);
10464         list_for_each_entry_safe(req, next, &module_list, list) {
10465                 request_module("%s", req->module);
10466                 req->done = true;
10467         }
10468         mutex_lock(&nft_net->commit_mutex);
10469         list_splice(&module_list, &nft_net->module_list);
10470 }
10471
10472 static void nf_tables_abort_release(struct nft_trans *trans)
10473 {
10474         switch (trans->msg_type) {
10475         case NFT_MSG_NEWTABLE:
10476                 nf_tables_table_destroy(&trans->ctx);
10477                 break;
10478         case NFT_MSG_NEWCHAIN:
10479                 if (nft_trans_chain_update(trans))
10480                         nft_hooks_destroy(&nft_trans_chain_hooks(trans));
10481                 else
10482                         nf_tables_chain_destroy(&trans->ctx);
10483                 break;
10484         case NFT_MSG_NEWRULE:
10485                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
10486                 break;
10487         case NFT_MSG_NEWSET:
10488                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
10489                 break;
10490         case NFT_MSG_NEWSETELEM:
10491                 nft_set_elem_destroy(nft_trans_elem_set(trans),
10492                                      nft_trans_elem_priv(trans), true);
10493                 break;
10494         case NFT_MSG_NEWOBJ:
10495                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
10496                 break;
10497         case NFT_MSG_NEWFLOWTABLE:
10498                 if (nft_trans_flowtable_update(trans))
10499                         nft_hooks_destroy(&nft_trans_flowtable_hooks(trans));
10500                 else
10501                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
10502                 break;
10503         }
10504         kfree(trans);
10505 }
10506
10507 static void nft_set_abort_update(struct list_head *set_update_list)
10508 {
10509         struct nft_set *set, *next;
10510
10511         list_for_each_entry_safe(set, next, set_update_list, pending_update) {
10512                 list_del_init(&set->pending_update);
10513
10514                 if (!set->ops->abort)
10515                         continue;
10516
10517                 set->ops->abort(set);
10518         }
10519 }
10520
10521 static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action)
10522 {
10523         struct nftables_pernet *nft_net = nft_pernet(net);
10524         struct nft_trans *trans, *next;
10525         LIST_HEAD(set_update_list);
10526         struct nft_trans_elem *te;
10527         int err = 0;
10528
10529         if (action == NFNL_ABORT_VALIDATE &&
10530             nf_tables_validate(net) < 0)
10531                 err = -EAGAIN;
10532
10533         list_for_each_entry_safe_reverse(trans, next, &nft_net->commit_list,
10534                                          list) {
10535                 switch (trans->msg_type) {
10536                 case NFT_MSG_NEWTABLE:
10537                         if (nft_trans_table_update(trans)) {
10538                                 if (!(trans->ctx.table->flags & __NFT_TABLE_F_UPDATE)) {
10539                                         nft_trans_destroy(trans);
10540                                         break;
10541                                 }
10542                                 if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_DORMANT) {
10543                                         nf_tables_table_disable(net, trans->ctx.table);
10544                                         trans->ctx.table->flags |= NFT_TABLE_F_DORMANT;
10545                                 } else if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_AWAKEN) {
10546                                         trans->ctx.table->flags &= ~NFT_TABLE_F_DORMANT;
10547                                 }
10548                                 if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_ORPHAN) {
10549                                         trans->ctx.table->flags &= ~NFT_TABLE_F_OWNER;
10550                                         trans->ctx.table->nlpid = 0;
10551                                 }
10552                                 trans->ctx.table->flags &= ~__NFT_TABLE_F_UPDATE;
10553                                 nft_trans_destroy(trans);
10554                         } else {
10555                                 list_del_rcu(&trans->ctx.table->list);
10556                         }
10557                         break;
10558                 case NFT_MSG_DELTABLE:
10559                 case NFT_MSG_DESTROYTABLE:
10560                         nft_clear(trans->ctx.net, trans->ctx.table);
10561                         nft_trans_destroy(trans);
10562                         break;
10563                 case NFT_MSG_NEWCHAIN:
10564                         if (nft_trans_chain_update(trans)) {
10565                                 if (!(trans->ctx.table->flags & NFT_TABLE_F_DORMANT)) {
10566                                         nft_netdev_unregister_hooks(net,
10567                                                                     &nft_trans_chain_hooks(trans),
10568                                                                     true);
10569                                 }
10570                                 free_percpu(nft_trans_chain_stats(trans));
10571                                 kfree(nft_trans_chain_name(trans));
10572                                 nft_trans_destroy(trans);
10573                         } else {
10574                                 if (nft_trans_chain_bound(trans)) {
10575                                         nft_trans_destroy(trans);
10576                                         break;
10577                                 }
10578                                 nft_use_dec_restore(&trans->ctx.table->use);
10579                                 nft_chain_del(trans->ctx.chain);
10580                                 nf_tables_unregister_hook(trans->ctx.net,
10581                                                           trans->ctx.table,
10582                                                           trans->ctx.chain);
10583                         }
10584                         break;
10585                 case NFT_MSG_DELCHAIN:
10586                 case NFT_MSG_DESTROYCHAIN:
10587                         if (nft_trans_chain_update(trans)) {
10588                                 list_splice(&nft_trans_chain_hooks(trans),
10589                                             &nft_trans_basechain(trans)->hook_list);
10590                         } else {
10591                                 nft_use_inc_restore(&trans->ctx.table->use);
10592                                 nft_clear(trans->ctx.net, trans->ctx.chain);
10593                         }
10594                         nft_trans_destroy(trans);
10595                         break;
10596                 case NFT_MSG_NEWRULE:
10597                         if (nft_trans_rule_bound(trans)) {
10598                                 nft_trans_destroy(trans);
10599                                 break;
10600                         }
10601                         nft_use_dec_restore(&trans->ctx.chain->use);
10602                         list_del_rcu(&nft_trans_rule(trans)->list);
10603                         nft_rule_expr_deactivate(&trans->ctx,
10604                                                  nft_trans_rule(trans),
10605                                                  NFT_TRANS_ABORT);
10606                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
10607                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10608                         break;
10609                 case NFT_MSG_DELRULE:
10610                 case NFT_MSG_DESTROYRULE:
10611                         nft_use_inc_restore(&trans->ctx.chain->use);
10612                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
10613                         nft_rule_expr_activate(&trans->ctx, nft_trans_rule(trans));
10614                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
10615                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10616
10617                         nft_trans_destroy(trans);
10618                         break;
10619                 case NFT_MSG_NEWSET:
10620                         if (nft_trans_set_update(trans)) {
10621                                 nft_trans_destroy(trans);
10622                                 break;
10623                         }
10624                         nft_use_dec_restore(&trans->ctx.table->use);
10625                         if (nft_trans_set_bound(trans)) {
10626                                 nft_trans_destroy(trans);
10627                                 break;
10628                         }
10629                         nft_trans_set(trans)->dead = 1;
10630                         list_del_rcu(&nft_trans_set(trans)->list);
10631                         break;
10632                 case NFT_MSG_DELSET:
10633                 case NFT_MSG_DESTROYSET:
10634                         nft_use_inc_restore(&trans->ctx.table->use);
10635                         nft_clear(trans->ctx.net, nft_trans_set(trans));
10636                         if (nft_trans_set(trans)->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
10637                                 nft_map_activate(&trans->ctx, nft_trans_set(trans));
10638
10639                         nft_trans_destroy(trans);
10640                         break;
10641                 case NFT_MSG_NEWSETELEM:
10642                         if (nft_trans_elem_set_bound(trans)) {
10643                                 nft_trans_destroy(trans);
10644                                 break;
10645                         }
10646                         te = (struct nft_trans_elem *)trans->data;
10647                         nft_setelem_remove(net, te->set, te->elem_priv);
10648                         if (!nft_setelem_is_catchall(te->set, te->elem_priv))
10649                                 atomic_dec(&te->set->nelems);
10650
10651                         if (te->set->ops->abort &&
10652                             list_empty(&te->set->pending_update)) {
10653                                 list_add_tail(&te->set->pending_update,
10654                                               &set_update_list);
10655                         }
10656                         break;
10657                 case NFT_MSG_DELSETELEM:
10658                 case NFT_MSG_DESTROYSETELEM:
10659                         te = (struct nft_trans_elem *)trans->data;
10660
10661                         if (!nft_setelem_active_next(net, te->set, te->elem_priv)) {
10662                                 nft_setelem_data_activate(net, te->set, te->elem_priv);
10663                                 nft_setelem_activate(net, te->set, te->elem_priv);
10664                         }
10665                         if (!nft_setelem_is_catchall(te->set, te->elem_priv))
10666                                 te->set->ndeact--;
10667
10668                         if (te->set->ops->abort &&
10669                             list_empty(&te->set->pending_update)) {
10670                                 list_add_tail(&te->set->pending_update,
10671                                               &set_update_list);
10672                         }
10673                         nft_trans_destroy(trans);
10674                         break;
10675                 case NFT_MSG_NEWOBJ:
10676                         if (nft_trans_obj_update(trans)) {
10677                                 nft_obj_destroy(&trans->ctx, nft_trans_obj_newobj(trans));
10678                                 nft_trans_destroy(trans);
10679                         } else {
10680                                 nft_use_dec_restore(&trans->ctx.table->use);
10681                                 nft_obj_del(nft_trans_obj(trans));
10682                         }
10683                         break;
10684                 case NFT_MSG_DELOBJ:
10685                 case NFT_MSG_DESTROYOBJ:
10686                         nft_use_inc_restore(&trans->ctx.table->use);
10687                         nft_clear(trans->ctx.net, nft_trans_obj(trans));
10688                         nft_trans_destroy(trans);
10689                         break;
10690                 case NFT_MSG_NEWFLOWTABLE:
10691                         if (nft_trans_flowtable_update(trans)) {
10692                                 nft_unregister_flowtable_net_hooks(net,
10693                                                 &nft_trans_flowtable_hooks(trans));
10694                         } else {
10695                                 nft_use_dec_restore(&trans->ctx.table->use);
10696                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
10697                                 nft_unregister_flowtable_net_hooks(net,
10698                                                 &nft_trans_flowtable(trans)->hook_list);
10699                         }
10700                         break;
10701                 case NFT_MSG_DELFLOWTABLE:
10702                 case NFT_MSG_DESTROYFLOWTABLE:
10703                         if (nft_trans_flowtable_update(trans)) {
10704                                 list_splice(&nft_trans_flowtable_hooks(trans),
10705                                             &nft_trans_flowtable(trans)->hook_list);
10706                         } else {
10707                                 nft_use_inc_restore(&trans->ctx.table->use);
10708                                 nft_clear(trans->ctx.net, nft_trans_flowtable(trans));
10709                         }
10710                         nft_trans_destroy(trans);
10711                         break;
10712                 }
10713         }
10714
10715         nft_set_abort_update(&set_update_list);
10716
10717         synchronize_rcu();
10718
10719         list_for_each_entry_safe_reverse(trans, next,
10720                                          &nft_net->commit_list, list) {
10721                 nft_trans_list_del(trans);
10722                 nf_tables_abort_release(trans);
10723         }
10724
10725         return err;
10726 }
10727
10728 static int nf_tables_abort(struct net *net, struct sk_buff *skb,
10729                            enum nfnl_abort_action action)
10730 {
10731         struct nftables_pernet *nft_net = nft_pernet(net);
10732         unsigned int gc_seq;
10733         int ret;
10734
10735         gc_seq = nft_gc_seq_begin(nft_net);
10736         ret = __nf_tables_abort(net, action);
10737         nft_gc_seq_end(nft_net, gc_seq);
10738
10739         WARN_ON_ONCE(!list_empty(&nft_net->commit_list));
10740
10741         /* module autoload needs to happen after GC sequence update because it
10742          * temporarily releases and grabs mutex again.
10743          */
10744         if (action == NFNL_ABORT_AUTOLOAD)
10745                 nf_tables_module_autoload(net);
10746         else
10747                 nf_tables_module_autoload_cleanup(net);
10748
10749         mutex_unlock(&nft_net->commit_mutex);
10750
10751         return ret;
10752 }
10753
10754 static bool nf_tables_valid_genid(struct net *net, u32 genid)
10755 {
10756         struct nftables_pernet *nft_net = nft_pernet(net);
10757         bool genid_ok;
10758
10759         mutex_lock(&nft_net->commit_mutex);
10760         nft_net->tstamp = get_jiffies_64();
10761
10762         genid_ok = genid == 0 || nft_net->base_seq == genid;
10763         if (!genid_ok)
10764                 mutex_unlock(&nft_net->commit_mutex);
10765
10766         /* else, commit mutex has to be released by commit or abort function */
10767         return genid_ok;
10768 }
10769
10770 static const struct nfnetlink_subsystem nf_tables_subsys = {
10771         .name           = "nf_tables",
10772         .subsys_id      = NFNL_SUBSYS_NFTABLES,
10773         .cb_count       = NFT_MSG_MAX,
10774         .cb             = nf_tables_cb,
10775         .commit         = nf_tables_commit,
10776         .abort          = nf_tables_abort,
10777         .valid_genid    = nf_tables_valid_genid,
10778         .owner          = THIS_MODULE,
10779 };
10780
10781 int nft_chain_validate_dependency(const struct nft_chain *chain,
10782                                   enum nft_chain_types type)
10783 {
10784         const struct nft_base_chain *basechain;
10785
10786         if (nft_is_base_chain(chain)) {
10787                 basechain = nft_base_chain(chain);
10788                 if (basechain->type->type != type)
10789                         return -EOPNOTSUPP;
10790         }
10791         return 0;
10792 }
10793 EXPORT_SYMBOL_GPL(nft_chain_validate_dependency);
10794
10795 int nft_chain_validate_hooks(const struct nft_chain *chain,
10796                              unsigned int hook_flags)
10797 {
10798         struct nft_base_chain *basechain;
10799
10800         if (nft_is_base_chain(chain)) {
10801                 basechain = nft_base_chain(chain);
10802
10803                 if ((1 << basechain->ops.hooknum) & hook_flags)
10804                         return 0;
10805
10806                 return -EOPNOTSUPP;
10807         }
10808
10809         return 0;
10810 }
10811 EXPORT_SYMBOL_GPL(nft_chain_validate_hooks);
10812
10813 /*
10814  * Loop detection - walk through the ruleset beginning at the destination chain
10815  * of a new jump until either the source chain is reached (loop) or all
10816  * reachable chains have been traversed.
10817  *
10818  * The loop check is performed whenever a new jump verdict is added to an
10819  * expression or verdict map or a verdict map is bound to a new chain.
10820  */
10821
10822 static int nf_tables_check_loops(const struct nft_ctx *ctx,
10823                                  const struct nft_chain *chain);
10824
10825 static int nft_check_loops(const struct nft_ctx *ctx,
10826                            const struct nft_set_ext *ext)
10827 {
10828         const struct nft_data *data;
10829         int ret;
10830
10831         data = nft_set_ext_data(ext);
10832         switch (data->verdict.code) {
10833         case NFT_JUMP:
10834         case NFT_GOTO:
10835                 ret = nf_tables_check_loops(ctx, data->verdict.chain);
10836                 break;
10837         default:
10838                 ret = 0;
10839                 break;
10840         }
10841
10842         return ret;
10843 }
10844
10845 static int nf_tables_loop_check_setelem(const struct nft_ctx *ctx,
10846                                         struct nft_set *set,
10847                                         const struct nft_set_iter *iter,
10848                                         struct nft_elem_priv *elem_priv)
10849 {
10850         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
10851
10852         if (!nft_set_elem_active(ext, iter->genmask))
10853                 return 0;
10854
10855         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
10856             *nft_set_ext_flags(ext) & NFT_SET_ELEM_INTERVAL_END)
10857                 return 0;
10858
10859         return nft_check_loops(ctx, ext);
10860 }
10861
10862 static int nft_set_catchall_loops(const struct nft_ctx *ctx,
10863                                   struct nft_set *set)
10864 {
10865         u8 genmask = nft_genmask_next(ctx->net);
10866         struct nft_set_elem_catchall *catchall;
10867         struct nft_set_ext *ext;
10868         int ret = 0;
10869
10870         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
10871                 ext = nft_set_elem_ext(set, catchall->elem);
10872                 if (!nft_set_elem_active(ext, genmask))
10873                         continue;
10874
10875                 ret = nft_check_loops(ctx, ext);
10876                 if (ret < 0)
10877                         return ret;
10878         }
10879
10880         return ret;
10881 }
10882
10883 static int nf_tables_check_loops(const struct nft_ctx *ctx,
10884                                  const struct nft_chain *chain)
10885 {
10886         const struct nft_rule *rule;
10887         const struct nft_expr *expr, *last;
10888         struct nft_set *set;
10889         struct nft_set_binding *binding;
10890         struct nft_set_iter iter;
10891
10892         if (ctx->chain == chain)
10893                 return -ELOOP;
10894
10895         if (fatal_signal_pending(current))
10896                 return -EINTR;
10897
10898         list_for_each_entry(rule, &chain->rules, list) {
10899                 nft_rule_for_each_expr(expr, last, rule) {
10900                         struct nft_immediate_expr *priv;
10901                         const struct nft_data *data;
10902                         int err;
10903
10904                         if (strcmp(expr->ops->type->name, "immediate"))
10905                                 continue;
10906
10907                         priv = nft_expr_priv(expr);
10908                         if (priv->dreg != NFT_REG_VERDICT)
10909                                 continue;
10910
10911                         data = &priv->data;
10912                         switch (data->verdict.code) {
10913                         case NFT_JUMP:
10914                         case NFT_GOTO:
10915                                 err = nf_tables_check_loops(ctx,
10916                                                         data->verdict.chain);
10917                                 if (err < 0)
10918                                         return err;
10919                                 break;
10920                         default:
10921                                 break;
10922                         }
10923                 }
10924         }
10925
10926         list_for_each_entry(set, &ctx->table->sets, list) {
10927                 if (!nft_is_active_next(ctx->net, set))
10928                         continue;
10929                 if (!(set->flags & NFT_SET_MAP) ||
10930                     set->dtype != NFT_DATA_VERDICT)
10931                         continue;
10932
10933                 list_for_each_entry(binding, &set->bindings, list) {
10934                         if (!(binding->flags & NFT_SET_MAP) ||
10935                             binding->chain != chain)
10936                                 continue;
10937
10938                         iter.genmask    = nft_genmask_next(ctx->net);
10939                         iter.type       = NFT_ITER_UPDATE;
10940                         iter.skip       = 0;
10941                         iter.count      = 0;
10942                         iter.err        = 0;
10943                         iter.fn         = nf_tables_loop_check_setelem;
10944
10945                         set->ops->walk(ctx, set, &iter);
10946                         if (!iter.err)
10947                                 iter.err = nft_set_catchall_loops(ctx, set);
10948
10949                         if (iter.err < 0)
10950                                 return iter.err;
10951                 }
10952         }
10953
10954         return 0;
10955 }
10956
10957 /**
10958  *      nft_parse_u32_check - fetch u32 attribute and check for maximum value
10959  *
10960  *      @attr: netlink attribute to fetch value from
10961  *      @max: maximum value to be stored in dest
10962  *      @dest: pointer to the variable
10963  *
10964  *      Parse, check and store a given u32 netlink attribute into variable.
10965  *      This function returns -ERANGE if the value goes over maximum value.
10966  *      Otherwise a 0 is returned and the attribute value is stored in the
10967  *      destination variable.
10968  */
10969 int nft_parse_u32_check(const struct nlattr *attr, int max, u32 *dest)
10970 {
10971         u32 val;
10972
10973         val = ntohl(nla_get_be32(attr));
10974         if (val > max)
10975                 return -ERANGE;
10976
10977         *dest = val;
10978         return 0;
10979 }
10980 EXPORT_SYMBOL_GPL(nft_parse_u32_check);
10981
10982 static int nft_parse_register(const struct nlattr *attr, u32 *preg)
10983 {
10984         unsigned int reg;
10985
10986         reg = ntohl(nla_get_be32(attr));
10987         switch (reg) {
10988         case NFT_REG_VERDICT...NFT_REG_4:
10989                 *preg = reg * NFT_REG_SIZE / NFT_REG32_SIZE;
10990                 break;
10991         case NFT_REG32_00...NFT_REG32_15:
10992                 *preg = reg + NFT_REG_SIZE / NFT_REG32_SIZE - NFT_REG32_00;
10993                 break;
10994         default:
10995                 return -ERANGE;
10996         }
10997
10998         return 0;
10999 }
11000
11001 /**
11002  *      nft_dump_register - dump a register value to a netlink attribute
11003  *
11004  *      @skb: socket buffer
11005  *      @attr: attribute number
11006  *      @reg: register number
11007  *
11008  *      Construct a netlink attribute containing the register number. For
11009  *      compatibility reasons, register numbers being a multiple of 4 are
11010  *      translated to the corresponding 128 bit register numbers.
11011  */
11012 int nft_dump_register(struct sk_buff *skb, unsigned int attr, unsigned int reg)
11013 {
11014         if (reg % (NFT_REG_SIZE / NFT_REG32_SIZE) == 0)
11015                 reg = reg / (NFT_REG_SIZE / NFT_REG32_SIZE);
11016         else
11017                 reg = reg - NFT_REG_SIZE / NFT_REG32_SIZE + NFT_REG32_00;
11018
11019         return nla_put_be32(skb, attr, htonl(reg));
11020 }
11021 EXPORT_SYMBOL_GPL(nft_dump_register);
11022
11023 static int nft_validate_register_load(enum nft_registers reg, unsigned int len)
11024 {
11025         if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
11026                 return -EINVAL;
11027         if (len == 0)
11028                 return -EINVAL;
11029         if (reg * NFT_REG32_SIZE + len > sizeof_field(struct nft_regs, data))
11030                 return -ERANGE;
11031
11032         return 0;
11033 }
11034
11035 int nft_parse_register_load(const struct nlattr *attr, u8 *sreg, u32 len)
11036 {
11037         u32 reg;
11038         int err;
11039
11040         err = nft_parse_register(attr, &reg);
11041         if (err < 0)
11042                 return err;
11043
11044         err = nft_validate_register_load(reg, len);
11045         if (err < 0)
11046                 return err;
11047
11048         *sreg = reg;
11049         return 0;
11050 }
11051 EXPORT_SYMBOL_GPL(nft_parse_register_load);
11052
11053 static int nft_validate_register_store(const struct nft_ctx *ctx,
11054                                        enum nft_registers reg,
11055                                        const struct nft_data *data,
11056                                        enum nft_data_types type,
11057                                        unsigned int len)
11058 {
11059         int err;
11060
11061         switch (reg) {
11062         case NFT_REG_VERDICT:
11063                 if (type != NFT_DATA_VERDICT)
11064                         return -EINVAL;
11065
11066                 if (data != NULL &&
11067                     (data->verdict.code == NFT_GOTO ||
11068                      data->verdict.code == NFT_JUMP)) {
11069                         err = nf_tables_check_loops(ctx, data->verdict.chain);
11070                         if (err < 0)
11071                                 return err;
11072                 }
11073
11074                 return 0;
11075         default:
11076                 if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
11077                         return -EINVAL;
11078                 if (len == 0)
11079                         return -EINVAL;
11080                 if (reg * NFT_REG32_SIZE + len >
11081                     sizeof_field(struct nft_regs, data))
11082                         return -ERANGE;
11083
11084                 if (data != NULL && type != NFT_DATA_VALUE)
11085                         return -EINVAL;
11086                 return 0;
11087         }
11088 }
11089
11090 int nft_parse_register_store(const struct nft_ctx *ctx,
11091                              const struct nlattr *attr, u8 *dreg,
11092                              const struct nft_data *data,
11093                              enum nft_data_types type, unsigned int len)
11094 {
11095         int err;
11096         u32 reg;
11097
11098         err = nft_parse_register(attr, &reg);
11099         if (err < 0)
11100                 return err;
11101
11102         err = nft_validate_register_store(ctx, reg, data, type, len);
11103         if (err < 0)
11104                 return err;
11105
11106         *dreg = reg;
11107         return 0;
11108 }
11109 EXPORT_SYMBOL_GPL(nft_parse_register_store);
11110
11111 static const struct nla_policy nft_verdict_policy[NFTA_VERDICT_MAX + 1] = {
11112         [NFTA_VERDICT_CODE]     = { .type = NLA_U32 },
11113         [NFTA_VERDICT_CHAIN]    = { .type = NLA_STRING,
11114                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
11115         [NFTA_VERDICT_CHAIN_ID] = { .type = NLA_U32 },
11116 };
11117
11118 static int nft_verdict_init(const struct nft_ctx *ctx, struct nft_data *data,
11119                             struct nft_data_desc *desc, const struct nlattr *nla)
11120 {
11121         u8 genmask = nft_genmask_next(ctx->net);
11122         struct nlattr *tb[NFTA_VERDICT_MAX + 1];
11123         struct nft_chain *chain;
11124         int err;
11125
11126         err = nla_parse_nested_deprecated(tb, NFTA_VERDICT_MAX, nla,
11127                                           nft_verdict_policy, NULL);
11128         if (err < 0)
11129                 return err;
11130
11131         if (!tb[NFTA_VERDICT_CODE])
11132                 return -EINVAL;
11133
11134         /* zero padding hole for memcmp */
11135         memset(data, 0, sizeof(*data));
11136         data->verdict.code = ntohl(nla_get_be32(tb[NFTA_VERDICT_CODE]));
11137
11138         switch (data->verdict.code) {
11139         case NF_ACCEPT:
11140         case NF_DROP:
11141         case NF_QUEUE:
11142                 break;
11143         case NFT_CONTINUE:
11144         case NFT_BREAK:
11145         case NFT_RETURN:
11146                 break;
11147         case NFT_JUMP:
11148         case NFT_GOTO:
11149                 if (tb[NFTA_VERDICT_CHAIN]) {
11150                         chain = nft_chain_lookup(ctx->net, ctx->table,
11151                                                  tb[NFTA_VERDICT_CHAIN],
11152                                                  genmask);
11153                 } else if (tb[NFTA_VERDICT_CHAIN_ID]) {
11154                         chain = nft_chain_lookup_byid(ctx->net, ctx->table,
11155                                                       tb[NFTA_VERDICT_CHAIN_ID],
11156                                                       genmask);
11157                         if (IS_ERR(chain))
11158                                 return PTR_ERR(chain);
11159                 } else {
11160                         return -EINVAL;
11161                 }
11162
11163                 if (IS_ERR(chain))
11164                         return PTR_ERR(chain);
11165                 if (nft_is_base_chain(chain))
11166                         return -EOPNOTSUPP;
11167                 if (nft_chain_is_bound(chain))
11168                         return -EINVAL;
11169                 if (desc->flags & NFT_DATA_DESC_SETELEM &&
11170                     chain->flags & NFT_CHAIN_BINDING)
11171                         return -EINVAL;
11172                 if (!nft_use_inc(&chain->use))
11173                         return -EMFILE;
11174
11175                 data->verdict.chain = chain;
11176                 break;
11177         default:
11178                 return -EINVAL;
11179         }
11180
11181         desc->len = sizeof(data->verdict);
11182
11183         return 0;
11184 }
11185
11186 static void nft_verdict_uninit(const struct nft_data *data)
11187 {
11188         struct nft_chain *chain;
11189
11190         switch (data->verdict.code) {
11191         case NFT_JUMP:
11192         case NFT_GOTO:
11193                 chain = data->verdict.chain;
11194                 nft_use_dec(&chain->use);
11195                 break;
11196         }
11197 }
11198
11199 int nft_verdict_dump(struct sk_buff *skb, int type, const struct nft_verdict *v)
11200 {
11201         struct nlattr *nest;
11202
11203         nest = nla_nest_start_noflag(skb, type);
11204         if (!nest)
11205                 goto nla_put_failure;
11206
11207         if (nla_put_be32(skb, NFTA_VERDICT_CODE, htonl(v->code)))
11208                 goto nla_put_failure;
11209
11210         switch (v->code) {
11211         case NFT_JUMP:
11212         case NFT_GOTO:
11213                 if (nla_put_string(skb, NFTA_VERDICT_CHAIN,
11214                                    v->chain->name))
11215                         goto nla_put_failure;
11216         }
11217         nla_nest_end(skb, nest);
11218         return 0;
11219
11220 nla_put_failure:
11221         return -1;
11222 }
11223
11224 static int nft_value_init(const struct nft_ctx *ctx,
11225                           struct nft_data *data, struct nft_data_desc *desc,
11226                           const struct nlattr *nla)
11227 {
11228         unsigned int len;
11229
11230         len = nla_len(nla);
11231         if (len == 0)
11232                 return -EINVAL;
11233         if (len > desc->size)
11234                 return -EOVERFLOW;
11235         if (desc->len) {
11236                 if (len != desc->len)
11237                         return -EINVAL;
11238         } else {
11239                 desc->len = len;
11240         }
11241
11242         nla_memcpy(data->data, nla, len);
11243
11244         return 0;
11245 }
11246
11247 static int nft_value_dump(struct sk_buff *skb, const struct nft_data *data,
11248                           unsigned int len)
11249 {
11250         return nla_put(skb, NFTA_DATA_VALUE, len, data->data);
11251 }
11252
11253 static const struct nla_policy nft_data_policy[NFTA_DATA_MAX + 1] = {
11254         [NFTA_DATA_VALUE]       = { .type = NLA_BINARY },
11255         [NFTA_DATA_VERDICT]     = { .type = NLA_NESTED },
11256 };
11257
11258 /**
11259  *      nft_data_init - parse nf_tables data netlink attributes
11260  *
11261  *      @ctx: context of the expression using the data
11262  *      @data: destination struct nft_data
11263  *      @desc: data description
11264  *      @nla: netlink attribute containing data
11265  *
11266  *      Parse the netlink data attributes and initialize a struct nft_data.
11267  *      The type and length of data are returned in the data description.
11268  *
11269  *      The caller can indicate that it only wants to accept data of type
11270  *      NFT_DATA_VALUE by passing NULL for the ctx argument.
11271  */
11272 int nft_data_init(const struct nft_ctx *ctx, struct nft_data *data,
11273                   struct nft_data_desc *desc, const struct nlattr *nla)
11274 {
11275         struct nlattr *tb[NFTA_DATA_MAX + 1];
11276         int err;
11277
11278         if (WARN_ON_ONCE(!desc->size))
11279                 return -EINVAL;
11280
11281         err = nla_parse_nested_deprecated(tb, NFTA_DATA_MAX, nla,
11282                                           nft_data_policy, NULL);
11283         if (err < 0)
11284                 return err;
11285
11286         if (tb[NFTA_DATA_VALUE]) {
11287                 if (desc->type != NFT_DATA_VALUE)
11288                         return -EINVAL;
11289
11290                 err = nft_value_init(ctx, data, desc, tb[NFTA_DATA_VALUE]);
11291         } else if (tb[NFTA_DATA_VERDICT] && ctx != NULL) {
11292                 if (desc->type != NFT_DATA_VERDICT)
11293                         return -EINVAL;
11294
11295                 err = nft_verdict_init(ctx, data, desc, tb[NFTA_DATA_VERDICT]);
11296         } else {
11297                 err = -EINVAL;
11298         }
11299
11300         return err;
11301 }
11302 EXPORT_SYMBOL_GPL(nft_data_init);
11303
11304 /**
11305  *      nft_data_release - release a nft_data item
11306  *
11307  *      @data: struct nft_data to release
11308  *      @type: type of data
11309  *
11310  *      Release a nft_data item. NFT_DATA_VALUE types can be silently discarded,
11311  *      all others need to be released by calling this function.
11312  */
11313 void nft_data_release(const struct nft_data *data, enum nft_data_types type)
11314 {
11315         if (type < NFT_DATA_VERDICT)
11316                 return;
11317         switch (type) {
11318         case NFT_DATA_VERDICT:
11319                 return nft_verdict_uninit(data);
11320         default:
11321                 WARN_ON(1);
11322         }
11323 }
11324 EXPORT_SYMBOL_GPL(nft_data_release);
11325
11326 int nft_data_dump(struct sk_buff *skb, int attr, const struct nft_data *data,
11327                   enum nft_data_types type, unsigned int len)
11328 {
11329         struct nlattr *nest;
11330         int err;
11331
11332         nest = nla_nest_start_noflag(skb, attr);
11333         if (nest == NULL)
11334                 return -1;
11335
11336         switch (type) {
11337         case NFT_DATA_VALUE:
11338                 err = nft_value_dump(skb, data, len);
11339                 break;
11340         case NFT_DATA_VERDICT:
11341                 err = nft_verdict_dump(skb, NFTA_DATA_VERDICT, &data->verdict);
11342                 break;
11343         default:
11344                 err = -EINVAL;
11345                 WARN_ON(1);
11346         }
11347
11348         nla_nest_end(skb, nest);
11349         return err;
11350 }
11351 EXPORT_SYMBOL_GPL(nft_data_dump);
11352
11353 int __nft_release_basechain(struct nft_ctx *ctx)
11354 {
11355         struct nft_rule *rule, *nr;
11356
11357         if (WARN_ON(!nft_is_base_chain(ctx->chain)))
11358                 return 0;
11359
11360         nf_tables_unregister_hook(ctx->net, ctx->chain->table, ctx->chain);
11361         list_for_each_entry_safe(rule, nr, &ctx->chain->rules, list) {
11362                 list_del(&rule->list);
11363                 nft_use_dec(&ctx->chain->use);
11364                 nf_tables_rule_release(ctx, rule);
11365         }
11366         nft_chain_del(ctx->chain);
11367         nft_use_dec(&ctx->table->use);
11368         nf_tables_chain_destroy(ctx);
11369
11370         return 0;
11371 }
11372 EXPORT_SYMBOL_GPL(__nft_release_basechain);
11373
11374 static void __nft_release_hook(struct net *net, struct nft_table *table)
11375 {
11376         struct nft_flowtable *flowtable;
11377         struct nft_chain *chain;
11378
11379         list_for_each_entry(chain, &table->chains, list)
11380                 __nf_tables_unregister_hook(net, table, chain, true);
11381         list_for_each_entry(flowtable, &table->flowtables, list)
11382                 __nft_unregister_flowtable_net_hooks(net, &flowtable->hook_list,
11383                                                      true);
11384 }
11385
11386 static void __nft_release_hooks(struct net *net)
11387 {
11388         struct nftables_pernet *nft_net = nft_pernet(net);
11389         struct nft_table *table;
11390
11391         list_for_each_entry(table, &nft_net->tables, list) {
11392                 if (nft_table_has_owner(table))
11393                         continue;
11394
11395                 __nft_release_hook(net, table);
11396         }
11397 }
11398
11399 static void __nft_release_table(struct net *net, struct nft_table *table)
11400 {
11401         struct nft_flowtable *flowtable, *nf;
11402         struct nft_chain *chain, *nc;
11403         struct nft_object *obj, *ne;
11404         struct nft_rule *rule, *nr;
11405         struct nft_set *set, *ns;
11406         struct nft_ctx ctx = {
11407                 .net    = net,
11408                 .family = NFPROTO_NETDEV,
11409         };
11410
11411         ctx.family = table->family;
11412         ctx.table = table;
11413         list_for_each_entry(chain, &table->chains, list) {
11414                 if (nft_chain_binding(chain))
11415                         continue;
11416
11417                 ctx.chain = chain;
11418                 list_for_each_entry_safe(rule, nr, &chain->rules, list) {
11419                         list_del(&rule->list);
11420                         nft_use_dec(&chain->use);
11421                         nf_tables_rule_release(&ctx, rule);
11422                 }
11423         }
11424         list_for_each_entry_safe(flowtable, nf, &table->flowtables, list) {
11425                 list_del(&flowtable->list);
11426                 nft_use_dec(&table->use);
11427                 nf_tables_flowtable_destroy(flowtable);
11428         }
11429         list_for_each_entry_safe(set, ns, &table->sets, list) {
11430                 list_del(&set->list);
11431                 nft_use_dec(&table->use);
11432                 if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
11433                         nft_map_deactivate(&ctx, set);
11434
11435                 nft_set_destroy(&ctx, set);
11436         }
11437         list_for_each_entry_safe(obj, ne, &table->objects, list) {
11438                 nft_obj_del(obj);
11439                 nft_use_dec(&table->use);
11440                 nft_obj_destroy(&ctx, obj);
11441         }
11442         list_for_each_entry_safe(chain, nc, &table->chains, list) {
11443                 ctx.chain = chain;
11444                 nft_chain_del(chain);
11445                 nft_use_dec(&table->use);
11446                 nf_tables_chain_destroy(&ctx);
11447         }
11448         nf_tables_table_destroy(&ctx);
11449 }
11450
11451 static void __nft_release_tables(struct net *net)
11452 {
11453         struct nftables_pernet *nft_net = nft_pernet(net);
11454         struct nft_table *table, *nt;
11455
11456         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
11457                 if (nft_table_has_owner(table))
11458                         continue;
11459
11460                 list_del(&table->list);
11461
11462                 __nft_release_table(net, table);
11463         }
11464 }
11465
11466 static int nft_rcv_nl_event(struct notifier_block *this, unsigned long event,
11467                             void *ptr)
11468 {
11469         struct nft_table *table, *to_delete[8];
11470         struct nftables_pernet *nft_net;
11471         struct netlink_notify *n = ptr;
11472         struct net *net = n->net;
11473         unsigned int deleted;
11474         bool restart = false;
11475         unsigned int gc_seq;
11476
11477         if (event != NETLINK_URELEASE || n->protocol != NETLINK_NETFILTER)
11478                 return NOTIFY_DONE;
11479
11480         nft_net = nft_pernet(net);
11481         deleted = 0;
11482         mutex_lock(&nft_net->commit_mutex);
11483
11484         gc_seq = nft_gc_seq_begin(nft_net);
11485
11486         if (!list_empty(&nf_tables_destroy_list))
11487                 nf_tables_trans_destroy_flush_work();
11488 again:
11489         list_for_each_entry(table, &nft_net->tables, list) {
11490                 if (nft_table_has_owner(table) &&
11491                     n->portid == table->nlpid) {
11492                         if (table->flags & NFT_TABLE_F_PERSIST) {
11493                                 table->flags &= ~NFT_TABLE_F_OWNER;
11494                                 continue;
11495                         }
11496                         __nft_release_hook(net, table);
11497                         list_del_rcu(&table->list);
11498                         to_delete[deleted++] = table;
11499                         if (deleted >= ARRAY_SIZE(to_delete))
11500                                 break;
11501                 }
11502         }
11503         if (deleted) {
11504                 restart = deleted >= ARRAY_SIZE(to_delete);
11505                 synchronize_rcu();
11506                 while (deleted)
11507                         __nft_release_table(net, to_delete[--deleted]);
11508
11509                 if (restart)
11510                         goto again;
11511         }
11512         nft_gc_seq_end(nft_net, gc_seq);
11513
11514         mutex_unlock(&nft_net->commit_mutex);
11515
11516         return NOTIFY_DONE;
11517 }
11518
11519 static struct notifier_block nft_nl_notifier = {
11520         .notifier_call  = nft_rcv_nl_event,
11521 };
11522
11523 static int __net_init nf_tables_init_net(struct net *net)
11524 {
11525         struct nftables_pernet *nft_net = nft_pernet(net);
11526
11527         INIT_LIST_HEAD(&nft_net->tables);
11528         INIT_LIST_HEAD(&nft_net->commit_list);
11529         INIT_LIST_HEAD(&nft_net->binding_list);
11530         INIT_LIST_HEAD(&nft_net->module_list);
11531         INIT_LIST_HEAD(&nft_net->notify_list);
11532         mutex_init(&nft_net->commit_mutex);
11533         nft_net->base_seq = 1;
11534         nft_net->gc_seq = 0;
11535         nft_net->validate_state = NFT_VALIDATE_SKIP;
11536
11537         return 0;
11538 }
11539
11540 static void __net_exit nf_tables_pre_exit_net(struct net *net)
11541 {
11542         struct nftables_pernet *nft_net = nft_pernet(net);
11543
11544         mutex_lock(&nft_net->commit_mutex);
11545         __nft_release_hooks(net);
11546         mutex_unlock(&nft_net->commit_mutex);
11547 }
11548
11549 static void __net_exit nf_tables_exit_net(struct net *net)
11550 {
11551         struct nftables_pernet *nft_net = nft_pernet(net);
11552         unsigned int gc_seq;
11553
11554         mutex_lock(&nft_net->commit_mutex);
11555
11556         gc_seq = nft_gc_seq_begin(nft_net);
11557
11558         WARN_ON_ONCE(!list_empty(&nft_net->commit_list));
11559
11560         if (!list_empty(&nft_net->module_list))
11561                 nf_tables_module_autoload_cleanup(net);
11562
11563         __nft_release_tables(net);
11564
11565         nft_gc_seq_end(nft_net, gc_seq);
11566
11567         mutex_unlock(&nft_net->commit_mutex);
11568         WARN_ON_ONCE(!list_empty(&nft_net->tables));
11569         WARN_ON_ONCE(!list_empty(&nft_net->module_list));
11570         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
11571 }
11572
11573 static void nf_tables_exit_batch(struct list_head *net_exit_list)
11574 {
11575         flush_work(&trans_gc_work);
11576 }
11577
11578 static struct pernet_operations nf_tables_net_ops = {
11579         .init           = nf_tables_init_net,
11580         .pre_exit       = nf_tables_pre_exit_net,
11581         .exit           = nf_tables_exit_net,
11582         .exit_batch     = nf_tables_exit_batch,
11583         .id             = &nf_tables_net_id,
11584         .size           = sizeof(struct nftables_pernet),
11585 };
11586
11587 static int __init nf_tables_module_init(void)
11588 {
11589         int err;
11590
11591         err = register_pernet_subsys(&nf_tables_net_ops);
11592         if (err < 0)
11593                 return err;
11594
11595         err = nft_chain_filter_init();
11596         if (err < 0)
11597                 goto err_chain_filter;
11598
11599         err = nf_tables_core_module_init();
11600         if (err < 0)
11601                 goto err_core_module;
11602
11603         err = register_netdevice_notifier(&nf_tables_flowtable_notifier);
11604         if (err < 0)
11605                 goto err_netdev_notifier;
11606
11607         err = rhltable_init(&nft_objname_ht, &nft_objname_ht_params);
11608         if (err < 0)
11609                 goto err_rht_objname;
11610
11611         err = nft_offload_init();
11612         if (err < 0)
11613                 goto err_offload;
11614
11615         err = netlink_register_notifier(&nft_nl_notifier);
11616         if (err < 0)
11617                 goto err_netlink_notifier;
11618
11619         /* must be last */
11620         err = nfnetlink_subsys_register(&nf_tables_subsys);
11621         if (err < 0)
11622                 goto err_nfnl_subsys;
11623
11624         nft_chain_route_init();
11625
11626         return err;
11627
11628 err_nfnl_subsys:
11629         netlink_unregister_notifier(&nft_nl_notifier);
11630 err_netlink_notifier:
11631         nft_offload_exit();
11632 err_offload:
11633         rhltable_destroy(&nft_objname_ht);
11634 err_rht_objname:
11635         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
11636 err_netdev_notifier:
11637         nf_tables_core_module_exit();
11638 err_core_module:
11639         nft_chain_filter_fini();
11640 err_chain_filter:
11641         unregister_pernet_subsys(&nf_tables_net_ops);
11642         return err;
11643 }
11644
11645 static void __exit nf_tables_module_exit(void)
11646 {
11647         nfnetlink_subsys_unregister(&nf_tables_subsys);
11648         netlink_unregister_notifier(&nft_nl_notifier);
11649         nft_offload_exit();
11650         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
11651         nft_chain_filter_fini();
11652         nft_chain_route_fini();
11653         nf_tables_trans_destroy_flush_work();
11654         unregister_pernet_subsys(&nf_tables_net_ops);
11655         cancel_work_sync(&trans_gc_work);
11656         cancel_work_sync(&trans_destroy_work);
11657         rcu_barrier();
11658         rhltable_destroy(&nft_objname_ht);
11659         nf_tables_core_module_exit();
11660 }
11661
11662 module_init(nf_tables_module_init);
11663 module_exit(nf_tables_module_exit);
11664
11665 MODULE_LICENSE("GPL");
11666 MODULE_AUTHOR("Patrick McHardy <[email protected]>");
11667 MODULE_DESCRIPTION("Framework for packet filtering and classification");
11668 MODULE_ALIAS_NFNL_SUBSYS(NFNL_SUBSYS_NFTABLES);