net/netfilter/nf_tables_api.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2007-2009 Patrick McHardy <[email protected]>
   4  *
   5  * Development of this code funded by Astaro AG (http://www.astaro.com/)
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/init.h>
  10 #include <linux/list.h>
  11 #include <linux/skbuff.h>
  12 #include <linux/netlink.h>
  13 #include <linux/vmalloc.h>
  14 #include <linux/rhashtable.h>
  15 #include <linux/netfilter.h>
  16 #include <linux/netfilter/nfnetlink.h>
  17 #include <linux/netfilter/nf_tables.h>
  18 #include <net/netfilter/nf_flow_table.h>
  19 #include <net/netfilter/nf_tables_core.h>
  20 #include <net/netfilter/nf_tables.h>
  21 #include <net/netfilter/nf_tables_offload.h>
  22 #include <net/net_namespace.h>
  23 #include <net/sock.h>
  24
  25 #define NFT_MODULE_AUTOLOAD_LIMIT (MODULE_NAME_LEN - sizeof("nft-expr-255-"))
  26
  27 static LIST_HEAD(nf_tables_expressions);
  28 static LIST_HEAD(nf_tables_objects);
  29 static LIST_HEAD(nf_tables_flowtables);
  30 static LIST_HEAD(nf_tables_destroy_list);
  31 static DEFINE_SPINLOCK(nf_tables_destroy_list_lock);
  32 static u64 table_handle;
  33
  34 enum {
  35         NFT_VALIDATE_SKIP       = 0,
  36         NFT_VALIDATE_NEED,
  37         NFT_VALIDATE_DO,
  38 };
  39
  40 static struct rhltable nft_objname_ht;
  41
  42 static u32 nft_chain_hash(const void *data, u32 len, u32 seed);
  43 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed);
  44 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *, const void *);
  45
  46 static u32 nft_objname_hash(const void *data, u32 len, u32 seed);
  47 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed);
  48 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *, const void *);
  49
  50 static const struct rhashtable_params nft_chain_ht_params = {
  51         .head_offset            = offsetof(struct nft_chain, rhlhead),
  52         .key_offset             = offsetof(struct nft_chain, name),
  53         .hashfn                 = nft_chain_hash,
  54         .obj_hashfn             = nft_chain_hash_obj,
  55         .obj_cmpfn              = nft_chain_hash_cmp,
  56         .automatic_shrinking    = true,
  57 };
  58
  59 static const struct rhashtable_params nft_objname_ht_params = {
  60         .head_offset            = offsetof(struct nft_object, rhlhead),
  61         .key_offset             = offsetof(struct nft_object, key),
  62         .hashfn                 = nft_objname_hash,
  63         .obj_hashfn             = nft_objname_hash_obj,
  64         .obj_cmpfn              = nft_objname_hash_cmp,
  65         .automatic_shrinking    = true,
  66 };
  67
  68 static void nft_validate_state_update(struct net *net, u8 new_validate_state)
  69 {
  70         switch (net->nft.validate_state) {
  71         case NFT_VALIDATE_SKIP:
  72                 WARN_ON_ONCE(new_validate_state == NFT_VALIDATE_DO);
  73                 break;
  74         case NFT_VALIDATE_NEED:
  75                 break;
  76         case NFT_VALIDATE_DO:
  77                 if (new_validate_state == NFT_VALIDATE_NEED)
  78                         return;
  79         }
  80
  81         net->nft.validate_state = new_validate_state;
  82 }
  83 static void nf_tables_trans_destroy_work(struct work_struct *w);
  84 static DECLARE_WORK(trans_destroy_work, nf_tables_trans_destroy_work);
  85
  86 static void nft_ctx_init(struct nft_ctx *ctx,
  87                          struct net *net,
  88                          const struct sk_buff *skb,
  89                          const struct nlmsghdr *nlh,
  90                          u8 family,
  91                          struct nft_table *table,
  92                          struct nft_chain *chain,
  93                          const struct nlattr * const *nla)
  94 {
  95         ctx->net        = net;
  96         ctx->family     = family;
  97         ctx->level      = 0;
  98         ctx->table      = table;
  99         ctx->chain      = chain;
 100         ctx->nla        = nla;
 101         ctx->portid     = NETLINK_CB(skb).portid;
 102         ctx->report     = nlmsg_report(nlh);
 103         ctx->flags      = nlh->nlmsg_flags;
 104         ctx->seq        = nlh->nlmsg_seq;
 105 }
 106
 107 static struct nft_trans *nft_trans_alloc_gfp(const struct nft_ctx *ctx,
 108                                              int msg_type, u32 size, gfp_t gfp)
 109 {
 110         struct nft_trans *trans;
 111
 112         trans = kzalloc(sizeof(struct nft_trans) + size, gfp);
 113         if (trans == NULL)
 114                 return NULL;
 115
 116         trans->msg_type = msg_type;
 117         trans->ctx      = *ctx;
 118
 119         return trans;
 120 }
 121
 122 static struct nft_trans *nft_trans_alloc(const struct nft_ctx *ctx,
 123                                          int msg_type, u32 size)
 124 {
 125         return nft_trans_alloc_gfp(ctx, msg_type, size, GFP_KERNEL);
 126 }
 127
 128 static void nft_trans_destroy(struct nft_trans *trans)
 129 {
 130         list_del(&trans->list);
 131         kfree(trans);
 132 }
 133
 134 static void nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set)
 135 {
 136         struct net *net = ctx->net;
 137         struct nft_trans *trans;
 138
 139         if (!nft_set_is_anonymous(set))
 140                 return;
 141
 142         list_for_each_entry_reverse(trans, &net->nft.commit_list, list) {
 143                 switch (trans->msg_type) {
 144                 case NFT_MSG_NEWSET:
 145                         if (nft_trans_set(trans) == set)
 146                                 nft_trans_set_bound(trans) = true;
 147                         break;
 148                 case NFT_MSG_NEWSETELEM:
 149                         if (nft_trans_elem_set(trans) == set)
 150                                 nft_trans_elem_set_bound(trans) = true;
 151                         break;
 152                 }
 153         }
 154 }
 155
 156 static int nft_netdev_register_hooks(struct net *net,
 157                                      struct list_head *hook_list)
 158 {
 159         struct nft_hook *hook;
 160         int err, j;
 161
 162         j = 0;
 163         list_for_each_entry(hook, hook_list, list) {
 164                 err = nf_register_net_hook(net, &hook->ops);
 165                 if (err < 0)
 166                         goto err_register;
 167
 168                 j++;
 169         }
 170         return 0;
 171
 172 err_register:
 173         list_for_each_entry(hook, hook_list, list) {
 174                 if (j-- <= 0)
 175                         break;
 176
 177                 nf_unregister_net_hook(net, &hook->ops);
 178         }
 179         return err;
 180 }
 181
 182 static void nft_netdev_unregister_hooks(struct net *net,
 183                                         struct list_head *hook_list)
 184 {
 185         struct nft_hook *hook;
 186
 187         list_for_each_entry(hook, hook_list, list)
 188                 nf_unregister_net_hook(net, &hook->ops);
 189 }
 190
 191 static int nft_register_basechain_hooks(struct net *net, int family,
 192                                         struct nft_base_chain *basechain)
 193 {
 194         if (family == NFPROTO_NETDEV)
 195                 return nft_netdev_register_hooks(net, &basechain->hook_list);
 196
 197         return nf_register_net_hook(net, &basechain->ops);
 198 }
 199
 200 static void nft_unregister_basechain_hooks(struct net *net, int family,
 201                                            struct nft_base_chain *basechain)
 202 {
 203         if (family == NFPROTO_NETDEV)
 204                 nft_netdev_unregister_hooks(net, &basechain->hook_list);
 205         else
 206                 nf_unregister_net_hook(net, &basechain->ops);
 207 }
 208
 209 static int nf_tables_register_hook(struct net *net,
 210                                    const struct nft_table *table,
 211                                    struct nft_chain *chain)
 212 {
 213         struct nft_base_chain *basechain;
 214         const struct nf_hook_ops *ops;
 215
 216         if (table->flags & NFT_TABLE_F_DORMANT ||
 217             !nft_is_base_chain(chain))
 218                 return 0;
 219
 220         basechain = nft_base_chain(chain);
 221         ops = &basechain->ops;
 222
 223         if (basechain->type->ops_register)
 224                 return basechain->type->ops_register(net, ops);
 225
 226         return nft_register_basechain_hooks(net, table->family, basechain);
 227 }
 228
 229 static void nf_tables_unregister_hook(struct net *net,
 230                                       const struct nft_table *table,
 231                                       struct nft_chain *chain)
 232 {
 233         struct nft_base_chain *basechain;
 234         const struct nf_hook_ops *ops;
 235
 236         if (table->flags & NFT_TABLE_F_DORMANT ||
 237             !nft_is_base_chain(chain))
 238                 return;
 239         basechain = nft_base_chain(chain);
 240         ops = &basechain->ops;
 241
 242         if (basechain->type->ops_unregister)
 243                 return basechain->type->ops_unregister(net, ops);
 244
 245         nft_unregister_basechain_hooks(net, table->family, basechain);
 246 }
 247
 248 static int nft_trans_table_add(struct nft_ctx *ctx, int msg_type)
 249 {
 250         struct nft_trans *trans;
 251
 252         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_table));
 253         if (trans == NULL)
 254                 return -ENOMEM;
 255
 256         if (msg_type == NFT_MSG_NEWTABLE)
 257                 nft_activate_next(ctx->net, ctx->table);
 258
 259         list_add_tail(&trans->list, &ctx->net->nft.commit_list);
 260         return 0;
 261 }
 262
 263 static int nft_deltable(struct nft_ctx *ctx)
 264 {
 265         int err;
 266
 267         err = nft_trans_table_add(ctx, NFT_MSG_DELTABLE);
 268         if (err < 0)
 269                 return err;
 270
 271         nft_deactivate_next(ctx->net, ctx->table);
 272         return err;
 273 }
 274
 275 static struct nft_trans *nft_trans_chain_add(struct nft_ctx *ctx, int msg_type)
 276 {
 277         struct nft_trans *trans;
 278
 279         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_chain));
 280         if (trans == NULL)
 281                 return ERR_PTR(-ENOMEM);
 282
 283         if (msg_type == NFT_MSG_NEWCHAIN) {
 284                 nft_activate_next(ctx->net, ctx->chain);
 285
 286                 if (ctx->nla[NFTA_CHAIN_ID]) {
 287                         nft_trans_chain_id(trans) =
 288                                 ntohl(nla_get_be32(ctx->nla[NFTA_CHAIN_ID]));
 289                 }
 290         }
 291
 292         list_add_tail(&trans->list, &ctx->net->nft.commit_list);
 293         return trans;
 294 }
 295
 296 static int nft_delchain(struct nft_ctx *ctx)
 297 {
 298         struct nft_trans *trans;
 299
 300         trans = nft_trans_chain_add(ctx, NFT_MSG_DELCHAIN);
 301         if (IS_ERR(trans))
 302                 return PTR_ERR(trans);
 303
 304         ctx->table->use--;
 305         nft_deactivate_next(ctx->net, ctx->chain);
 306
 307         return 0;
 308 }
 309
 310 static void nft_rule_expr_activate(const struct nft_ctx *ctx,
 311                                    struct nft_rule *rule)
 312 {
 313         struct nft_expr *expr;
 314
 315         expr = nft_expr_first(rule);
 316         while (expr != nft_expr_last(rule) && expr->ops) {
 317                 if (expr->ops->activate)
 318                         expr->ops->activate(ctx, expr);
 319
 320                 expr = nft_expr_next(expr);
 321         }
 322 }
 323
 324 static void nft_rule_expr_deactivate(const struct nft_ctx *ctx,
 325                                      struct nft_rule *rule,
 326                                      enum nft_trans_phase phase)
 327 {
 328         struct nft_expr *expr;
 329
 330         expr = nft_expr_first(rule);
 331         while (expr != nft_expr_last(rule) && expr->ops) {
 332                 if (expr->ops->deactivate)
 333                         expr->ops->deactivate(ctx, expr, phase);
 334
 335                 expr = nft_expr_next(expr);
 336         }
 337 }
 338
 339 static int
 340 nf_tables_delrule_deactivate(struct nft_ctx *ctx, struct nft_rule *rule)
 341 {
 342         /* You cannot delete the same rule twice */
 343         if (nft_is_active_next(ctx->net, rule)) {
 344                 nft_deactivate_next(ctx->net, rule);
 345                 ctx->chain->use--;
 346                 return 0;
 347         }
 348         return -ENOENT;
 349 }
 350
 351 static struct nft_trans *nft_trans_rule_add(struct nft_ctx *ctx, int msg_type,
 352                                             struct nft_rule *rule)
 353 {
 354         struct nft_trans *trans;
 355
 356         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_rule));
 357         if (trans == NULL)
 358                 return NULL;
 359
 360         if (msg_type == NFT_MSG_NEWRULE && ctx->nla[NFTA_RULE_ID] != NULL) {
 361                 nft_trans_rule_id(trans) =
 362                         ntohl(nla_get_be32(ctx->nla[NFTA_RULE_ID]));
 363         }
 364         nft_trans_rule(trans) = rule;
 365         list_add_tail(&trans->list, &ctx->net->nft.commit_list);
 366
 367         return trans;
 368 }
 369
 370 static int nft_delrule(struct nft_ctx *ctx, struct nft_rule *rule)
 371 {
 372         struct nft_flow_rule *flow;
 373         struct nft_trans *trans;
 374         int err;
 375
 376         trans = nft_trans_rule_add(ctx, NFT_MSG_DELRULE, rule);
 377         if (trans == NULL)
 378                 return -ENOMEM;
 379
 380         if (ctx->chain->flags & NFT_CHAIN_HW_OFFLOAD) {
 381                 flow = nft_flow_rule_create(ctx->net, rule);
 382                 if (IS_ERR(flow)) {
 383                         nft_trans_destroy(trans);
 384                         return PTR_ERR(flow);
 385                 }
 386
 387                 nft_trans_flow_rule(trans) = flow;
 388         }
 389
 390         err = nf_tables_delrule_deactivate(ctx, rule);
 391         if (err < 0) {
 392                 nft_trans_destroy(trans);
 393                 return err;
 394         }
 395         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_PREPARE);
 396
 397         return 0;
 398 }
 399
 400 static int nft_delrule_by_chain(struct nft_ctx *ctx)
 401 {
 402         struct nft_rule *rule;
 403         int err;
 404
 405         list_for_each_entry(rule, &ctx->chain->rules, list) {
 406                 if (!nft_is_active_next(ctx->net, rule))
 407                         continue;
 408
 409                 err = nft_delrule(ctx, rule);
 410                 if (err < 0)
 411                         return err;
 412         }
 413         return 0;
 414 }
 415
 416 static int nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
 417                              struct nft_set *set)
 418 {
 419         struct nft_trans *trans;
 420
 421         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_set));
 422         if (trans == NULL)
 423                 return -ENOMEM;
 424
 425         if (msg_type == NFT_MSG_NEWSET && ctx->nla[NFTA_SET_ID] != NULL) {
 426                 nft_trans_set_id(trans) =
 427                         ntohl(nla_get_be32(ctx->nla[NFTA_SET_ID]));
 428                 nft_activate_next(ctx->net, set);
 429         }
 430         nft_trans_set(trans) = set;
 431         list_add_tail(&trans->list, &ctx->net->nft.commit_list);
 432
 433         return 0;
 434 }
 435
 436 static int nft_delset(const struct nft_ctx *ctx, struct nft_set *set)
 437 {
 438         int err;
 439
 440         err = nft_trans_set_add(ctx, NFT_MSG_DELSET, set);
 441         if (err < 0)
 442                 return err;
 443
 444         nft_deactivate_next(ctx->net, set);
 445         ctx->table->use--;
 446
 447         return err;
 448 }
 449
 450 static int nft_trans_obj_add(struct nft_ctx *ctx, int msg_type,
 451                              struct nft_object *obj)
 452 {
 453         struct nft_trans *trans;
 454
 455         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_obj));
 456         if (trans == NULL)
 457                 return -ENOMEM;
 458
 459         if (msg_type == NFT_MSG_NEWOBJ)
 460                 nft_activate_next(ctx->net, obj);
 461
 462         nft_trans_obj(trans) = obj;
 463         list_add_tail(&trans->list, &ctx->net->nft.commit_list);
 464
 465         return 0;
 466 }
 467
 468 static int nft_delobj(struct nft_ctx *ctx, struct nft_object *obj)
 469 {
 470         int err;
 471
 472         err = nft_trans_obj_add(ctx, NFT_MSG_DELOBJ, obj);
 473         if (err < 0)
 474                 return err;
 475
 476         nft_deactivate_next(ctx->net, obj);
 477         ctx->table->use--;
 478
 479         return err;
 480 }
 481
 482 static int nft_trans_flowtable_add(struct nft_ctx *ctx, int msg_type,
 483                                    struct nft_flowtable *flowtable)
 484 {
 485         struct nft_trans *trans;
 486
 487         trans = nft_trans_alloc(ctx, msg_type,
 488                                 sizeof(struct nft_trans_flowtable));
 489         if (trans == NULL)
 490                 return -ENOMEM;
 491
 492         if (msg_type == NFT_MSG_NEWFLOWTABLE)
 493                 nft_activate_next(ctx->net, flowtable);
 494
 495         nft_trans_flowtable(trans) = flowtable;
 496         list_add_tail(&trans->list, &ctx->net->nft.commit_list);
 497
 498         return 0;
 499 }
 500
 501 static int nft_delflowtable(struct nft_ctx *ctx,
 502                             struct nft_flowtable *flowtable)
 503 {
 504         int err;
 505
 506         err = nft_trans_flowtable_add(ctx, NFT_MSG_DELFLOWTABLE, flowtable);
 507         if (err < 0)
 508                 return err;
 509
 510         nft_deactivate_next(ctx->net, flowtable);
 511         ctx->table->use--;
 512
 513         return err;
 514 }
 515
 516 /*
 517  * Tables
 518  */
 519
 520 static struct nft_table *nft_table_lookup(const struct net *net,
 521                                           const struct nlattr *nla,
 522                                           u8 family, u8 genmask)
 523 {
 524         struct nft_table *table;
 525
 526         if (nla == NULL)
 527                 return ERR_PTR(-EINVAL);
 528
 529         list_for_each_entry_rcu(table, &net->nft.tables, list,
 530                                 lockdep_is_held(&net->nft.commit_mutex)) {
 531                 if (!nla_strcmp(nla, table->name) &&
 532                     table->family == family &&
 533                     nft_active_genmask(table, genmask))
 534                         return table;
 535         }
 536
 537         return ERR_PTR(-ENOENT);
 538 }
 539
 540 static struct nft_table *nft_table_lookup_byhandle(const struct net *net,
 541                                                    const struct nlattr *nla,
 542                                                    u8 genmask)
 543 {
 544         struct nft_table *table;
 545
 546         list_for_each_entry(table, &net->nft.tables, list) {
 547                 if (be64_to_cpu(nla_get_be64(nla)) == table->handle &&
 548                     nft_active_genmask(table, genmask))
 549                         return table;
 550         }
 551
 552         return ERR_PTR(-ENOENT);
 553 }
 554
 555 static inline u64 nf_tables_alloc_handle(struct nft_table *table)
 556 {
 557         return ++table->hgenerator;
 558 }
 559
 560 static const struct nft_chain_type *chain_type[NFPROTO_NUMPROTO][NFT_CHAIN_T_MAX];
 561
 562 static const struct nft_chain_type *
 563 __nft_chain_type_get(u8 family, enum nft_chain_types type)
 564 {
 565         if (family >= NFPROTO_NUMPROTO ||
 566             type >= NFT_CHAIN_T_MAX)
 567                 return NULL;
 568
 569         return chain_type[family][type];
 570 }
 571
 572 static const struct nft_chain_type *
 573 __nf_tables_chain_type_lookup(const struct nlattr *nla, u8 family)
 574 {
 575         const struct nft_chain_type *type;
 576         int i;
 577
 578         for (i = 0; i < NFT_CHAIN_T_MAX; i++) {
 579                 type = __nft_chain_type_get(family, i);
 580                 if (!type)
 581                         continue;
 582                 if (!nla_strcmp(nla, type->name))
 583                         return type;
 584         }
 585         return NULL;
 586 }
 587
 588 struct nft_module_request {
 589         struct list_head        list;
 590         char                    module[MODULE_NAME_LEN];
 591         bool                    done;
 592 };
 593
 594 #ifdef CONFIG_MODULES
 595 static int nft_request_module(struct net *net, const char *fmt, ...)
 596 {
 597         char module_name[MODULE_NAME_LEN];
 598         struct nft_module_request *req;
 599         va_list args;
 600         int ret;
 601
 602         va_start(args, fmt);
 603         ret = vsnprintf(module_name, MODULE_NAME_LEN, fmt, args);
 604         va_end(args);
 605         if (ret >= MODULE_NAME_LEN)
 606                 return 0;
 607
 608         list_for_each_entry(req, &net->nft.module_list, list) {
 609                 if (!strcmp(req->module, module_name)) {
 610                         if (req->done)
 611                                 return 0;
 612
 613                         /* A request to load this module already exists. */
 614                         return -EAGAIN;
 615                 }
 616         }
 617
 618         req = kmalloc(sizeof(*req), GFP_KERNEL);
 619         if (!req)
 620                 return -ENOMEM;
 621
 622         req->done = false;
 623         strlcpy(req->module, module_name, MODULE_NAME_LEN);
 624         list_add_tail(&req->list, &net->nft.module_list);
 625
 626         return -EAGAIN;
 627 }
 628 #endif
 629
 630 static void lockdep_nfnl_nft_mutex_not_held(void)
 631 {
 632 #ifdef CONFIG_PROVE_LOCKING
 633         WARN_ON_ONCE(lockdep_nfnl_is_held(NFNL_SUBSYS_NFTABLES));
 634 #endif
 635 }
 636
 637 static const struct nft_chain_type *
 638 nf_tables_chain_type_lookup(struct net *net, const struct nlattr *nla,
 639                             u8 family, bool autoload)
 640 {
 641         const struct nft_chain_type *type;
 642
 643         type = __nf_tables_chain_type_lookup(nla, family);
 644         if (type != NULL)
 645                 return type;
 646
 647         lockdep_nfnl_nft_mutex_not_held();
 648 #ifdef CONFIG_MODULES
 649         if (autoload) {
 650                 if (nft_request_module(net, "nft-chain-%u-%.*s", family,
 651                                        nla_len(nla),
 652                                        (const char *)nla_data(nla)) == -EAGAIN)
 653                         return ERR_PTR(-EAGAIN);
 654         }
 655 #endif
 656         return ERR_PTR(-ENOENT);
 657 }
 658
 659 static const struct nla_policy nft_table_policy[NFTA_TABLE_MAX + 1] = {
 660         [NFTA_TABLE_NAME]       = { .type = NLA_STRING,
 661                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
 662         [NFTA_TABLE_FLAGS]      = { .type = NLA_U32 },
 663         [NFTA_TABLE_HANDLE]     = { .type = NLA_U64 },
 664 };
 665
 666 static int nf_tables_fill_table_info(struct sk_buff *skb, struct net *net,
 667                                      u32 portid, u32 seq, int event, u32 flags,
 668                                      int family, const struct nft_table *table)
 669 {
 670         struct nlmsghdr *nlh;
 671         struct nfgenmsg *nfmsg;
 672
 673         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
 674         nlh = nlmsg_put(skb, portid, seq, event, sizeof(struct nfgenmsg), flags);
 675         if (nlh == NULL)
 676                 goto nla_put_failure;
 677
 678         nfmsg = nlmsg_data(nlh);
 679         nfmsg->nfgen_family     = family;
 680         nfmsg->version          = NFNETLINK_V0;
 681         nfmsg->res_id           = htons(net->nft.base_seq & 0xffff);
 682
 683         if (nla_put_string(skb, NFTA_TABLE_NAME, table->name) ||
 684             nla_put_be32(skb, NFTA_TABLE_FLAGS, htonl(table->flags)) ||
 685             nla_put_be32(skb, NFTA_TABLE_USE, htonl(table->use)) ||
 686             nla_put_be64(skb, NFTA_TABLE_HANDLE, cpu_to_be64(table->handle),
 687                          NFTA_TABLE_PAD))
 688                 goto nla_put_failure;
 689
 690         nlmsg_end(skb, nlh);
 691         return 0;
 692
 693 nla_put_failure:
 694         nlmsg_trim(skb, nlh);
 695         return -1;
 696 }
 697
 698 static void nf_tables_table_notify(const struct nft_ctx *ctx, int event)
 699 {
 700         struct sk_buff *skb;
 701         int err;
 702
 703         if (!ctx->report &&
 704             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
 705                 return;
 706
 707         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
 708         if (skb == NULL)
 709                 goto err;
 710
 711         err = nf_tables_fill_table_info(skb, ctx->net, ctx->portid, ctx->seq,
 712                                         event, 0, ctx->family, ctx->table);
 713         if (err < 0) {
 714                 kfree_skb(skb);
 715                 goto err;
 716         }
 717
 718         nfnetlink_send(skb, ctx->net, ctx->portid, NFNLGRP_NFTABLES,
 719                        ctx->report, GFP_KERNEL);
 720         return;
 721 err:
 722         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
 723 }
 724
 725 static int nf_tables_dump_tables(struct sk_buff *skb,
 726                                  struct netlink_callback *cb)
 727 {
 728         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
 729         const struct nft_table *table;
 730         unsigned int idx = 0, s_idx = cb->args[0];
 731         struct net *net = sock_net(skb->sk);
 732         int family = nfmsg->nfgen_family;
 733
 734         rcu_read_lock();
 735         cb->seq = net->nft.base_seq;
 736
 737         list_for_each_entry_rcu(table, &net->nft.tables, list) {
 738                 if (family != NFPROTO_UNSPEC && family != table->family)
 739                         continue;
 740
 741                 if (idx < s_idx)
 742                         goto cont;
 743                 if (idx > s_idx)
 744                         memset(&cb->args[1], 0,
 745                                sizeof(cb->args) - sizeof(cb->args[0]));
 746                 if (!nft_is_active(net, table))
 747                         continue;
 748                 if (nf_tables_fill_table_info(skb, net,
 749                                               NETLINK_CB(cb->skb).portid,
 750                                               cb->nlh->nlmsg_seq,
 751                                               NFT_MSG_NEWTABLE, NLM_F_MULTI,
 752                                               table->family, table) < 0)
 753                         goto done;
 754
 755                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
 756 cont:
 757                 idx++;
 758         }
 759 done:
 760         rcu_read_unlock();
 761         cb->args[0] = idx;
 762         return skb->len;
 763 }
 764
 765 static int nft_netlink_dump_start_rcu(struct sock *nlsk, struct sk_buff *skb,
 766                                       const struct nlmsghdr *nlh,
 767                                       struct netlink_dump_control *c)
 768 {
 769         int err;
 770
 771         if (!try_module_get(THIS_MODULE))
 772                 return -EINVAL;
 773
 774         rcu_read_unlock();
 775         err = netlink_dump_start(nlsk, skb, nlh, c);
 776         rcu_read_lock();
 777         module_put(THIS_MODULE);
 778
 779         return err;
 780 }
 781
 782 /* called with rcu_read_lock held */
 783 static int nf_tables_gettable(struct net *net, struct sock *nlsk,
 784                               struct sk_buff *skb, const struct nlmsghdr *nlh,
 785                               const struct nlattr * const nla[],
 786                               struct netlink_ext_ack *extack)
 787 {
 788         const struct nfgenmsg *nfmsg = nlmsg_data(nlh);
 789         u8 genmask = nft_genmask_cur(net);
 790         const struct nft_table *table;
 791         struct sk_buff *skb2;
 792         int family = nfmsg->nfgen_family;
 793         int err;
 794
 795         if (nlh->nlmsg_flags & NLM_F_DUMP) {
 796                 struct netlink_dump_control c = {
 797                         .dump = nf_tables_dump_tables,
 798                         .module = THIS_MODULE,
 799                 };
 800
 801                 return nft_netlink_dump_start_rcu(nlsk, skb, nlh, &c);
 802         }
 803
 804         table = nft_table_lookup(net, nla[NFTA_TABLE_NAME], family, genmask);
 805         if (IS_ERR(table)) {
 806                 NL_SET_BAD_ATTR(extack, nla[NFTA_TABLE_NAME]);
 807                 return PTR_ERR(table);
 808         }
 809
 810         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
 811         if (!skb2)
 812                 return -ENOMEM;
 813
 814         err = nf_tables_fill_table_info(skb2, net, NETLINK_CB(skb).portid,
 815                                         nlh->nlmsg_seq, NFT_MSG_NEWTABLE, 0,
 816                                         family, table);
 817         if (err < 0)
 818                 goto err;
 819
 820         return nlmsg_unicast(nlsk, skb2, NETLINK_CB(skb).portid);
 821
 822 err:
 823         kfree_skb(skb2);
 824         return err;
 825 }
 826
 827 static void nft_table_disable(struct net *net, struct nft_table *table, u32 cnt)
 828 {
 829         struct nft_chain *chain;
 830         u32 i = 0;
 831
 832         list_for_each_entry(chain, &table->chains, list) {
 833                 if (!nft_is_active_next(net, chain))
 834                         continue;
 835                 if (!nft_is_base_chain(chain))
 836                         continue;
 837
 838                 if (cnt && i++ == cnt)
 839                         break;
 840
 841                 nft_unregister_basechain_hooks(net, table->family,
 842                                                nft_base_chain(chain));
 843         }
 844 }
 845
 846 static int nf_tables_table_enable(struct net *net, struct nft_table *table)
 847 {
 848         struct nft_chain *chain;
 849         int err, i = 0;
 850
 851         list_for_each_entry(chain, &table->chains, list) {
 852                 if (!nft_is_active_next(net, chain))
 853                         continue;
 854                 if (!nft_is_base_chain(chain))
 855                         continue;
 856
 857                 err = nft_register_basechain_hooks(net, table->family,
 858                                                    nft_base_chain(chain));
 859                 if (err < 0)
 860                         goto err_register_hooks;
 861
 862                 i++;
 863         }
 864         return 0;
 865
 866 err_register_hooks:
 867         if (i)
 868                 nft_table_disable(net, table, i);
 869         return err;
 870 }
 871
 872 static void nf_tables_table_disable(struct net *net, struct nft_table *table)
 873 {
 874         nft_table_disable(net, table, 0);
 875 }
 876
 877 static int nf_tables_updtable(struct nft_ctx *ctx)
 878 {
 879         struct nft_trans *trans;
 880         u32 flags;
 881         int ret = 0;
 882
 883         if (!ctx->nla[NFTA_TABLE_FLAGS])
 884                 return 0;
 885
 886         flags = ntohl(nla_get_be32(ctx->nla[NFTA_TABLE_FLAGS]));
 887         if (flags & ~NFT_TABLE_F_DORMANT)
 888                 return -EINVAL;
 889
 890         if (flags == ctx->table->flags)
 891                 return 0;
 892
 893         trans = nft_trans_alloc(ctx, NFT_MSG_NEWTABLE,
 894                                 sizeof(struct nft_trans_table));
 895         if (trans == NULL)
 896                 return -ENOMEM;
 897
 898         if ((flags & NFT_TABLE_F_DORMANT) &&
 899             !(ctx->table->flags & NFT_TABLE_F_DORMANT)) {
 900                 nft_trans_table_enable(trans) = false;
 901         } else if (!(flags & NFT_TABLE_F_DORMANT) &&
 902                    ctx->table->flags & NFT_TABLE_F_DORMANT) {
 903                 ret = nf_tables_table_enable(ctx->net, ctx->table);
 904                 if (ret >= 0) {
 905                         ctx->table->flags &= ~NFT_TABLE_F_DORMANT;
 906                         nft_trans_table_enable(trans) = true;
 907                 }
 908         }
 909         if (ret < 0)
 910                 goto err;
 911
 912         nft_trans_table_update(trans) = true;
 913         list_add_tail(&trans->list, &ctx->net->nft.commit_list);
 914         return 0;
 915 err:
 916         nft_trans_destroy(trans);
 917         return ret;
 918 }
 919
 920 static u32 nft_chain_hash(const void *data, u32 len, u32 seed)
 921 {
 922         const char *name = data;
 923
 924         return jhash(name, strlen(name), seed);
 925 }
 926
 927 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed)
 928 {
 929         const struct nft_chain *chain = data;
 930
 931         return nft_chain_hash(chain->name, 0, seed);
 932 }
 933
 934 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *arg,
 935                               const void *ptr)
 936 {
 937         const struct nft_chain *chain = ptr;
 938         const char *name = arg->key;
 939
 940         return strcmp(chain->name, name);
 941 }
 942
 943 static u32 nft_objname_hash(const void *data, u32 len, u32 seed)
 944 {
 945         const struct nft_object_hash_key *k = data;
 946
 947         seed ^= hash_ptr(k->table, 32);
 948
 949         return jhash(k->name, strlen(k->name), seed);
 950 }
 951
 952 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed)
 953 {
 954         const struct nft_object *obj = data;
 955
 956         return nft_objname_hash(&obj->key, 0, seed);
 957 }
 958
 959 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *arg,
 960                                 const void *ptr)
 961 {
 962         const struct nft_object_hash_key *k = arg->key;
 963         const struct nft_object *obj = ptr;
 964
 965         if (obj->key.table != k->table)
 966                 return -1;
 967
 968         return strcmp(obj->key.name, k->name);
 969 }
 970
 971 static int nf_tables_newtable(struct net *net, struct sock *nlsk,
 972                               struct sk_buff *skb, const struct nlmsghdr *nlh,
 973                               const struct nlattr * const nla[],
 974                               struct netlink_ext_ack *extack)
 975 {
 976         const struct nfgenmsg *nfmsg = nlmsg_data(nlh);
 977         u8 genmask = nft_genmask_next(net);
 978         int family = nfmsg->nfgen_family;
 979         const struct nlattr *attr;
 980         struct nft_table *table;
 981         u32 flags = 0;
 982         struct nft_ctx ctx;
 983         int err;
 984
 985         lockdep_assert_held(&net->nft.commit_mutex);
 986         attr = nla[NFTA_TABLE_NAME];
 987         table = nft_table_lookup(net, attr, family, genmask);
 988         if (IS_ERR(table)) {
 989                 if (PTR_ERR(table) != -ENOENT)
 990                         return PTR_ERR(table);
 991         } else {
 992                 if (nlh->nlmsg_flags & NLM_F_EXCL) {
 993                         NL_SET_BAD_ATTR(extack, attr);
 994                         return -EEXIST;
 995                 }
 996                 if (nlh->nlmsg_flags & NLM_F_REPLACE)
 997                         return -EOPNOTSUPP;
 998
 999                 nft_ctx_init(&ctx, net, skb, nlh, family, table, NULL, nla);
1000                 return nf_tables_updtable(&ctx);
1001         }
1002
1003         if (nla[NFTA_TABLE_FLAGS]) {
1004                 flags = ntohl(nla_get_be32(nla[NFTA_TABLE_FLAGS]));
1005                 if (flags & ~NFT_TABLE_F_DORMANT)
1006                         return -EINVAL;
1007         }
1008
1009         err = -ENOMEM;
1010         table = kzalloc(sizeof(*table), GFP_KERNEL);
1011         if (table == NULL)
1012                 goto err_kzalloc;
1013
1014         table->name = nla_strdup(attr, GFP_KERNEL);
1015         if (table->name == NULL)
1016                 goto err_strdup;
1017
1018         err = rhltable_init(&table->chains_ht, &nft_chain_ht_params);
1019         if (err)
1020                 goto err_chain_ht;
1021
1022         INIT_LIST_HEAD(&table->chains);
1023         INIT_LIST_HEAD(&table->sets);
1024         INIT_LIST_HEAD(&table->objects);
1025         INIT_LIST_HEAD(&table->flowtables);
1026         table->family = family;
1027         table->flags = flags;
1028         table->handle = ++table_handle;
1029
1030         nft_ctx_init(&ctx, net, skb, nlh, family, table, NULL, nla);
1031         err = nft_trans_table_add(&ctx, NFT_MSG_NEWTABLE);
1032         if (err < 0)
1033                 goto err_trans;
1034
1035         list_add_tail_rcu(&table->list, &net->nft.tables);
1036         return 0;
1037 err_trans:
1038         rhltable_destroy(&table->chains_ht);
1039 err_chain_ht:
1040         kfree(table->name);
1041 err_strdup:
1042         kfree(table);
1043 err_kzalloc:
1044         return err;
1045 }
1046
1047 static int nft_flush_table(struct nft_ctx *ctx)
1048 {
1049         struct nft_flowtable *flowtable, *nft;
1050         struct nft_chain *chain, *nc;
1051         struct nft_object *obj, *ne;
1052         struct nft_set *set, *ns;
1053         int err;
1054
1055         list_for_each_entry(chain, &ctx->table->chains, list) {
1056                 if (!nft_is_active_next(ctx->net, chain))
1057                         continue;
1058
1059                 if (nft_chain_is_bound(chain))
1060                         continue;
1061
1062                 ctx->chain = chain;
1063
1064                 err = nft_delrule_by_chain(ctx);
1065                 if (err < 0)
1066                         goto out;
1067         }
1068
1069         list_for_each_entry_safe(set, ns, &ctx->table->sets, list) {
1070                 if (!nft_is_active_next(ctx->net, set))
1071                         continue;
1072
1073                 if (nft_set_is_anonymous(set) &&
1074                     !list_empty(&set->bindings))
1075                         continue;
1076
1077                 err = nft_delset(ctx, set);
1078                 if (err < 0)
1079                         goto out;
1080         }
1081
1082         list_for_each_entry_safe(flowtable, nft, &ctx->table->flowtables, list) {
1083                 if (!nft_is_active_next(ctx->net, flowtable))
1084                         continue;
1085
1086                 err = nft_delflowtable(ctx, flowtable);
1087                 if (err < 0)
1088                         goto out;
1089         }
1090
1091         list_for_each_entry_safe(obj, ne, &ctx->table->objects, list) {
1092                 if (!nft_is_active_next(ctx->net, obj))
1093                         continue;
1094
1095                 err = nft_delobj(ctx, obj);
1096                 if (err < 0)
1097                         goto out;
1098         }
1099
1100         list_for_each_entry_safe(chain, nc, &ctx->table->chains, list) {
1101                 if (!nft_is_active_next(ctx->net, chain))
1102                         continue;
1103
1104                 if (nft_chain_is_bound(chain))
1105                         continue;
1106
1107                 ctx->chain = chain;
1108
1109                 err = nft_delchain(ctx);
1110                 if (err < 0)
1111                         goto out;
1112         }
1113
1114         err = nft_deltable(ctx);
1115 out:
1116         return err;
1117 }
1118
1119 static int nft_flush(struct nft_ctx *ctx, int family)
1120 {
1121         struct nft_table *table, *nt;
1122         const struct nlattr * const *nla = ctx->nla;
1123         int err = 0;
1124
1125         list_for_each_entry_safe(table, nt, &ctx->net->nft.tables, list) {
1126                 if (family != AF_UNSPEC && table->family != family)
1127                         continue;
1128
1129                 ctx->family = table->family;
1130
1131                 if (!nft_is_active_next(ctx->net, table))
1132                         continue;
1133
1134                 if (nla[NFTA_TABLE_NAME] &&
1135                     nla_strcmp(nla[NFTA_TABLE_NAME], table->name) != 0)
1136                         continue;
1137
1138                 ctx->table = table;
1139
1140                 err = nft_flush_table(ctx);
1141                 if (err < 0)
1142                         goto out;
1143         }
1144 out:
1145         return err;
1146 }
1147
1148 static int nf_tables_deltable(struct net *net, struct sock *nlsk,
1149                               struct sk_buff *skb, const struct nlmsghdr *nlh,
1150                               const struct nlattr * const nla[],
1151                               struct netlink_ext_ack *extack)
1152 {
1153         const struct nfgenmsg *nfmsg = nlmsg_data(nlh);
1154         u8 genmask = nft_genmask_next(net);
1155         int family = nfmsg->nfgen_family;
1156         const struct nlattr *attr;
1157         struct nft_table *table;
1158         struct nft_ctx ctx;
1159
1160         nft_ctx_init(&ctx, net, skb, nlh, 0, NULL, NULL, nla);
1161         if (family == AF_UNSPEC ||
1162             (!nla[NFTA_TABLE_NAME] && !nla[NFTA_TABLE_HANDLE]))
1163                 return nft_flush(&ctx, family);
1164
1165         if (nla[NFTA_TABLE_HANDLE]) {
1166                 attr = nla[NFTA_TABLE_HANDLE];
1167                 table = nft_table_lookup_byhandle(net, attr, genmask);
1168         } else {
1169                 attr = nla[NFTA_TABLE_NAME];
1170                 table = nft_table_lookup(net, attr, family, genmask);
1171         }
1172
1173         if (IS_ERR(table)) {
1174                 NL_SET_BAD_ATTR(extack, attr);
1175                 return PTR_ERR(table);
1176         }
1177
1178         if (nlh->nlmsg_flags & NLM_F_NONREC &&
1179             table->use > 0)
1180                 return -EBUSY;
1181
1182         ctx.family = family;
1183         ctx.table = table;
1184
1185         return nft_flush_table(&ctx);
1186 }
1187
1188 static void nf_tables_table_destroy(struct nft_ctx *ctx)
1189 {
1190         if (WARN_ON(ctx->table->use > 0))
1191                 return;
1192
1193         rhltable_destroy(&ctx->table->chains_ht);
1194         kfree(ctx->table->name);
1195         kfree(ctx->table);
1196 }
1197
1198 void nft_register_chain_type(const struct nft_chain_type *ctype)
1199 {
1200         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1201         if (WARN_ON(__nft_chain_type_get(ctype->family, ctype->type))) {
1202                 nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1203                 return;
1204         }
1205         chain_type[ctype->family][ctype->type] = ctype;
1206         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1207 }
1208 EXPORT_SYMBOL_GPL(nft_register_chain_type);
1209
1210 void nft_unregister_chain_type(const struct nft_chain_type *ctype)
1211 {
1212         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1213         chain_type[ctype->family][ctype->type] = NULL;
1214         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1215 }
1216 EXPORT_SYMBOL_GPL(nft_unregister_chain_type);
1217
1218 /*
1219  * Chains
1220  */
1221
1222 static struct nft_chain *
1223 nft_chain_lookup_byhandle(const struct nft_table *table, u64 handle, u8 genmask)
1224 {
1225         struct nft_chain *chain;
1226
1227         list_for_each_entry(chain, &table->chains, list) {
1228                 if (chain->handle == handle &&
1229                     nft_active_genmask(chain, genmask))
1230                         return chain;
1231         }
1232
1233         return ERR_PTR(-ENOENT);
1234 }
1235
1236 static bool lockdep_commit_lock_is_held(const struct net *net)
1237 {
1238 #ifdef CONFIG_PROVE_LOCKING
1239         return lockdep_is_held(&net->nft.commit_mutex);
1240 #else
1241         return true;
1242 #endif
1243 }
1244
1245 static struct nft_chain *nft_chain_lookup(struct net *net,
1246                                           struct nft_table *table,
1247                                           const struct nlattr *nla, u8 genmask)
1248 {
1249         char search[NFT_CHAIN_MAXNAMELEN + 1];
1250         struct rhlist_head *tmp, *list;
1251         struct nft_chain *chain;
1252
1253         if (nla == NULL)
1254                 return ERR_PTR(-EINVAL);
1255
1256         nla_strlcpy(search, nla, sizeof(search));
1257
1258         WARN_ON(!rcu_read_lock_held() &&
1259                 !lockdep_commit_lock_is_held(net));
1260
1261         chain = ERR_PTR(-ENOENT);
1262         rcu_read_lock();
1263         list = rhltable_lookup(&table->chains_ht, search, nft_chain_ht_params);
1264         if (!list)
1265                 goto out_unlock;
1266
1267         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
1268                 if (nft_active_genmask(chain, genmask))
1269                         goto out_unlock;
1270         }
1271         chain = ERR_PTR(-ENOENT);
1272 out_unlock:
1273         rcu_read_unlock();
1274         return chain;
1275 }
1276
1277 static const struct nla_policy nft_chain_policy[NFTA_CHAIN_MAX + 1] = {
1278         [NFTA_CHAIN_TABLE]      = { .type = NLA_STRING,
1279                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
1280         [NFTA_CHAIN_HANDLE]     = { .type = NLA_U64 },
1281         [NFTA_CHAIN_NAME]       = { .type = NLA_STRING,
1282                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
1283         [NFTA_CHAIN_HOOK]       = { .type = NLA_NESTED },
1284         [NFTA_CHAIN_POLICY]     = { .type = NLA_U32 },
1285         [NFTA_CHAIN_TYPE]       = { .type = NLA_STRING,
1286                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
1287         [NFTA_CHAIN_COUNTERS]   = { .type = NLA_NESTED },
1288         [NFTA_CHAIN_FLAGS]      = { .type = NLA_U32 },
1289         [NFTA_CHAIN_ID]         = { .type = NLA_U32 },
1290 };
1291
1292 static const struct nla_policy nft_hook_policy[NFTA_HOOK_MAX + 1] = {
1293         [NFTA_HOOK_HOOKNUM]     = { .type = NLA_U32 },
1294         [NFTA_HOOK_PRIORITY]    = { .type = NLA_U32 },
1295         [NFTA_HOOK_DEV]         = { .type = NLA_STRING,
1296                                     .len = IFNAMSIZ - 1 },
1297 };
1298
1299 static int nft_dump_stats(struct sk_buff *skb, struct nft_stats __percpu *stats)
1300 {
1301         struct nft_stats *cpu_stats, total;
1302         struct nlattr *nest;
1303         unsigned int seq;
1304         u64 pkts, bytes;
1305         int cpu;
1306
1307         if (!stats)
1308                 return 0;
1309
1310         memset(&total, 0, sizeof(total));
1311         for_each_possible_cpu(cpu) {
1312                 cpu_stats = per_cpu_ptr(stats, cpu);
1313                 do {
1314                         seq = u64_stats_fetch_begin_irq(&cpu_stats->syncp);
1315                         pkts = cpu_stats->pkts;
1316                         bytes = cpu_stats->bytes;
1317                 } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, seq));
1318                 total.pkts += pkts;
1319                 total.bytes += bytes;
1320         }
1321         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_COUNTERS);
1322         if (nest == NULL)
1323                 goto nla_put_failure;
1324
1325         if (nla_put_be64(skb, NFTA_COUNTER_PACKETS, cpu_to_be64(total.pkts),
1326                          NFTA_COUNTER_PAD) ||
1327             nla_put_be64(skb, NFTA_COUNTER_BYTES, cpu_to_be64(total.bytes),
1328                          NFTA_COUNTER_PAD))
1329                 goto nla_put_failure;
1330
1331         nla_nest_end(skb, nest);
1332         return 0;
1333
1334 nla_put_failure:
1335         return -ENOSPC;
1336 }
1337
1338 static int nft_dump_basechain_hook(struct sk_buff *skb, int family,
1339                                    const struct nft_base_chain *basechain)
1340 {
1341         const struct nf_hook_ops *ops = &basechain->ops;
1342         struct nft_hook *hook, *first = NULL;
1343         struct nlattr *nest, *nest_devs;
1344         int n = 0;
1345
1346         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_HOOK);
1347         if (nest == NULL)
1348                 goto nla_put_failure;
1349         if (nla_put_be32(skb, NFTA_HOOK_HOOKNUM, htonl(ops->hooknum)))
1350                 goto nla_put_failure;
1351         if (nla_put_be32(skb, NFTA_HOOK_PRIORITY, htonl(ops->priority)))
1352                 goto nla_put_failure;
1353
1354         if (family == NFPROTO_NETDEV) {
1355                 nest_devs = nla_nest_start_noflag(skb, NFTA_HOOK_DEVS);
1356                 list_for_each_entry(hook, &basechain->hook_list, list) {
1357                         if (!first)
1358                                 first = hook;
1359
1360                         if (nla_put_string(skb, NFTA_DEVICE_NAME,
1361                                            hook->ops.dev->name))
1362                                 goto nla_put_failure;
1363                         n++;
1364                 }
1365                 nla_nest_end(skb, nest_devs);
1366
1367                 if (n == 1 &&
1368                     nla_put_string(skb, NFTA_HOOK_DEV, first->ops.dev->name))
1369                         goto nla_put_failure;
1370         }
1371         nla_nest_end(skb, nest);
1372
1373         return 0;
1374 nla_put_failure:
1375         return -1;
1376 }
1377
1378 static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net,
1379                                      u32 portid, u32 seq, int event, u32 flags,
1380                                      int family, const struct nft_table *table,
1381                                      const struct nft_chain *chain)
1382 {
1383         struct nlmsghdr *nlh;
1384         struct nfgenmsg *nfmsg;
1385
1386         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
1387         nlh = nlmsg_put(skb, portid, seq, event, sizeof(struct nfgenmsg), flags);
1388         if (nlh == NULL)
1389                 goto nla_put_failure;
1390
1391         nfmsg = nlmsg_data(nlh);
1392         nfmsg->nfgen_family     = family;
1393         nfmsg->version          = NFNETLINK_V0;
1394         nfmsg->res_id           = htons(net->nft.base_seq & 0xffff);
1395
1396         if (nla_put_string(skb, NFTA_CHAIN_TABLE, table->name))
1397                 goto nla_put_failure;
1398         if (nla_put_be64(skb, NFTA_CHAIN_HANDLE, cpu_to_be64(chain->handle),
1399                          NFTA_CHAIN_PAD))
1400                 goto nla_put_failure;
1401         if (nla_put_string(skb, NFTA_CHAIN_NAME, chain->name))
1402                 goto nla_put_failure;
1403
1404         if (nft_is_base_chain(chain)) {
1405                 const struct nft_base_chain *basechain = nft_base_chain(chain);
1406                 struct nft_stats __percpu *stats;
1407
1408                 if (nft_dump_basechain_hook(skb, family, basechain))
1409                         goto nla_put_failure;
1410
1411                 if (nla_put_be32(skb, NFTA_CHAIN_POLICY,
1412                                  htonl(basechain->policy)))
1413                         goto nla_put_failure;
1414
1415                 if (nla_put_string(skb, NFTA_CHAIN_TYPE, basechain->type->name))
1416                         goto nla_put_failure;
1417
1418                 stats = rcu_dereference_check(basechain->stats,
1419                                               lockdep_commit_lock_is_held(net));
1420                 if (nft_dump_stats(skb, stats))
1421                         goto nla_put_failure;
1422         }
1423
1424         if (chain->flags &&
1425             nla_put_be32(skb, NFTA_CHAIN_FLAGS, htonl(chain->flags)))
1426                 goto nla_put_failure;
1427
1428         if (nla_put_be32(skb, NFTA_CHAIN_USE, htonl(chain->use)))
1429                 goto nla_put_failure;
1430
1431         nlmsg_end(skb, nlh);
1432         return 0;
1433
1434 nla_put_failure:
1435         nlmsg_trim(skb, nlh);
1436         return -1;
1437 }
1438
1439 static void nf_tables_chain_notify(const struct nft_ctx *ctx, int event)
1440 {
1441         struct sk_buff *skb;
1442         int err;
1443
1444         if (!ctx->report &&
1445             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
1446                 return;
1447
1448         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
1449         if (skb == NULL)
1450                 goto err;
1451
1452         err = nf_tables_fill_chain_info(skb, ctx->net, ctx->portid, ctx->seq,
1453                                         event, 0, ctx->family, ctx->table,
1454                                         ctx->chain);
1455         if (err < 0) {
1456                 kfree_skb(skb);
1457                 goto err;
1458         }
1459
1460         nfnetlink_send(skb, ctx->net, ctx->portid, NFNLGRP_NFTABLES,
1461                        ctx->report, GFP_KERNEL);
1462         return;
1463 err:
1464         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
1465 }
1466
1467 static int nf_tables_dump_chains(struct sk_buff *skb,
1468                                  struct netlink_callback *cb)
1469 {
1470         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
1471         const struct nft_table *table;
1472         const struct nft_chain *chain;
1473         unsigned int idx = 0, s_idx = cb->args[0];
1474         struct net *net = sock_net(skb->sk);
1475         int family = nfmsg->nfgen_family;
1476
1477         rcu_read_lock();
1478         cb->seq = net->nft.base_seq;
1479
1480         list_for_each_entry_rcu(table, &net->nft.tables, list) {
1481                 if (family != NFPROTO_UNSPEC && family != table->family)
1482                         continue;
1483
1484                 list_for_each_entry_rcu(chain, &table->chains, list) {
1485                         if (idx < s_idx)
1486                                 goto cont;
1487                         if (idx > s_idx)
1488                                 memset(&cb->args[1], 0,
1489                                        sizeof(cb->args) - sizeof(cb->args[0]));
1490                         if (!nft_is_active(net, chain))
1491                                 continue;
1492                         if (nf_tables_fill_chain_info(skb, net,
1493                                                       NETLINK_CB(cb->skb).portid,
1494                                                       cb->nlh->nlmsg_seq,
1495                                                       NFT_MSG_NEWCHAIN,
1496                                                       NLM_F_MULTI,
1497                                                       table->family, table,
1498                                                       chain) < 0)
1499                                 goto done;
1500
1501                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
1502 cont:
1503                         idx++;
1504                 }
1505         }
1506 done:
1507         rcu_read_unlock();
1508         cb->args[0] = idx;
1509         return skb->len;
1510 }
1511
1512 /* called with rcu_read_lock held */
1513 static int nf_tables_getchain(struct net *net, struct sock *nlsk,
1514                               struct sk_buff *skb, const struct nlmsghdr *nlh,
1515                               const struct nlattr * const nla[],
1516                               struct netlink_ext_ack *extack)
1517 {
1518         const struct nfgenmsg *nfmsg = nlmsg_data(nlh);
1519         u8 genmask = nft_genmask_cur(net);
1520         const struct nft_chain *chain;
1521         struct nft_table *table;
1522         struct sk_buff *skb2;
1523         int family = nfmsg->nfgen_family;
1524         int err;
1525
1526         if (nlh->nlmsg_flags & NLM_F_DUMP) {
1527                 struct netlink_dump_control c = {
1528                         .dump = nf_tables_dump_chains,
1529                         .module = THIS_MODULE,
1530                 };
1531
1532                 return nft_netlink_dump_start_rcu(nlsk, skb, nlh, &c);
1533         }
1534
1535         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask);
1536         if (IS_ERR(table)) {
1537                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
1538                 return PTR_ERR(table);
1539         }
1540
1541         chain = nft_chain_lookup(net, table, nla[NFTA_CHAIN_NAME], genmask);
1542         if (IS_ERR(chain)) {
1543                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
1544                 return PTR_ERR(chain);
1545         }
1546
1547         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
1548         if (!skb2)
1549                 return -ENOMEM;
1550
1551         err = nf_tables_fill_chain_info(skb2, net, NETLINK_CB(skb).portid,
1552                                         nlh->nlmsg_seq, NFT_MSG_NEWCHAIN, 0,
1553                                         family, table, chain);
1554         if (err < 0)
1555                 goto err;
1556
1557         return nlmsg_unicast(nlsk, skb2, NETLINK_CB(skb).portid);
1558
1559 err:
1560         kfree_skb(skb2);
1561         return err;
1562 }
1563
1564 static const struct nla_policy nft_counter_policy[NFTA_COUNTER_MAX + 1] = {
1565         [NFTA_COUNTER_PACKETS]  = { .type = NLA_U64 },
1566         [NFTA_COUNTER_BYTES]    = { .type = NLA_U64 },
1567 };
1568
1569 static struct nft_stats __percpu *nft_stats_alloc(const struct nlattr *attr)
1570 {
1571         struct nlattr *tb[NFTA_COUNTER_MAX+1];
1572         struct nft_stats __percpu *newstats;
1573         struct nft_stats *stats;
1574         int err;
1575
1576         err = nla_parse_nested_deprecated(tb, NFTA_COUNTER_MAX, attr,
1577                                           nft_counter_policy, NULL);
1578         if (err < 0)
1579                 return ERR_PTR(err);
1580
1581         if (!tb[NFTA_COUNTER_BYTES] || !tb[NFTA_COUNTER_PACKETS])
1582                 return ERR_PTR(-EINVAL);
1583
1584         newstats = netdev_alloc_pcpu_stats(struct nft_stats);
1585         if (newstats == NULL)
1586                 return ERR_PTR(-ENOMEM);
1587
1588         /* Restore old counters on this cpu, no problem. Per-cpu statistics
1589          * are not exposed to userspace.
1590          */
1591         preempt_disable();
1592         stats = this_cpu_ptr(newstats);
1593         stats->bytes = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_BYTES]));
1594         stats->pkts = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_PACKETS]));
1595         preempt_enable();
1596
1597         return newstats;
1598 }
1599
1600 static void nft_chain_stats_replace(struct nft_trans *trans)
1601 {
1602         struct nft_base_chain *chain = nft_base_chain(trans->ctx.chain);
1603
1604         if (!nft_trans_chain_stats(trans))
1605                 return;
1606
1607         nft_trans_chain_stats(trans) =
1608                 rcu_replace_pointer(chain->stats, nft_trans_chain_stats(trans),
1609                                     lockdep_commit_lock_is_held(trans->ctx.net));
1610
1611         if (!nft_trans_chain_stats(trans))
1612                 static_branch_inc(&nft_counters_enabled);
1613 }
1614
1615 static void nf_tables_chain_free_chain_rules(struct nft_chain *chain)
1616 {
1617         struct nft_rule **g0 = rcu_dereference_raw(chain->rules_gen_0);
1618         struct nft_rule **g1 = rcu_dereference_raw(chain->rules_gen_1);
1619
1620         if (g0 != g1)
1621                 kvfree(g1);
1622         kvfree(g0);
1623
1624         /* should be NULL either via abort or via successful commit */
1625         WARN_ON_ONCE(chain->rules_next);
1626         kvfree(chain->rules_next);
1627 }
1628
1629 void nf_tables_chain_destroy(struct nft_ctx *ctx)
1630 {
1631         struct nft_chain *chain = ctx->chain;
1632         struct nft_hook *hook, *next;
1633
1634         if (WARN_ON(chain->use > 0))
1635                 return;
1636
1637         /* no concurrent access possible anymore */
1638         nf_tables_chain_free_chain_rules(chain);
1639
1640         if (nft_is_base_chain(chain)) {
1641                 struct nft_base_chain *basechain = nft_base_chain(chain);
1642
1643                 if (ctx->family == NFPROTO_NETDEV) {
1644                         list_for_each_entry_safe(hook, next,
1645                                                  &basechain->hook_list, list) {
1646                                 list_del_rcu(&hook->list);
1647                                 kfree_rcu(hook, rcu);
1648                         }
1649                 }
1650                 module_put(basechain->type->owner);
1651                 if (rcu_access_pointer(basechain->stats)) {
1652                         static_branch_dec(&nft_counters_enabled);
1653                         free_percpu(rcu_dereference_raw(basechain->stats));
1654                 }
1655                 kfree(chain->name);
1656                 kfree(basechain);
1657         } else {
1658                 kfree(chain->name);
1659                 kfree(chain);
1660         }
1661 }
1662
1663 static struct nft_hook *nft_netdev_hook_alloc(struct net *net,
1664                                               const struct nlattr *attr)
1665 {
1666         struct net_device *dev;
1667         char ifname[IFNAMSIZ];
1668         struct nft_hook *hook;
1669         int err;
1670
1671         hook = kmalloc(sizeof(struct nft_hook), GFP_KERNEL);
1672         if (!hook) {
1673                 err = -ENOMEM;
1674                 goto err_hook_alloc;
1675         }
1676
1677         nla_strlcpy(ifname, attr, IFNAMSIZ);
1678         dev = __dev_get_by_name(net, ifname);
1679         if (!dev) {
1680                 err = -ENOENT;
1681                 goto err_hook_dev;
1682         }
1683         hook->ops.dev = dev;
1684         hook->inactive = false;
1685
1686         return hook;
1687
1688 err_hook_dev:
1689         kfree(hook);
1690 err_hook_alloc:
1691         return ERR_PTR(err);
1692 }
1693
1694 static struct nft_hook *nft_hook_list_find(struct list_head *hook_list,
1695                                            const struct nft_hook *this)
1696 {
1697         struct nft_hook *hook;
1698
1699         list_for_each_entry(hook, hook_list, list) {
1700                 if (this->ops.dev == hook->ops.dev)
1701                         return hook;
1702         }
1703
1704         return NULL;
1705 }
1706
1707 static int nf_tables_parse_netdev_hooks(struct net *net,
1708                                         const struct nlattr *attr,
1709                                         struct list_head *hook_list)
1710 {
1711         struct nft_hook *hook, *next;
1712         const struct nlattr *tmp;
1713         int rem, n = 0, err;
1714
1715         nla_for_each_nested(tmp, attr, rem) {
1716                 if (nla_type(tmp) != NFTA_DEVICE_NAME) {
1717                         err = -EINVAL;
1718                         goto err_hook;
1719                 }
1720
1721                 hook = nft_netdev_hook_alloc(net, tmp);
1722                 if (IS_ERR(hook)) {
1723                         err = PTR_ERR(hook);
1724                         goto err_hook;
1725                 }
1726                 if (nft_hook_list_find(hook_list, hook)) {
1727                         kfree(hook);
1728                         err = -EEXIST;
1729                         goto err_hook;
1730                 }
1731                 list_add_tail(&hook->list, hook_list);
1732                 n++;
1733
1734                 if (n == NFT_NETDEVICE_MAX) {
1735                         err = -EFBIG;
1736                         goto err_hook;
1737                 }
1738         }
1739
1740         return 0;
1741
1742 err_hook:
1743         list_for_each_entry_safe(hook, next, hook_list, list) {
1744                 list_del(&hook->list);
1745                 kfree(hook);
1746         }
1747         return err;
1748 }
1749
1750 struct nft_chain_hook {
1751         u32                             num;
1752         s32                             priority;
1753         const struct nft_chain_type     *type;
1754         struct list_head                list;
1755 };
1756
1757 static int nft_chain_parse_netdev(struct net *net,
1758                                   struct nlattr *tb[],
1759                                   struct list_head *hook_list)
1760 {
1761         struct nft_hook *hook;
1762         int err;
1763
1764         if (tb[NFTA_HOOK_DEV]) {
1765                 hook = nft_netdev_hook_alloc(net, tb[NFTA_HOOK_DEV]);
1766                 if (IS_ERR(hook))
1767                         return PTR_ERR(hook);
1768
1769                 list_add_tail(&hook->list, hook_list);
1770         } else if (tb[NFTA_HOOK_DEVS]) {
1771                 err = nf_tables_parse_netdev_hooks(net, tb[NFTA_HOOK_DEVS],
1772                                                    hook_list);
1773                 if (err < 0)
1774                         return err;
1775
1776                 if (list_empty(hook_list))
1777                         return -EINVAL;
1778         } else {
1779                 return -EINVAL;
1780         }
1781
1782         return 0;
1783 }
1784
1785 static int nft_chain_parse_hook(struct net *net,
1786                                 const struct nlattr * const nla[],
1787                                 struct nft_chain_hook *hook, u8 family,
1788                                 bool autoload)
1789 {
1790         struct nlattr *ha[NFTA_HOOK_MAX + 1];
1791         const struct nft_chain_type *type;
1792         int err;
1793
1794         lockdep_assert_held(&net->nft.commit_mutex);
1795         lockdep_nfnl_nft_mutex_not_held();
1796
1797         err = nla_parse_nested_deprecated(ha, NFTA_HOOK_MAX,
1798                                           nla[NFTA_CHAIN_HOOK],
1799                                           nft_hook_policy, NULL);
1800         if (err < 0)
1801                 return err;
1802
1803         if (ha[NFTA_HOOK_HOOKNUM] == NULL ||
1804             ha[NFTA_HOOK_PRIORITY] == NULL)
1805                 return -EINVAL;
1806
1807         hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
1808         hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
1809
1810         type = __nft_chain_type_get(family, NFT_CHAIN_T_DEFAULT);
1811         if (!type)
1812                 return -EOPNOTSUPP;
1813
1814         if (nla[NFTA_CHAIN_TYPE]) {
1815                 type = nf_tables_chain_type_lookup(net, nla[NFTA_CHAIN_TYPE],
1816                                                    family, autoload);
1817                 if (IS_ERR(type))
1818                         return PTR_ERR(type);
1819         }
1820         if (hook->num > NF_MAX_HOOKS || !(type->hook_mask & (1 << hook->num)))
1821                 return -EOPNOTSUPP;
1822
1823         if (type->type == NFT_CHAIN_T_NAT &&
1824             hook->priority <= NF_IP_PRI_CONNTRACK)
1825                 return -EOPNOTSUPP;
1826
1827         if (!try_module_get(type->owner))
1828                 return -ENOENT;
1829
1830         hook->type = type;
1831
1832         INIT_LIST_HEAD(&hook->list);
1833         if (family == NFPROTO_NETDEV) {
1834                 err = nft_chain_parse_netdev(net, ha, &hook->list);
1835                 if (err < 0) {
1836                         module_put(type->owner);
1837                         return err;
1838                 }
1839         } else if (ha[NFTA_HOOK_DEV] || ha[NFTA_HOOK_DEVS]) {
1840                 module_put(type->owner);
1841                 return -EOPNOTSUPP;
1842         }
1843
1844         return 0;
1845 }
1846
1847 static void nft_chain_release_hook(struct nft_chain_hook *hook)
1848 {
1849         struct nft_hook *h, *next;
1850
1851         list_for_each_entry_safe(h, next, &hook->list, list) {
1852                 list_del(&h->list);
1853                 kfree(h);
1854         }
1855         module_put(hook->type->owner);
1856 }
1857
1858 struct nft_rules_old {
1859         struct rcu_head h;
1860         struct nft_rule **start;
1861 };
1862
1863 static struct nft_rule **nf_tables_chain_alloc_rules(const struct nft_chain *chain,
1864                                                      unsigned int alloc)
1865 {
1866         if (alloc > INT_MAX)
1867                 return NULL;
1868
1869         alloc += 1;     /* NULL, ends rules */
1870         if (sizeof(struct nft_rule *) > INT_MAX / alloc)
1871                 return NULL;
1872
1873         alloc *= sizeof(struct nft_rule *);
1874         alloc += sizeof(struct nft_rules_old);
1875
1876         return kvmalloc(alloc, GFP_KERNEL);
1877 }
1878
1879 static void nft_basechain_hook_init(struct nf_hook_ops *ops, u8 family,
1880                                     const struct nft_chain_hook *hook,
1881                                     struct nft_chain *chain)
1882 {
1883         ops->pf         = family;
1884         ops->hooknum    = hook->num;
1885         ops->priority   = hook->priority;
1886         ops->priv       = chain;
1887         ops->hook       = hook->type->hooks[ops->hooknum];
1888 }
1889
1890 static int nft_basechain_init(struct nft_base_chain *basechain, u8 family,
1891                               struct nft_chain_hook *hook, u32 flags)
1892 {
1893         struct nft_chain *chain;
1894         struct nft_hook *h;
1895
1896         basechain->type = hook->type;
1897         INIT_LIST_HEAD(&basechain->hook_list);
1898         chain = &basechain->chain;
1899
1900         if (family == NFPROTO_NETDEV) {
1901                 list_splice_init(&hook->list, &basechain->hook_list);
1902                 list_for_each_entry(h, &basechain->hook_list, list)
1903                         nft_basechain_hook_init(&h->ops, family, hook, chain);
1904
1905                 basechain->ops.hooknum  = hook->num;
1906                 basechain->ops.priority = hook->priority;
1907         } else {
1908                 nft_basechain_hook_init(&basechain->ops, family, hook, chain);
1909         }
1910
1911         chain->flags |= NFT_CHAIN_BASE | flags;
1912         basechain->policy = NF_ACCEPT;
1913         if (chain->flags & NFT_CHAIN_HW_OFFLOAD &&
1914             nft_chain_offload_priority(basechain) < 0)
1915                 return -EOPNOTSUPP;
1916
1917         flow_block_init(&basechain->flow_block);
1918
1919         return 0;
1920 }
1921
1922 static int nft_chain_add(struct nft_table *table, struct nft_chain *chain)
1923 {
1924         int err;
1925
1926         err = rhltable_insert_key(&table->chains_ht, chain->name,
1927                                   &chain->rhlhead, nft_chain_ht_params);
1928         if (err)
1929                 return err;
1930
1931         list_add_tail_rcu(&chain->list, &table->chains);
1932
1933         return 0;
1934 }
1935
1936 static u64 chain_id;
1937
1938 static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 genmask,
1939                               u8 policy, u32 flags)
1940 {
1941         const struct nlattr * const *nla = ctx->nla;
1942         struct nft_table *table = ctx->table;
1943         struct nft_base_chain *basechain;
1944         struct nft_stats __percpu *stats;
1945         struct net *net = ctx->net;
1946         char name[NFT_NAME_MAXLEN];
1947         struct nft_trans *trans;
1948         struct nft_chain *chain;
1949         struct nft_rule **rules;
1950         int err;
1951
1952         if (table->use == UINT_MAX)
1953                 return -EOVERFLOW;
1954
1955         if (nla[NFTA_CHAIN_HOOK]) {
1956                 struct nft_chain_hook hook;
1957
1958                 if (flags & NFT_CHAIN_BINDING)
1959                         return -EOPNOTSUPP;
1960
1961                 err = nft_chain_parse_hook(net, nla, &hook, family, true);
1962                 if (err < 0)
1963                         return err;
1964
1965                 basechain = kzalloc(sizeof(*basechain), GFP_KERNEL);
1966                 if (basechain == NULL) {
1967                         nft_chain_release_hook(&hook);
1968                         return -ENOMEM;
1969                 }
1970                 chain = &basechain->chain;
1971
1972                 if (nla[NFTA_CHAIN_COUNTERS]) {
1973                         stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
1974                         if (IS_ERR(stats)) {
1975                                 nft_chain_release_hook(&hook);
1976                                 kfree(basechain);
1977                                 return PTR_ERR(stats);
1978                         }
1979                         rcu_assign_pointer(basechain->stats, stats);
1980                         static_branch_inc(&nft_counters_enabled);
1981                 }
1982
1983                 err = nft_basechain_init(basechain, family, &hook, flags);
1984                 if (err < 0) {
1985                         nft_chain_release_hook(&hook);
1986                         kfree(basechain);
1987                         return err;
1988                 }
1989         } else {
1990                 if (flags & NFT_CHAIN_BASE)
1991                         return -EINVAL;
1992                 if (flags & NFT_CHAIN_HW_OFFLOAD)
1993                         return -EOPNOTSUPP;
1994
1995                 chain = kzalloc(sizeof(*chain), GFP_KERNEL);
1996                 if (chain == NULL)
1997                         return -ENOMEM;
1998
1999                 chain->flags = flags;
2000         }
2001         ctx->chain = chain;
2002
2003         INIT_LIST_HEAD(&chain->rules);
2004         chain->handle = nf_tables_alloc_handle(table);
2005         chain->table = table;
2006
2007         if (nla[NFTA_CHAIN_NAME]) {
2008                 chain->name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL);
2009         } else {
2010                 if (!(flags & NFT_CHAIN_BINDING))
2011                         return -EINVAL;
2012
2013                 snprintf(name, sizeof(name), "__chain%llu", ++chain_id);
2014                 chain->name = kstrdup(name, GFP_KERNEL);
2015         }
2016
2017         if (!chain->name) {
2018                 err = -ENOMEM;
2019                 goto err1;
2020         }
2021
2022         rules = nf_tables_chain_alloc_rules(chain, 0);
2023         if (!rules) {
2024                 err = -ENOMEM;
2025                 goto err1;
2026         }
2027
2028         *rules = NULL;
2029         rcu_assign_pointer(chain->rules_gen_0, rules);
2030         rcu_assign_pointer(chain->rules_gen_1, rules);
2031
2032         err = nf_tables_register_hook(net, table, chain);
2033         if (err < 0)
2034                 goto err1;
2035
2036         trans = nft_trans_chain_add(ctx, NFT_MSG_NEWCHAIN);
2037         if (IS_ERR(trans)) {
2038                 err = PTR_ERR(trans);
2039                 goto err2;
2040         }
2041
2042         nft_trans_chain_policy(trans) = NFT_CHAIN_POLICY_UNSET;
2043         if (nft_is_base_chain(chain))
2044                 nft_trans_chain_policy(trans) = policy;
2045
2046         err = nft_chain_add(table, chain);
2047         if (err < 0) {
2048                 nft_trans_destroy(trans);
2049                 goto err2;
2050         }
2051
2052         table->use++;
2053
2054         return 0;
2055 err2:
2056         nf_tables_unregister_hook(net, table, chain);
2057 err1:
2058         nf_tables_chain_destroy(ctx);
2059
2060         return err;
2061 }
2062
2063 static bool nft_hook_list_equal(struct list_head *hook_list1,
2064                                 struct list_head *hook_list2)
2065 {
2066         struct nft_hook *hook;
2067         int n = 0, m = 0;
2068
2069         n = 0;
2070         list_for_each_entry(hook, hook_list2, list) {
2071                 if (!nft_hook_list_find(hook_list1, hook))
2072                         return false;
2073
2074                 n++;
2075         }
2076         list_for_each_entry(hook, hook_list1, list)
2077                 m++;
2078
2079         return n == m;
2080 }
2081
2082 static int nf_tables_updchain(struct nft_ctx *ctx, u8 genmask, u8 policy,
2083                               u32 flags)
2084 {
2085         const struct nlattr * const *nla = ctx->nla;
2086         struct nft_table *table = ctx->table;
2087         struct nft_chain *chain = ctx->chain;
2088         struct nft_base_chain *basechain;
2089         struct nft_stats *stats = NULL;
2090         struct nft_chain_hook hook;
2091         struct nf_hook_ops *ops;
2092         struct nft_trans *trans;
2093         int err;
2094
2095         if (chain->flags ^ flags)
2096                 return -EOPNOTSUPP;
2097
2098         if (nla[NFTA_CHAIN_HOOK]) {
2099                 if (!nft_is_base_chain(chain))
2100                         return -EBUSY;
2101
2102                 err = nft_chain_parse_hook(ctx->net, nla, &hook, ctx->family,
2103                                            false);
2104                 if (err < 0)
2105                         return err;
2106
2107                 basechain = nft_base_chain(chain);
2108                 if (basechain->type != hook.type) {
2109                         nft_chain_release_hook(&hook);
2110                         return -EBUSY;
2111                 }
2112
2113                 if (ctx->family == NFPROTO_NETDEV) {
2114                         if (!nft_hook_list_equal(&basechain->hook_list,
2115                                                  &hook.list)) {
2116                                 nft_chain_release_hook(&hook);
2117                                 return -EBUSY;
2118                         }
2119                 } else {
2120                         ops = &basechain->ops;
2121                         if (ops->hooknum != hook.num ||
2122                             ops->priority != hook.priority) {
2123                                 nft_chain_release_hook(&hook);
2124                                 return -EBUSY;
2125                         }
2126                 }
2127                 nft_chain_release_hook(&hook);
2128         }
2129
2130         if (nla[NFTA_CHAIN_HANDLE] &&
2131             nla[NFTA_CHAIN_NAME]) {
2132                 struct nft_chain *chain2;
2133
2134                 chain2 = nft_chain_lookup(ctx->net, table,
2135                                           nla[NFTA_CHAIN_NAME], genmask);
2136                 if (!IS_ERR(chain2))
2137                         return -EEXIST;
2138         }
2139
2140         if (nla[NFTA_CHAIN_COUNTERS]) {
2141                 if (!nft_is_base_chain(chain))
2142                         return -EOPNOTSUPP;
2143
2144                 stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2145                 if (IS_ERR(stats))
2146                         return PTR_ERR(stats);
2147         }
2148
2149         err = -ENOMEM;
2150         trans = nft_trans_alloc(ctx, NFT_MSG_NEWCHAIN,
2151                                 sizeof(struct nft_trans_chain));
2152         if (trans == NULL)
2153                 goto err;
2154
2155         nft_trans_chain_stats(trans) = stats;
2156         nft_trans_chain_update(trans) = true;
2157
2158         if (nla[NFTA_CHAIN_POLICY])
2159                 nft_trans_chain_policy(trans) = policy;
2160         else
2161                 nft_trans_chain_policy(trans) = -1;
2162
2163         if (nla[NFTA_CHAIN_HANDLE] &&
2164             nla[NFTA_CHAIN_NAME]) {
2165                 struct nft_trans *tmp;
2166                 char *name;
2167
2168                 err = -ENOMEM;
2169                 name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL);
2170                 if (!name)
2171                         goto err;
2172
2173                 err = -EEXIST;
2174                 list_for_each_entry(tmp, &ctx->net->nft.commit_list, list) {
2175                         if (tmp->msg_type == NFT_MSG_NEWCHAIN &&
2176                             tmp->ctx.table == table &&
2177                             nft_trans_chain_update(tmp) &&
2178                             nft_trans_chain_name(tmp) &&
2179                             strcmp(name, nft_trans_chain_name(tmp)) == 0) {
2180                                 kfree(name);
2181                                 goto err;
2182                         }
2183                 }
2184
2185                 nft_trans_chain_name(trans) = name;
2186         }
2187         list_add_tail(&trans->list, &ctx->net->nft.commit_list);
2188
2189         return 0;
2190 err:
2191         free_percpu(stats);
2192         kfree(trans);
2193         return err;
2194 }
2195
2196 static struct nft_chain *nft_chain_lookup_byid(const struct net *net,
2197                                                const struct nlattr *nla)
2198 {
2199         u32 id = ntohl(nla_get_be32(nla));
2200         struct nft_trans *trans;
2201
2202         list_for_each_entry(trans, &net->nft.commit_list, list) {
2203                 struct nft_chain *chain = trans->ctx.chain;
2204
2205                 if (trans->msg_type == NFT_MSG_NEWCHAIN &&
2206                     id == nft_trans_chain_id(trans))
2207                         return chain;
2208         }
2209         return ERR_PTR(-ENOENT);
2210 }
2211
2212 static int nf_tables_newchain(struct net *net, struct sock *nlsk,
2213                               struct sk_buff *skb, const struct nlmsghdr *nlh,
2214                               const struct nlattr * const nla[],
2215                               struct netlink_ext_ack *extack)
2216 {
2217         const struct nfgenmsg *nfmsg = nlmsg_data(nlh);
2218         u8 genmask = nft_genmask_next(net);
2219         int family = nfmsg->nfgen_family;
2220         struct nft_chain *chain = NULL;
2221         const struct nlattr *attr;
2222         struct nft_table *table;
2223         u8 policy = NF_ACCEPT;
2224         struct nft_ctx ctx;
2225         u64 handle = 0;
2226         u32 flags = 0;
2227
2228         lockdep_assert_held(&net->nft.commit_mutex);
2229
2230         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask);
2231         if (IS_ERR(table)) {
2232                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2233                 return PTR_ERR(table);
2234         }
2235
2236         chain = NULL;
2237         attr = nla[NFTA_CHAIN_NAME];
2238
2239         if (nla[NFTA_CHAIN_HANDLE]) {
2240                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_CHAIN_HANDLE]));
2241                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2242                 if (IS_ERR(chain)) {
2243                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_HANDLE]);
2244                         return PTR_ERR(chain);
2245                 }
2246                 attr = nla[NFTA_CHAIN_HANDLE];
2247         } else if (nla[NFTA_CHAIN_NAME]) {
2248                 chain = nft_chain_lookup(net, table, attr, genmask);
2249                 if (IS_ERR(chain)) {
2250                         if (PTR_ERR(chain) != -ENOENT) {
2251                                 NL_SET_BAD_ATTR(extack, attr);
2252                                 return PTR_ERR(chain);
2253                         }
2254                         chain = NULL;
2255                 }
2256         } else if (!nla[NFTA_CHAIN_ID]) {
2257                 return -EINVAL;
2258         }
2259
2260         if (nla[NFTA_CHAIN_POLICY]) {
2261                 if (chain != NULL &&
2262                     !nft_is_base_chain(chain)) {
2263                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2264                         return -EOPNOTSUPP;
2265                 }
2266
2267                 if (chain == NULL &&
2268                     nla[NFTA_CHAIN_HOOK] == NULL) {
2269                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2270                         return -EOPNOTSUPP;
2271                 }
2272
2273                 policy = ntohl(nla_get_be32(nla[NFTA_CHAIN_POLICY]));
2274                 switch (policy) {
2275                 case NF_DROP:
2276                 case NF_ACCEPT:
2277                         break;
2278                 default:
2279                         return -EINVAL;
2280                 }
2281         }
2282
2283         if (nla[NFTA_CHAIN_FLAGS])
2284                 flags = ntohl(nla_get_be32(nla[NFTA_CHAIN_FLAGS]));
2285         else if (chain)
2286                 flags = chain->flags;
2287
2288         if (flags & ~NFT_CHAIN_FLAGS)
2289                 return -EOPNOTSUPP;
2290
2291         nft_ctx_init(&ctx, net, skb, nlh, family, table, chain, nla);
2292
2293         if (chain != NULL) {
2294                 if (nlh->nlmsg_flags & NLM_F_EXCL) {
2295                         NL_SET_BAD_ATTR(extack, attr);
2296                         return -EEXIST;
2297                 }
2298                 if (nlh->nlmsg_flags & NLM_F_REPLACE)
2299                         return -EOPNOTSUPP;
2300
2301                 flags |= chain->flags & NFT_CHAIN_BASE;
2302                 return nf_tables_updchain(&ctx, genmask, policy, flags);
2303         }
2304
2305         return nf_tables_addchain(&ctx, family, genmask, policy, flags);
2306 }
2307
2308 static int nf_tables_delchain(struct net *net, struct sock *nlsk,
2309                               struct sk_buff *skb, const struct nlmsghdr *nlh,
2310                               const struct nlattr * const nla[],
2311                               struct netlink_ext_ack *extack)
2312 {
2313         const struct nfgenmsg *nfmsg = nlmsg_data(nlh);
2314         u8 genmask = nft_genmask_next(net);
2315         int family = nfmsg->nfgen_family;
2316         const struct nlattr *attr;
2317         struct nft_table *table;
2318         struct nft_chain *chain;
2319         struct nft_rule *rule;
2320         struct nft_ctx ctx;
2321         u64 handle;
2322         u32 use;
2323         int err;
2324
2325         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask);
2326         if (IS_ERR(table)) {
2327                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2328                 return PTR_ERR(table);
2329         }
2330
2331         if (nla[NFTA_CHAIN_HANDLE]) {
2332                 attr = nla[NFTA_CHAIN_HANDLE];
2333                 handle = be64_to_cpu(nla_get_be64(attr));
2334                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2335         } else {
2336                 attr = nla[NFTA_CHAIN_NAME];
2337                 chain = nft_chain_lookup(net, table, attr, genmask);
2338         }
2339         if (IS_ERR(chain)) {
2340                 NL_SET_BAD_ATTR(extack, attr);
2341                 return PTR_ERR(chain);
2342         }
2343
2344         if (nlh->nlmsg_flags & NLM_F_NONREC &&
2345             chain->use > 0)
2346                 return -EBUSY;
2347
2348         nft_ctx_init(&ctx, net, skb, nlh, family, table, chain, nla);
2349
2350         use = chain->use;
2351         list_for_each_entry(rule, &chain->rules, list) {
2352                 if (!nft_is_active_next(net, rule))
2353                         continue;
2354                 use--;
2355
2356                 err = nft_delrule(&ctx, rule);
2357                 if (err < 0)
2358                         return err;
2359         }
2360
2361         /* There are rules and elements that are still holding references to us,
2362          * we cannot do a recursive removal in this case.
2363          */
2364         if (use > 0) {
2365                 NL_SET_BAD_ATTR(extack, attr);
2366                 return -EBUSY;
2367         }
2368
2369         return nft_delchain(&ctx);
2370 }
2371
2372 /*
2373  * Expressions
2374  */
2375
2376 /**
2377  *      nft_register_expr - register nf_tables expr type
2378  *      @type: expr type
2379  *
2380  *      Registers the expr type for use with nf_tables. Returns zero on
2381  *      success or a negative errno code otherwise.
2382  */
2383 int nft_register_expr(struct nft_expr_type *type)
2384 {
2385         nfnl_lock(NFNL_SUBSYS_NFTABLES);
2386         if (type->family == NFPROTO_UNSPEC)
2387                 list_add_tail_rcu(&type->list, &nf_tables_expressions);
2388         else
2389                 list_add_rcu(&type->list, &nf_tables_expressions);
2390         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
2391         return 0;
2392 }
2393 EXPORT_SYMBOL_GPL(nft_register_expr);
2394
2395 /**
2396  *      nft_unregister_expr - unregister nf_tables expr type
2397  *      @type: expr type
2398  *
2399  *      Unregisters the expr typefor use with nf_tables.
2400  */
2401 void nft_unregister_expr(struct nft_expr_type *type)
2402 {
2403         nfnl_lock(NFNL_SUBSYS_NFTABLES);
2404         list_del_rcu(&type->list);
2405         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
2406 }
2407 EXPORT_SYMBOL_GPL(nft_unregister_expr);
2408
2409 static const struct nft_expr_type *__nft_expr_type_get(u8 family,
2410                                                        struct nlattr *nla)
2411 {
2412         const struct nft_expr_type *type, *candidate = NULL;
2413
2414         list_for_each_entry(type, &nf_tables_expressions, list) {
2415                 if (!nla_strcmp(nla, type->name)) {
2416                         if (!type->family && !candidate)
2417                                 candidate = type;
2418                         else if (type->family == family)
2419                                 candidate = type;
2420                 }
2421         }
2422         return candidate;
2423 }
2424
2425 #ifdef CONFIG_MODULES
2426 static int nft_expr_type_request_module(struct net *net, u8 family,
2427                                         struct nlattr *nla)
2428 {
2429         if (nft_request_module(net, "nft-expr-%u-%.*s", family,
2430                                nla_len(nla), (char *)nla_data(nla)) == -EAGAIN)
2431                 return -EAGAIN;
2432
2433         return 0;
2434 }
2435 #endif
2436
2437 static const struct nft_expr_type *nft_expr_type_get(struct net *net,
2438                                                      u8 family,
2439                                                      struct nlattr *nla)
2440 {
2441         const struct nft_expr_type *type;
2442
2443         if (nla == NULL)
2444                 return ERR_PTR(-EINVAL);
2445
2446         type = __nft_expr_type_get(family, nla);
2447         if (type != NULL && try_module_get(type->owner))
2448                 return type;
2449
2450         lockdep_nfnl_nft_mutex_not_held();
2451 #ifdef CONFIG_MODULES
2452         if (type == NULL) {
2453                 if (nft_expr_type_request_module(net, family, nla) == -EAGAIN)
2454                         return ERR_PTR(-EAGAIN);
2455
2456                 if (nft_request_module(net, "nft-expr-%.*s",
2457                                        nla_len(nla),
2458                                        (char *)nla_data(nla)) == -EAGAIN)
2459                         return ERR_PTR(-EAGAIN);
2460         }
2461 #endif
2462         return ERR_PTR(-ENOENT);
2463 }
2464
2465 static const struct nla_policy nft_expr_policy[NFTA_EXPR_MAX + 1] = {
2466         [NFTA_EXPR_NAME]        = { .type = NLA_STRING,
2467                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
2468         [NFTA_EXPR_DATA]        = { .type = NLA_NESTED },
2469 };
2470
2471 static int nf_tables_fill_expr_info(struct sk_buff *skb,
2472                                     const struct nft_expr *expr)
2473 {
2474         if (nla_put_string(skb, NFTA_EXPR_NAME, expr->ops->type->name))
2475                 goto nla_put_failure;
2476
2477         if (expr->ops->dump) {
2478                 struct nlattr *data = nla_nest_start_noflag(skb,
2479                                                             NFTA_EXPR_DATA);
2480                 if (data == NULL)
2481                         goto nla_put_failure;
2482                 if (expr->ops->dump(skb, expr) < 0)
2483                         goto nla_put_failure;
2484                 nla_nest_end(skb, data);
2485         }
2486
2487         return skb->len;
2488
2489 nla_put_failure:
2490         return -1;
2491 };
2492
2493 int nft_expr_dump(struct sk_buff *skb, unsigned int attr,
2494                   const struct nft_expr *expr)
2495 {
2496         struct nlattr *nest;
2497
2498         nest = nla_nest_start_noflag(skb, attr);
2499         if (!nest)
2500                 goto nla_put_failure;
2501         if (nf_tables_fill_expr_info(skb, expr) < 0)
2502                 goto nla_put_failure;
2503         nla_nest_end(skb, nest);
2504         return 0;
2505
2506 nla_put_failure:
2507         return -1;
2508 }
2509
2510 struct nft_expr_info {
2511         const struct nft_expr_ops       *ops;
2512         struct nlattr                   *tb[NFT_EXPR_MAXATTR + 1];
2513 };
2514
2515 static int nf_tables_expr_parse(const struct nft_ctx *ctx,
2516                                 const struct nlattr *nla,
2517                                 struct nft_expr_info *info)
2518 {
2519         const struct nft_expr_type *type;
2520         const struct nft_expr_ops *ops;
2521         struct nlattr *tb[NFTA_EXPR_MAX + 1];
2522         int err;
2523
2524         err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
2525                                           nft_expr_policy, NULL);
2526         if (err < 0)
2527                 return err;
2528
2529         type = nft_expr_type_get(ctx->net, ctx->family, tb[NFTA_EXPR_NAME]);
2530         if (IS_ERR(type))
2531                 return PTR_ERR(type);
2532
2533         if (tb[NFTA_EXPR_DATA]) {
2534                 err = nla_parse_nested_deprecated(info->tb, type->maxattr,
2535                                                   tb[NFTA_EXPR_DATA],
2536                                                   type->policy, NULL);
2537                 if (err < 0)
2538                         goto err1;
2539         } else
2540                 memset(info->tb, 0, sizeof(info->tb[0]) * (type->maxattr + 1));
2541
2542         if (type->select_ops != NULL) {
2543                 ops = type->select_ops(ctx,
2544                                        (const struct nlattr * const *)info->tb);
2545                 if (IS_ERR(ops)) {
2546                         err = PTR_ERR(ops);
2547 #ifdef CONFIG_MODULES
2548                         if (err == -EAGAIN)
2549                                 if (nft_expr_type_request_module(ctx->net,
2550                                                                  ctx->family,
2551                                                                  tb[NFTA_EXPR_NAME]) != -EAGAIN)
2552                                         err = -ENOENT;
2553 #endif
2554                         goto err1;
2555                 }
2556         } else
2557                 ops = type->ops;
2558
2559         info->ops = ops;
2560         return 0;
2561
2562 err1:
2563         module_put(type->owner);
2564         return err;
2565 }
2566
2567 static int nf_tables_newexpr(const struct nft_ctx *ctx,
2568                              const struct nft_expr_info *info,
2569                              struct nft_expr *expr)
2570 {
2571         const struct nft_expr_ops *ops = info->ops;
2572         int err;
2573
2574         expr->ops = ops;
2575         if (ops->init) {
2576                 err = ops->init(ctx, expr, (const struct nlattr **)info->tb);
2577                 if (err < 0)
2578                         goto err1;
2579         }
2580
2581         return 0;
2582 err1:
2583         expr->ops = NULL;
2584         return err;
2585 }
2586
2587 static void nf_tables_expr_destroy(const struct nft_ctx *ctx,
2588                                    struct nft_expr *expr)
2589 {
2590         const struct nft_expr_type *type = expr->ops->type;
2591
2592         if (expr->ops->destroy)
2593                 expr->ops->destroy(ctx, expr);
2594         module_put(type->owner);
2595 }
2596
2597 static struct nft_expr *nft_expr_init(const struct nft_ctx *ctx,
2598                                       const struct nlattr *nla)
2599 {
2600         struct nft_expr_info info;
2601         struct nft_expr *expr;
2602         struct module *owner;
2603         int err;
2604
2605         err = nf_tables_expr_parse(ctx, nla, &info);
2606         if (err < 0)
2607                 goto err1;
2608
2609         err = -ENOMEM;
2610         expr = kzalloc(info.ops->size, GFP_KERNEL);
2611         if (expr == NULL)
2612                 goto err2;
2613
2614         err = nf_tables_newexpr(ctx, &info, expr);
2615         if (err < 0)
2616                 goto err3;
2617
2618         return expr;
2619 err3:
2620         kfree(expr);
2621 err2:
2622         owner = info.ops->type->owner;
2623         if (info.ops->type->release_ops)
2624                 info.ops->type->release_ops(info.ops);
2625
2626         module_put(owner);
2627 err1:
2628         return ERR_PTR(err);
2629 }
2630
2631 int nft_expr_clone(struct nft_expr *dst, struct nft_expr *src)
2632 {
2633         int err;
2634
2635         if (src->ops->clone) {
2636                 dst->ops = src->ops;
2637                 err = src->ops->clone(dst, src);
2638                 if (err < 0)
2639                         return err;
2640         } else {
2641                 memcpy(dst, src, src->ops->size);
2642         }
2643
2644         __module_get(src->ops->type->owner);
2645
2646         return 0;
2647 }
2648
2649 void nft_expr_destroy(const struct nft_ctx *ctx, struct nft_expr *expr)
2650 {
2651         nf_tables_expr_destroy(ctx, expr);
2652         kfree(expr);
2653 }
2654
2655 /*
2656  * Rules
2657  */
2658
2659 static struct nft_rule *__nft_rule_lookup(const struct nft_chain *chain,
2660                                           u64 handle)
2661 {
2662         struct nft_rule *rule;
2663
2664         // FIXME: this sucks
2665         list_for_each_entry_rcu(rule, &chain->rules, list) {
2666                 if (handle == rule->handle)
2667                         return rule;
2668         }
2669
2670         return ERR_PTR(-ENOENT);
2671 }
2672
2673 static struct nft_rule *nft_rule_lookup(const struct nft_chain *chain,
2674                                         const struct nlattr *nla)
2675 {
2676         if (nla == NULL)
2677                 return ERR_PTR(-EINVAL);
2678
2679         return __nft_rule_lookup(chain, be64_to_cpu(nla_get_be64(nla)));
2680 }
2681
2682 static const struct nla_policy nft_rule_policy[NFTA_RULE_MAX + 1] = {
2683         [NFTA_RULE_TABLE]       = { .type = NLA_STRING,
2684                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
2685         [NFTA_RULE_CHAIN]       = { .type = NLA_STRING,
2686                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
2687         [NFTA_RULE_HANDLE]      = { .type = NLA_U64 },
2688         [NFTA_RULE_EXPRESSIONS] = { .type = NLA_NESTED },
2689         [NFTA_RULE_COMPAT]      = { .type = NLA_NESTED },
2690         [NFTA_RULE_POSITION]    = { .type = NLA_U64 },
2691         [NFTA_RULE_USERDATA]    = { .type = NLA_BINARY,
2692                                     .len = NFT_USERDATA_MAXLEN },
2693         [NFTA_RULE_ID]          = { .type = NLA_U32 },
2694         [NFTA_RULE_POSITION_ID] = { .type = NLA_U32 },
2695         [NFTA_RULE_CHAIN_ID]    = { .type = NLA_U32 },
2696 };
2697
2698 static int nf_tables_fill_rule_info(struct sk_buff *skb, struct net *net,
2699                                     u32 portid, u32 seq, int event,
2700                                     u32 flags, int family,
2701                                     const struct nft_table *table,
2702                                     const struct nft_chain *chain,
2703                                     const struct nft_rule *rule,
2704                                     const struct nft_rule *prule)
2705 {
2706         struct nlmsghdr *nlh;
2707         struct nfgenmsg *nfmsg;
2708         const struct nft_expr *expr, *next;
2709         struct nlattr *list;
2710         u16 type = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
2711
2712         nlh = nlmsg_put(skb, portid, seq, type, sizeof(struct nfgenmsg), flags);
2713         if (nlh == NULL)
2714                 goto nla_put_failure;
2715
2716         nfmsg = nlmsg_data(nlh);
2717         nfmsg->nfgen_family     = family;
2718         nfmsg->version          = NFNETLINK_V0;
2719         nfmsg->res_id           = htons(net->nft.base_seq & 0xffff);
2720
2721         if (nla_put_string(skb, NFTA_RULE_TABLE, table->name))
2722                 goto nla_put_failure;
2723         if (nla_put_string(skb, NFTA_RULE_CHAIN, chain->name))
2724                 goto nla_put_failure;
2725         if (nla_put_be64(skb, NFTA_RULE_HANDLE, cpu_to_be64(rule->handle),
2726                          NFTA_RULE_PAD))
2727                 goto nla_put_failure;
2728
2729         if (event != NFT_MSG_DELRULE && prule) {
2730                 if (nla_put_be64(skb, NFTA_RULE_POSITION,
2731                                  cpu_to_be64(prule->handle),
2732                                  NFTA_RULE_PAD))
2733                         goto nla_put_failure;
2734         }
2735
2736         list = nla_nest_start_noflag(skb, NFTA_RULE_EXPRESSIONS);
2737         if (list == NULL)
2738                 goto nla_put_failure;
2739         nft_rule_for_each_expr(expr, next, rule) {
2740                 if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr) < 0)
2741                         goto nla_put_failure;
2742         }
2743         nla_nest_end(skb, list);
2744
2745         if (rule->udata) {
2746                 struct nft_userdata *udata = nft_userdata(rule);
2747                 if (nla_put(skb, NFTA_RULE_USERDATA, udata->len + 1,
2748                             udata->data) < 0)
2749                         goto nla_put_failure;
2750         }
2751
2752         nlmsg_end(skb, nlh);
2753         return 0;
2754
2755 nla_put_failure:
2756         nlmsg_trim(skb, nlh);
2757         return -1;
2758 }
2759
2760 static void nf_tables_rule_notify(const struct nft_ctx *ctx,
2761                                   const struct nft_rule *rule, int event)
2762 {
2763         struct sk_buff *skb;
2764         int err;
2765
2766         if (!ctx->report &&
2767             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
2768                 return;
2769
2770         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
2771         if (skb == NULL)
2772                 goto err;
2773
2774         err = nf_tables_fill_rule_info(skb, ctx->net, ctx->portid, ctx->seq,
2775                                        event, 0, ctx->family, ctx->table,
2776                                        ctx->chain, rule, NULL);
2777         if (err < 0) {
2778                 kfree_skb(skb);
2779                 goto err;
2780         }
2781
2782         nfnetlink_send(skb, ctx->net, ctx->portid, NFNLGRP_NFTABLES,
2783                        ctx->report, GFP_KERNEL);
2784         return;
2785 err:
2786         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
2787 }
2788
2789 struct nft_rule_dump_ctx {
2790         char *table;
2791         char *chain;
2792 };
2793
2794 static int __nf_tables_dump_rules(struct sk_buff *skb,
2795                                   unsigned int *idx,
2796                                   struct netlink_callback *cb,
2797                                   const struct nft_table *table,
2798                                   const struct nft_chain *chain)
2799 {
2800         struct net *net = sock_net(skb->sk);
2801         const struct nft_rule *rule, *prule;
2802         unsigned int s_idx = cb->args[0];
2803
2804         prule = NULL;
2805         list_for_each_entry_rcu(rule, &chain->rules, list) {
2806                 if (!nft_is_active(net, rule))
2807                         goto cont_skip;
2808                 if (*idx < s_idx)
2809                         goto cont;
2810                 if (*idx > s_idx) {
2811                         memset(&cb->args[1], 0,
2812                                         sizeof(cb->args) - sizeof(cb->args[0]));
2813                 }
2814                 if (nf_tables_fill_rule_info(skb, net, NETLINK_CB(cb->skb).portid,
2815                                         cb->nlh->nlmsg_seq,
2816                                         NFT_MSG_NEWRULE,
2817                                         NLM_F_MULTI | NLM_F_APPEND,
2818                                         table->family,
2819                                         table, chain, rule, prule) < 0)
2820                         return 1;
2821
2822                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
2823 cont:
2824                 prule = rule;
2825 cont_skip:
2826                 (*idx)++;
2827         }
2828         return 0;
2829 }
2830
2831 static int nf_tables_dump_rules(struct sk_buff *skb,
2832                                 struct netlink_callback *cb)
2833 {
2834         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
2835         const struct nft_rule_dump_ctx *ctx = cb->data;
2836         struct nft_table *table;
2837         const struct nft_chain *chain;
2838         unsigned int idx = 0;
2839         struct net *net = sock_net(skb->sk);
2840         int family = nfmsg->nfgen_family;
2841
2842         rcu_read_lock();
2843         cb->seq = net->nft.base_seq;
2844
2845         list_for_each_entry_rcu(table, &net->nft.tables, list) {
2846                 if (family != NFPROTO_UNSPEC && family != table->family)
2847                         continue;
2848
2849                 if (ctx && ctx->table && strcmp(ctx->table, table->name) != 0)
2850                         continue;
2851
2852                 if (ctx && ctx->table && ctx->chain) {
2853                         struct rhlist_head *list, *tmp;
2854
2855                         list = rhltable_lookup(&table->chains_ht, ctx->chain,
2856                                                nft_chain_ht_params);
2857                         if (!list)
2858                                 goto done;
2859
2860                         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
2861                                 if (!nft_is_active(net, chain))
2862                                         continue;
2863                                 __nf_tables_dump_rules(skb, &idx,
2864                                                        cb, table, chain);
2865                                 break;
2866                         }
2867                         goto done;
2868                 }
2869
2870                 list_for_each_entry_rcu(chain, &table->chains, list) {
2871                         if (__nf_tables_dump_rules(skb, &idx, cb, table, chain))
2872                                 goto done;
2873                 }
2874
2875                 if (ctx && ctx->table)
2876                         break;
2877         }
2878 done:
2879         rcu_read_unlock();
2880
2881         cb->args[0] = idx;
2882         return skb->len;
2883 }
2884
2885 static int nf_tables_dump_rules_start(struct netlink_callback *cb)
2886 {
2887         const struct nlattr * const *nla = cb->data;
2888         struct nft_rule_dump_ctx *ctx = NULL;
2889
2890         if (nla[NFTA_RULE_TABLE] || nla[NFTA_RULE_CHAIN]) {
2891                 ctx = kzalloc(sizeof(*ctx), GFP_ATOMIC);
2892                 if (!ctx)
2893                         return -ENOMEM;
2894
2895                 if (nla[NFTA_RULE_TABLE]) {
2896                         ctx->table = nla_strdup(nla[NFTA_RULE_TABLE],
2897                                                         GFP_ATOMIC);
2898                         if (!ctx->table) {
2899                                 kfree(ctx);
2900                                 return -ENOMEM;
2901                         }
2902                 }
2903                 if (nla[NFTA_RULE_CHAIN]) {
2904                         ctx->chain = nla_strdup(nla[NFTA_RULE_CHAIN],
2905                                                 GFP_ATOMIC);
2906                         if (!ctx->chain) {
2907                                 kfree(ctx->table);
2908                                 kfree(ctx);
2909                                 return -ENOMEM;
2910                         }
2911                 }
2912         }
2913
2914         cb->data = ctx;
2915         return 0;
2916 }
2917
2918 static int nf_tables_dump_rules_done(struct netlink_callback *cb)
2919 {
2920         struct nft_rule_dump_ctx *ctx = cb->data;
2921
2922         if (ctx) {
2923                 kfree(ctx->table);
2924                 kfree(ctx->chain);
2925                 kfree(ctx);
2926         }
2927         return 0;
2928 }
2929
2930 /* called with rcu_read_lock held */
2931 static int nf_tables_getrule(struct net *net, struct sock *nlsk,
2932                              struct sk_buff *skb, const struct nlmsghdr *nlh,
2933                              const struct nlattr * const nla[],
2934                              struct netlink_ext_ack *extack)
2935 {
2936         const struct nfgenmsg *nfmsg = nlmsg_data(nlh);
2937         u8 genmask = nft_genmask_cur(net);
2938         const struct nft_chain *chain;
2939         const struct nft_rule *rule;
2940         struct nft_table *table;
2941         struct sk_buff *skb2;
2942         int family = nfmsg->nfgen_family;
2943         int err;
2944
2945         if (nlh->nlmsg_flags & NLM_F_DUMP) {
2946                 struct netlink_dump_control c = {
2947                         .start= nf_tables_dump_rules_start,
2948                         .dump = nf_tables_dump_rules,
2949                         .done = nf_tables_dump_rules_done,
2950                         .module = THIS_MODULE,
2951                         .data = (void *)nla,
2952                 };
2953
2954                 return nft_netlink_dump_start_rcu(nlsk, skb, nlh, &c);
2955         }
2956
2957         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask);
2958         if (IS_ERR(table)) {
2959                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
2960                 return PTR_ERR(table);
2961         }
2962
2963         chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN], genmask);
2964         if (IS_ERR(chain)) {
2965                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
2966                 return PTR_ERR(chain);
2967         }
2968
2969         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
2970         if (IS_ERR(rule)) {
2971                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
2972                 return PTR_ERR(rule);
2973         }
2974
2975         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
2976         if (!skb2)
2977                 return -ENOMEM;
2978
2979         err = nf_tables_fill_rule_info(skb2, net, NETLINK_CB(skb).portid,
2980                                        nlh->nlmsg_seq, NFT_MSG_NEWRULE, 0,
2981                                        family, table, chain, rule, NULL);
2982         if (err < 0)
2983                 goto err;
2984
2985         return nlmsg_unicast(nlsk, skb2, NETLINK_CB(skb).portid);
2986
2987 err:
2988         kfree_skb(skb2);
2989         return err;
2990 }
2991
2992 static void nf_tables_rule_destroy(const struct nft_ctx *ctx,
2993                                    struct nft_rule *rule)
2994 {
2995         struct nft_expr *expr, *next;
2996
2997         /*
2998          * Careful: some expressions might not be initialized in case this
2999          * is called on error from nf_tables_newrule().
3000          */
3001         expr = nft_expr_first(rule);
3002         while (expr != nft_expr_last(rule) && expr->ops) {
3003                 next = nft_expr_next(expr);
3004                 nf_tables_expr_destroy(ctx, expr);
3005                 expr = next;
3006         }
3007         kfree(rule);
3008 }
3009
3010 void nf_tables_rule_release(const struct nft_ctx *ctx, struct nft_rule *rule)
3011 {
3012         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_RELEASE);
3013         nf_tables_rule_destroy(ctx, rule);
3014 }
3015
3016 int nft_chain_validate(const struct nft_ctx *ctx, const struct nft_chain *chain)
3017 {
3018         struct nft_expr *expr, *last;
3019         const struct nft_data *data;
3020         struct nft_rule *rule;
3021         int err;
3022
3023         if (ctx->level == NFT_JUMP_STACK_SIZE)
3024                 return -EMLINK;
3025
3026         list_for_each_entry(rule, &chain->rules, list) {
3027                 if (!nft_is_active_next(ctx->net, rule))
3028                         continue;
3029
3030                 nft_rule_for_each_expr(expr, last, rule) {
3031                         if (!expr->ops->validate)
3032                                 continue;
3033
3034                         err = expr->ops->validate(ctx, expr, &data);
3035                         if (err < 0)
3036                                 return err;
3037                 }
3038         }
3039
3040         return 0;
3041 }
3042 EXPORT_SYMBOL_GPL(nft_chain_validate);
3043
3044 static int nft_table_validate(struct net *net, const struct nft_table *table)
3045 {
3046         struct nft_chain *chain;
3047         struct nft_ctx ctx = {
3048                 .net    = net,
3049                 .family = table->family,
3050         };
3051         int err;
3052
3053         list_for_each_entry(chain, &table->chains, list) {
3054                 if (!nft_is_base_chain(chain))
3055                         continue;
3056
3057                 ctx.chain = chain;
3058                 err = nft_chain_validate(&ctx, chain);
3059                 if (err < 0)
3060                         return err;
3061         }
3062
3063         return 0;
3064 }
3065
3066 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
3067                                              const struct nlattr *nla);
3068
3069 #define NFT_RULE_MAXEXPRS       128
3070
3071 static int nf_tables_newrule(struct net *net, struct sock *nlsk,
3072                              struct sk_buff *skb, const struct nlmsghdr *nlh,
3073                              const struct nlattr * const nla[],
3074                              struct netlink_ext_ack *extack)
3075 {
3076         const struct nfgenmsg *nfmsg = nlmsg_data(nlh);
3077         u8 genmask = nft_genmask_next(net);
3078         struct nft_expr_info *info = NULL;
3079         int family = nfmsg->nfgen_family;
3080         struct nft_flow_rule *flow;
3081         struct nft_table *table;
3082         struct nft_chain *chain;
3083         struct nft_rule *rule, *old_rule = NULL;
3084         struct nft_userdata *udata;
3085         struct nft_trans *trans = NULL;
3086         struct nft_expr *expr;
3087         struct nft_ctx ctx;
3088         struct nlattr *tmp;
3089         unsigned int size, i, n, ulen = 0, usize = 0;
3090         int err, rem;
3091         u64 handle, pos_handle;
3092
3093         lockdep_assert_held(&net->nft.commit_mutex);
3094
3095         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask);
3096         if (IS_ERR(table)) {
3097                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3098                 return PTR_ERR(table);
3099         }
3100
3101         if (nla[NFTA_RULE_CHAIN]) {
3102                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
3103                                          genmask);
3104                 if (IS_ERR(chain)) {
3105                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3106                         return PTR_ERR(chain);
3107                 }
3108                 if (nft_chain_is_bound(chain))
3109                         return -EOPNOTSUPP;
3110
3111         } else if (nla[NFTA_RULE_CHAIN_ID]) {
3112                 chain = nft_chain_lookup_byid(net, nla[NFTA_RULE_CHAIN_ID]);
3113                 if (IS_ERR(chain)) {
3114                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN_ID]);
3115                         return PTR_ERR(chain);
3116                 }
3117         } else {
3118                 return -EINVAL;
3119         }
3120
3121         if (nla[NFTA_RULE_HANDLE]) {
3122                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_HANDLE]));
3123                 rule = __nft_rule_lookup(chain, handle);
3124                 if (IS_ERR(rule)) {
3125                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3126                         return PTR_ERR(rule);
3127                 }
3128
3129                 if (nlh->nlmsg_flags & NLM_F_EXCL) {
3130                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3131                         return -EEXIST;
3132                 }
3133                 if (nlh->nlmsg_flags & NLM_F_REPLACE)
3134                         old_rule = rule;
3135                 else
3136                         return -EOPNOTSUPP;
3137         } else {
3138                 if (!(nlh->nlmsg_flags & NLM_F_CREATE) ||
3139                     nlh->nlmsg_flags & NLM_F_REPLACE)
3140                         return -EINVAL;
3141                 handle = nf_tables_alloc_handle(table);
3142
3143                 if (chain->use == UINT_MAX)
3144                         return -EOVERFLOW;
3145
3146                 if (nla[NFTA_RULE_POSITION]) {
3147                         pos_handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_POSITION]));
3148                         old_rule = __nft_rule_lookup(chain, pos_handle);
3149                         if (IS_ERR(old_rule)) {
3150                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION]);
3151                                 return PTR_ERR(old_rule);
3152                         }
3153                 } else if (nla[NFTA_RULE_POSITION_ID]) {
3154                         old_rule = nft_rule_lookup_byid(net, nla[NFTA_RULE_POSITION_ID]);
3155                         if (IS_ERR(old_rule)) {
3156                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION_ID]);
3157                                 return PTR_ERR(old_rule);
3158                         }
3159                 }
3160         }
3161
3162         nft_ctx_init(&ctx, net, skb, nlh, family, table, chain, nla);
3163
3164         n = 0;
3165         size = 0;
3166         if (nla[NFTA_RULE_EXPRESSIONS]) {
3167                 info = kvmalloc_array(NFT_RULE_MAXEXPRS,
3168                                       sizeof(struct nft_expr_info),
3169                                       GFP_KERNEL);
3170                 if (!info)
3171                         return -ENOMEM;
3172
3173                 nla_for_each_nested(tmp, nla[NFTA_RULE_EXPRESSIONS], rem) {
3174                         err = -EINVAL;
3175                         if (nla_type(tmp) != NFTA_LIST_ELEM)
3176                                 goto err1;
3177                         if (n == NFT_RULE_MAXEXPRS)
3178                                 goto err1;
3179                         err = nf_tables_expr_parse(&ctx, tmp, &info[n]);
3180                         if (err < 0)
3181                                 goto err1;
3182                         size += info[n].ops->size;
3183                         n++;
3184                 }
3185         }
3186         /* Check for overflow of dlen field */
3187         err = -EFBIG;
3188         if (size >= 1 << 12)
3189                 goto err1;
3190
3191         if (nla[NFTA_RULE_USERDATA]) {
3192                 ulen = nla_len(nla[NFTA_RULE_USERDATA]);
3193                 if (ulen > 0)
3194                         usize = sizeof(struct nft_userdata) + ulen;
3195         }
3196
3197         err = -ENOMEM;
3198         rule = kzalloc(sizeof(*rule) + size + usize, GFP_KERNEL);
3199         if (rule == NULL)
3200                 goto err1;
3201
3202         nft_activate_next(net, rule);
3203
3204         rule->handle = handle;
3205         rule->dlen   = size;
3206         rule->udata  = ulen ? 1 : 0;
3207
3208         if (ulen) {
3209                 udata = nft_userdata(rule);
3210                 udata->len = ulen - 1;
3211                 nla_memcpy(udata->data, nla[NFTA_RULE_USERDATA], ulen);
3212         }
3213
3214         expr = nft_expr_first(rule);
3215         for (i = 0; i < n; i++) {
3216                 err = nf_tables_newexpr(&ctx, &info[i], expr);
3217                 if (err < 0)
3218                         goto err2;
3219
3220                 if (info[i].ops->validate)
3221                         nft_validate_state_update(net, NFT_VALIDATE_NEED);
3222
3223                 info[i].ops = NULL;
3224                 expr = nft_expr_next(expr);
3225         }
3226
3227         if (nlh->nlmsg_flags & NLM_F_REPLACE) {
3228                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
3229                 if (trans == NULL) {
3230                         err = -ENOMEM;
3231                         goto err2;
3232                 }
3233                 err = nft_delrule(&ctx, old_rule);
3234                 if (err < 0) {
3235                         nft_trans_destroy(trans);
3236                         goto err2;
3237                 }
3238
3239                 list_add_tail_rcu(&rule->list, &old_rule->list);
3240         } else {
3241                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
3242                 if (!trans) {
3243                         err = -ENOMEM;
3244                         goto err2;
3245                 }
3246
3247                 if (nlh->nlmsg_flags & NLM_F_APPEND) {
3248                         if (old_rule)
3249                                 list_add_rcu(&rule->list, &old_rule->list);
3250                         else
3251                                 list_add_tail_rcu(&rule->list, &chain->rules);
3252                  } else {
3253                         if (old_rule)
3254                                 list_add_tail_rcu(&rule->list, &old_rule->list);
3255                         else
3256                                 list_add_rcu(&rule->list, &chain->rules);
3257                 }
3258         }
3259         kvfree(info);
3260         chain->use++;
3261
3262         if (net->nft.validate_state == NFT_VALIDATE_DO)
3263                 return nft_table_validate(net, table);
3264
3265         if (chain->flags & NFT_CHAIN_HW_OFFLOAD) {
3266                 flow = nft_flow_rule_create(net, rule);
3267                 if (IS_ERR(flow))
3268                         return PTR_ERR(flow);
3269
3270                 nft_trans_flow_rule(trans) = flow;
3271         }
3272
3273         return 0;
3274 err2:
3275         nf_tables_rule_release(&ctx, rule);
3276 err1:
3277         for (i = 0; i < n; i++) {
3278                 if (info[i].ops) {
3279                         module_put(info[i].ops->type->owner);
3280                         if (info[i].ops->type->release_ops)
3281                                 info[i].ops->type->release_ops(info[i].ops);
3282                 }
3283         }
3284         kvfree(info);
3285         return err;
3286 }
3287
3288 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
3289                                              const struct nlattr *nla)
3290 {
3291         u32 id = ntohl(nla_get_be32(nla));
3292         struct nft_trans *trans;
3293
3294         list_for_each_entry(trans, &net->nft.commit_list, list) {
3295                 struct nft_rule *rule = nft_trans_rule(trans);
3296
3297                 if (trans->msg_type == NFT_MSG_NEWRULE &&
3298                     id == nft_trans_rule_id(trans))
3299                         return rule;
3300         }
3301         return ERR_PTR(-ENOENT);
3302 }
3303
3304 static int nf_tables_delrule(struct net *net, struct sock *nlsk,
3305                              struct sk_buff *skb, const struct nlmsghdr *nlh,
3306                              const struct nlattr * const nla[],
3307                              struct netlink_ext_ack *extack)
3308 {
3309         const struct nfgenmsg *nfmsg = nlmsg_data(nlh);
3310         u8 genmask = nft_genmask_next(net);
3311         struct nft_table *table;
3312         struct nft_chain *chain = NULL;
3313         struct nft_rule *rule;
3314         int family = nfmsg->nfgen_family, err = 0;
3315         struct nft_ctx ctx;
3316
3317         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask);
3318         if (IS_ERR(table)) {
3319                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3320                 return PTR_ERR(table);
3321         }
3322
3323         if (nla[NFTA_RULE_CHAIN]) {
3324                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
3325                                          genmask);
3326                 if (IS_ERR(chain)) {
3327                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3328                         return PTR_ERR(chain);
3329                 }
3330                 if (nft_chain_is_bound(chain))
3331                         return -EOPNOTSUPP;
3332         }
3333
3334         nft_ctx_init(&ctx, net, skb, nlh, family, table, chain, nla);
3335
3336         if (chain) {
3337                 if (nla[NFTA_RULE_HANDLE]) {
3338                         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
3339                         if (IS_ERR(rule)) {
3340                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3341                                 return PTR_ERR(rule);
3342                         }
3343
3344                         err = nft_delrule(&ctx, rule);
3345                 } else if (nla[NFTA_RULE_ID]) {
3346                         rule = nft_rule_lookup_byid(net, nla[NFTA_RULE_ID]);
3347                         if (IS_ERR(rule)) {
3348                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_ID]);
3349                                 return PTR_ERR(rule);
3350                         }
3351
3352                         err = nft_delrule(&ctx, rule);
3353                 } else {
3354                         err = nft_delrule_by_chain(&ctx);
3355                 }
3356         } else {
3357                 list_for_each_entry(chain, &table->chains, list) {
3358                         if (!nft_is_active_next(net, chain))
3359                                 continue;
3360
3361                         ctx.chain = chain;
3362                         err = nft_delrule_by_chain(&ctx);
3363                         if (err < 0)
3364                                 break;
3365                 }
3366         }
3367
3368         return err;
3369 }
3370
3371 /*
3372  * Sets
3373  */
3374 static const struct nft_set_type *nft_set_types[] = {
3375         &nft_set_hash_fast_type,
3376         &nft_set_hash_type,
3377         &nft_set_rhash_type,
3378         &nft_set_bitmap_type,
3379         &nft_set_rbtree_type,
3380 #if defined(CONFIG_X86_64) && !defined(CONFIG_UML)
3381         &nft_set_pipapo_avx2_type,
3382 #endif
3383         &nft_set_pipapo_type,
3384 };
3385
3386 #define NFT_SET_FEATURES        (NFT_SET_INTERVAL | NFT_SET_MAP | \
3387                                  NFT_SET_TIMEOUT | NFT_SET_OBJECT | \
3388                                  NFT_SET_EVAL)
3389
3390 static bool nft_set_ops_candidate(const struct nft_set_type *type, u32 flags)
3391 {
3392         return (flags & type->features) == (flags & NFT_SET_FEATURES);
3393 }
3394
3395 /*
3396  * Select a set implementation based on the data characteristics and the
3397  * given policy. The total memory use might not be known if no size is
3398  * given, in that case the amount of memory per element is used.
3399  */
3400 static const struct nft_set_ops *
3401 nft_select_set_ops(const struct nft_ctx *ctx,
3402                    const struct nlattr * const nla[],
3403                    const struct nft_set_desc *desc,
3404                    enum nft_set_policies policy)
3405 {
3406         const struct nft_set_ops *ops, *bops;
3407         struct nft_set_estimate est, best;
3408         const struct nft_set_type *type;
3409         u32 flags = 0;
3410         int i;
3411
3412         lockdep_assert_held(&ctx->net->nft.commit_mutex);
3413         lockdep_nfnl_nft_mutex_not_held();
3414
3415         if (nla[NFTA_SET_FLAGS] != NULL)
3416                 flags = ntohl(nla_get_be32(nla[NFTA_SET_FLAGS]));
3417
3418         bops        = NULL;
3419         best.size   = ~0;
3420         best.lookup = ~0;
3421         best.space  = ~0;
3422
3423         for (i = 0; i < ARRAY_SIZE(nft_set_types); i++) {
3424                 type = nft_set_types[i];
3425                 ops = &type->ops;
3426
3427                 if (!nft_set_ops_candidate(type, flags))
3428                         continue;
3429                 if (!ops->estimate(desc, flags, &est))
3430                         continue;
3431
3432                 switch (policy) {
3433                 case NFT_SET_POL_PERFORMANCE:
3434                         if (est.lookup < best.lookup)
3435                                 break;
3436                         if (est.lookup == best.lookup &&
3437                             est.space < best.space)
3438                                 break;
3439                         continue;
3440                 case NFT_SET_POL_MEMORY:
3441                         if (!desc->size) {
3442                                 if (est.space < best.space)
3443                                         break;
3444                                 if (est.space == best.space &&
3445                                     est.lookup < best.lookup)
3446                                         break;
3447                         } else if (est.size < best.size || !bops) {
3448                                 break;
3449                         }
3450                         continue;
3451                 default:
3452                         break;
3453                 }
3454
3455                 bops = ops;
3456                 best = est;
3457         }
3458
3459         if (bops != NULL)
3460                 return bops;
3461
3462         return ERR_PTR(-EOPNOTSUPP);
3463 }
3464
3465 static const struct nla_policy nft_set_policy[NFTA_SET_MAX + 1] = {
3466         [NFTA_SET_TABLE]                = { .type = NLA_STRING,
3467                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
3468         [NFTA_SET_NAME]                 = { .type = NLA_STRING,
3469                                             .len = NFT_SET_MAXNAMELEN - 1 },
3470         [NFTA_SET_FLAGS]                = { .type = NLA_U32 },
3471         [NFTA_SET_KEY_TYPE]             = { .type = NLA_U32 },
3472         [NFTA_SET_KEY_LEN]              = { .type = NLA_U32 },
3473         [NFTA_SET_DATA_TYPE]            = { .type = NLA_U32 },
3474         [NFTA_SET_DATA_LEN]             = { .type = NLA_U32 },
3475         [NFTA_SET_POLICY]               = { .type = NLA_U32 },
3476         [NFTA_SET_DESC]                 = { .type = NLA_NESTED },
3477         [NFTA_SET_ID]                   = { .type = NLA_U32 },
3478         [NFTA_SET_TIMEOUT]              = { .type = NLA_U64 },
3479         [NFTA_SET_GC_INTERVAL]          = { .type = NLA_U32 },
3480         [NFTA_SET_USERDATA]             = { .type = NLA_BINARY,
3481                                             .len  = NFT_USERDATA_MAXLEN },
3482         [NFTA_SET_OBJ_TYPE]             = { .type = NLA_U32 },
3483         [NFTA_SET_HANDLE]               = { .type = NLA_U64 },
3484         [NFTA_SET_EXPR]                 = { .type = NLA_NESTED },
3485 };
3486
3487 static const struct nla_policy nft_set_desc_policy[NFTA_SET_DESC_MAX + 1] = {
3488         [NFTA_SET_DESC_SIZE]            = { .type = NLA_U32 },
3489         [NFTA_SET_DESC_CONCAT]          = { .type = NLA_NESTED },
3490 };
3491
3492 static int nft_ctx_init_from_setattr(struct nft_ctx *ctx, struct net *net,
3493                                      const struct sk_buff *skb,
3494                                      const struct nlmsghdr *nlh,
3495                                      const struct nlattr * const nla[],
3496                                      struct netlink_ext_ack *extack,
3497                                      u8 genmask)
3498 {
3499         const struct nfgenmsg *nfmsg = nlmsg_data(nlh);
3500         int family = nfmsg->nfgen_family;
3501         struct nft_table *table = NULL;
3502
3503         if (nla[NFTA_SET_TABLE] != NULL) {
3504                 table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
3505                                          genmask);
3506                 if (IS_ERR(table)) {
3507                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
3508                         return PTR_ERR(table);
3509                 }
3510         }
3511
3512         nft_ctx_init(ctx, net, skb, nlh, family, table, NULL, nla);
3513         return 0;
3514 }
3515
3516 static struct nft_set *nft_set_lookup(const struct nft_table *table,
3517                                       const struct nlattr *nla, u8 genmask)
3518 {
3519         struct nft_set *set;
3520
3521         if (nla == NULL)
3522                 return ERR_PTR(-EINVAL);
3523
3524         list_for_each_entry_rcu(set, &table->sets, list) {
3525                 if (!nla_strcmp(nla, set->name) &&
3526                     nft_active_genmask(set, genmask))
3527                         return set;
3528         }
3529         return ERR_PTR(-ENOENT);
3530 }
3531
3532 static struct nft_set *nft_set_lookup_byhandle(const struct nft_table *table,
3533                                                const struct nlattr *nla,
3534                                                u8 genmask)
3535 {
3536         struct nft_set *set;
3537
3538         list_for_each_entry(set, &table->sets, list) {
3539                 if (be64_to_cpu(nla_get_be64(nla)) == set->handle &&
3540                     nft_active_genmask(set, genmask))
3541                         return set;
3542         }
3543         return ERR_PTR(-ENOENT);
3544 }
3545
3546 static struct nft_set *nft_set_lookup_byid(const struct net *net,
3547                                            const struct nlattr *nla, u8 genmask)
3548 {
3549         struct nft_trans *trans;
3550         u32 id = ntohl(nla_get_be32(nla));
3551
3552         list_for_each_entry(trans, &net->nft.commit_list, list) {
3553                 if (trans->msg_type == NFT_MSG_NEWSET) {
3554                         struct nft_set *set = nft_trans_set(trans);
3555
3556                         if (id == nft_trans_set_id(trans) &&
3557                             nft_active_genmask(set, genmask))
3558                                 return set;
3559                 }
3560         }
3561         return ERR_PTR(-ENOENT);
3562 }
3563
3564 struct nft_set *nft_set_lookup_global(const struct net *net,
3565                                       const struct nft_table *table,
3566                                       const struct nlattr *nla_set_name,
3567                                       const struct nlattr *nla_set_id,
3568                                       u8 genmask)
3569 {
3570         struct nft_set *set;
3571
3572         set = nft_set_lookup(table, nla_set_name, genmask);
3573         if (IS_ERR(set)) {
3574                 if (!nla_set_id)
3575                         return set;
3576
3577                 set = nft_set_lookup_byid(net, nla_set_id, genmask);
3578         }
3579         return set;
3580 }
3581 EXPORT_SYMBOL_GPL(nft_set_lookup_global);
3582
3583 static int nf_tables_set_alloc_name(struct nft_ctx *ctx, struct nft_set *set,
3584                                     const char *name)
3585 {
3586         const struct nft_set *i;
3587         const char *p;
3588         unsigned long *inuse;
3589         unsigned int n = 0, min = 0;
3590
3591         p = strchr(name, '%');
3592         if (p != NULL) {
3593                 if (p[1] != 'd' || strchr(p + 2, '%'))
3594                         return -EINVAL;
3595
3596                 inuse = (unsigned long *)get_zeroed_page(GFP_KERNEL);
3597                 if (inuse == NULL)
3598                         return -ENOMEM;
3599 cont:
3600                 list_for_each_entry(i, &ctx->table->sets, list) {
3601                         int tmp;
3602
3603                         if (!nft_is_active_next(ctx->net, set))
3604                                 continue;
3605                         if (!sscanf(i->name, name, &tmp))
3606                                 continue;
3607                         if (tmp < min || tmp >= min + BITS_PER_BYTE * PAGE_SIZE)
3608                                 continue;
3609
3610                         set_bit(tmp - min, inuse);
3611                 }
3612
3613                 n = find_first_zero_bit(inuse, BITS_PER_BYTE * PAGE_SIZE);
3614                 if (n >= BITS_PER_BYTE * PAGE_SIZE) {
3615                         min += BITS_PER_BYTE * PAGE_SIZE;
3616                         memset(inuse, 0, PAGE_SIZE);
3617                         goto cont;
3618                 }
3619                 free_page((unsigned long)inuse);
3620         }
3621
3622         set->name = kasprintf(GFP_KERNEL, name, min + n);
3623         if (!set->name)
3624                 return -ENOMEM;
3625
3626         list_for_each_entry(i, &ctx->table->sets, list) {
3627                 if (!nft_is_active_next(ctx->net, i))
3628                         continue;
3629                 if (!strcmp(set->name, i->name)) {
3630                         kfree(set->name);
3631                         set->name = NULL;
3632                         return -ENFILE;
3633                 }
3634         }
3635         return 0;
3636 }
3637
3638 static int nf_msecs_to_jiffies64(const struct nlattr *nla, u64 *result)
3639 {
3640         u64 ms = be64_to_cpu(nla_get_be64(nla));
3641         u64 max = (u64)(~((u64)0));
3642
3643         max = div_u64(max, NSEC_PER_MSEC);
3644         if (ms >= max)
3645                 return -ERANGE;
3646
3647         ms *= NSEC_PER_MSEC;
3648         *result = nsecs_to_jiffies64(ms);
3649         return 0;
3650 }
3651
3652 static __be64 nf_jiffies64_to_msecs(u64 input)
3653 {
3654         return cpu_to_be64(jiffies64_to_msecs(input));
3655 }
3656
3657 static int nf_tables_fill_set_concat(struct sk_buff *skb,
3658                                      const struct nft_set *set)
3659 {
3660         struct nlattr *concat, *field;
3661         int i;
3662
3663         concat = nla_nest_start_noflag(skb, NFTA_SET_DESC_CONCAT);
3664         if (!concat)
3665                 return -ENOMEM;
3666
3667         for (i = 0; i < set->field_count; i++) {
3668                 field = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
3669                 if (!field)
3670                         return -ENOMEM;
3671
3672                 if (nla_put_be32(skb, NFTA_SET_FIELD_LEN,
3673                                  htonl(set->field_len[i])))
3674                         return -ENOMEM;
3675
3676                 nla_nest_end(skb, field);
3677         }
3678
3679         nla_nest_end(skb, concat);
3680
3681         return 0;
3682 }
3683
3684 static int nf_tables_fill_set(struct sk_buff *skb, const struct nft_ctx *ctx,
3685                               const struct nft_set *set, u16 event, u16 flags)
3686 {
3687         struct nfgenmsg *nfmsg;
3688         struct nlmsghdr *nlh;
3689         u32 portid = ctx->portid;
3690         struct nlattr *nest;
3691         u32 seq = ctx->seq;
3692
3693         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
3694         nlh = nlmsg_put(skb, portid, seq, event, sizeof(struct nfgenmsg),
3695                         flags);
3696         if (nlh == NULL)
3697                 goto nla_put_failure;
3698
3699         nfmsg = nlmsg_data(nlh);
3700         nfmsg->nfgen_family     = ctx->family;
3701         nfmsg->version          = NFNETLINK_V0;
3702         nfmsg->res_id           = htons(ctx->net->nft.base_seq & 0xffff);
3703
3704         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
3705                 goto nla_put_failure;
3706         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
3707                 goto nla_put_failure;
3708         if (nla_put_be64(skb, NFTA_SET_HANDLE, cpu_to_be64(set->handle),
3709                          NFTA_SET_PAD))
3710                 goto nla_put_failure;
3711         if (set->flags != 0)
3712                 if (nla_put_be32(skb, NFTA_SET_FLAGS, htonl(set->flags)))
3713                         goto nla_put_failure;
3714
3715         if (nla_put_be32(skb, NFTA_SET_KEY_TYPE, htonl(set->ktype)))
3716                 goto nla_put_failure;
3717         if (nla_put_be32(skb, NFTA_SET_KEY_LEN, htonl(set->klen)))
3718                 goto nla_put_failure;
3719         if (set->flags & NFT_SET_MAP) {
3720                 if (nla_put_be32(skb, NFTA_SET_DATA_TYPE, htonl(set->dtype)))
3721                         goto nla_put_failure;
3722                 if (nla_put_be32(skb, NFTA_SET_DATA_LEN, htonl(set->dlen)))
3723                         goto nla_put_failure;
3724         }
3725         if (set->flags & NFT_SET_OBJECT &&
3726             nla_put_be32(skb, NFTA_SET_OBJ_TYPE, htonl(set->objtype)))
3727                 goto nla_put_failure;
3728
3729         if (set->timeout &&
3730             nla_put_be64(skb, NFTA_SET_TIMEOUT,
3731                          nf_jiffies64_to_msecs(set->timeout),
3732                          NFTA_SET_PAD))
3733                 goto nla_put_failure;
3734         if (set->gc_int &&
3735             nla_put_be32(skb, NFTA_SET_GC_INTERVAL, htonl(set->gc_int)))
3736                 goto nla_put_failure;
3737
3738         if (set->policy != NFT_SET_POL_PERFORMANCE) {
3739                 if (nla_put_be32(skb, NFTA_SET_POLICY, htonl(set->policy)))
3740                         goto nla_put_failure;
3741         }
3742
3743         if (nla_put(skb, NFTA_SET_USERDATA, set->udlen, set->udata))
3744                 goto nla_put_failure;
3745
3746         nest = nla_nest_start_noflag(skb, NFTA_SET_DESC);
3747         if (!nest)
3748                 goto nla_put_failure;
3749         if (set->size &&
3750             nla_put_be32(skb, NFTA_SET_DESC_SIZE, htonl(set->size)))
3751                 goto nla_put_failure;
3752
3753         if (set->field_count > 1 &&
3754             nf_tables_fill_set_concat(skb, set))
3755                 goto nla_put_failure;
3756
3757         nla_nest_end(skb, nest);
3758
3759         if (set->expr) {
3760                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPR);
3761                 if (nf_tables_fill_expr_info(skb, set->expr) < 0)
3762                         goto nla_put_failure;
3763
3764                 nla_nest_end(skb, nest);
3765         }
3766
3767         nlmsg_end(skb, nlh);
3768         return 0;
3769
3770 nla_put_failure:
3771         nlmsg_trim(skb, nlh);
3772         return -1;
3773 }
3774
3775 static void nf_tables_set_notify(const struct nft_ctx *ctx,
3776                                  const struct nft_set *set, int event,
3777                                  gfp_t gfp_flags)
3778 {
3779         struct sk_buff *skb;
3780         u32 portid = ctx->portid;
3781         int err;
3782
3783         if (!ctx->report &&
3784             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
3785                 return;
3786
3787         skb = nlmsg_new(NLMSG_GOODSIZE, gfp_flags);
3788         if (skb == NULL)
3789                 goto err;
3790
3791         err = nf_tables_fill_set(skb, ctx, set, event, 0);
3792         if (err < 0) {
3793                 kfree_skb(skb);
3794                 goto err;
3795         }
3796
3797         nfnetlink_send(skb, ctx->net, portid, NFNLGRP_NFTABLES, ctx->report,
3798                        gfp_flags);
3799         return;
3800 err:
3801         nfnetlink_set_err(ctx->net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
3802 }
3803
3804 static int nf_tables_dump_sets(struct sk_buff *skb, struct netlink_callback *cb)
3805 {
3806         const struct nft_set *set;
3807         unsigned int idx, s_idx = cb->args[0];
3808         struct nft_table *table, *cur_table = (struct nft_table *)cb->args[2];
3809         struct net *net = sock_net(skb->sk);
3810         struct nft_ctx *ctx = cb->data, ctx_set;
3811
3812         if (cb->args[1])
3813                 return skb->len;
3814
3815         rcu_read_lock();
3816         cb->seq = net->nft.base_seq;
3817
3818         list_for_each_entry_rcu(table, &net->nft.tables, list) {
3819                 if (ctx->family != NFPROTO_UNSPEC &&
3820                     ctx->family != table->family)
3821                         continue;
3822
3823                 if (ctx->table && ctx->table != table)
3824                         continue;
3825
3826                 if (cur_table) {
3827                         if (cur_table != table)
3828                                 continue;
3829
3830                         cur_table = NULL;
3831                 }
3832                 idx = 0;
3833                 list_for_each_entry_rcu(set, &table->sets, list) {
3834                         if (idx < s_idx)
3835                                 goto cont;
3836                         if (!nft_is_active(net, set))
3837                                 goto cont;
3838
3839                         ctx_set = *ctx;
3840                         ctx_set.table = table;
3841                         ctx_set.family = table->family;
3842
3843                         if (nf_tables_fill_set(skb, &ctx_set, set,
3844                                                NFT_MSG_NEWSET,
3845                                                NLM_F_MULTI) < 0) {
3846                                 cb->args[0] = idx;
3847                                 cb->args[2] = (unsigned long) table;
3848                                 goto done;
3849                         }
3850                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
3851 cont:
3852                         idx++;
3853                 }
3854                 if (s_idx)
3855                         s_idx = 0;
3856         }
3857         cb->args[1] = 1;
3858 done:
3859         rcu_read_unlock();
3860         return skb->len;
3861 }
3862
3863 static int nf_tables_dump_sets_start(struct netlink_callback *cb)
3864 {
3865         struct nft_ctx *ctx_dump = NULL;
3866
3867         ctx_dump = kmemdup(cb->data, sizeof(*ctx_dump), GFP_ATOMIC);
3868         if (ctx_dump == NULL)
3869                 return -ENOMEM;
3870
3871         cb->data = ctx_dump;
3872         return 0;
3873 }
3874
3875 static int nf_tables_dump_sets_done(struct netlink_callback *cb)
3876 {
3877         kfree(cb->data);
3878         return 0;
3879 }
3880
3881 /* called with rcu_read_lock held */
3882 static int nf_tables_getset(struct net *net, struct sock *nlsk,
3883                             struct sk_buff *skb, const struct nlmsghdr *nlh,
3884                             const struct nlattr * const nla[],
3885                             struct netlink_ext_ack *extack)
3886 {
3887         u8 genmask = nft_genmask_cur(net);
3888         const struct nft_set *set;
3889         struct nft_ctx ctx;
3890         struct sk_buff *skb2;
3891         const struct nfgenmsg *nfmsg = nlmsg_data(nlh);
3892         int err;
3893
3894         /* Verify existence before starting dump */
3895         err = nft_ctx_init_from_setattr(&ctx, net, skb, nlh, nla, extack,
3896                                         genmask);
3897         if (err < 0)
3898                 return err;
3899
3900         if (nlh->nlmsg_flags & NLM_F_DUMP) {
3901                 struct netlink_dump_control c = {
3902                         .start = nf_tables_dump_sets_start,
3903                         .dump = nf_tables_dump_sets,
3904                         .done = nf_tables_dump_sets_done,
3905                         .data = &ctx,
3906                         .module = THIS_MODULE,
3907                 };
3908
3909                 return nft_netlink_dump_start_rcu(nlsk, skb, nlh, &c);
3910         }
3911
3912         /* Only accept unspec with dump */
3913         if (nfmsg->nfgen_family == NFPROTO_UNSPEC)
3914                 return -EAFNOSUPPORT;
3915         if (!nla[NFTA_SET_TABLE])
3916                 return -EINVAL;
3917
3918         set = nft_set_lookup(ctx.table, nla[NFTA_SET_NAME], genmask);
3919         if (IS_ERR(set))
3920                 return PTR_ERR(set);
3921
3922         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
3923         if (skb2 == NULL)
3924                 return -ENOMEM;
3925
3926         err = nf_tables_fill_set(skb2, &ctx, set, NFT_MSG_NEWSET, 0);
3927         if (err < 0)
3928                 goto err;
3929
3930         return nlmsg_unicast(nlsk, skb2, NETLINK_CB(skb).portid);
3931
3932 err:
3933         kfree_skb(skb2);
3934         return err;
3935 }
3936
3937 static const struct nla_policy nft_concat_policy[NFTA_SET_FIELD_MAX + 1] = {
3938         [NFTA_SET_FIELD_LEN]    = { .type = NLA_U32 },
3939 };
3940
3941 static int nft_set_desc_concat_parse(const struct nlattr *attr,
3942                                      struct nft_set_desc *desc)
3943 {
3944         struct nlattr *tb[NFTA_SET_FIELD_MAX + 1];
3945         u32 len;
3946         int err;
3947
3948         err = nla_parse_nested_deprecated(tb, NFTA_SET_FIELD_MAX, attr,
3949                                           nft_concat_policy, NULL);
3950         if (err < 0)
3951                 return err;
3952
3953         if (!tb[NFTA_SET_FIELD_LEN])
3954                 return -EINVAL;
3955
3956         len = ntohl(nla_get_be32(tb[NFTA_SET_FIELD_LEN]));
3957
3958         if (len * BITS_PER_BYTE / 32 > NFT_REG32_COUNT)
3959                 return -E2BIG;
3960
3961         desc->field_len[desc->field_count++] = len;
3962
3963         return 0;
3964 }
3965
3966 static int nft_set_desc_concat(struct nft_set_desc *desc,
3967                                const struct nlattr *nla)
3968 {
3969         struct nlattr *attr;
3970         int rem, err;
3971
3972         nla_for_each_nested(attr, nla, rem) {
3973                 if (nla_type(attr) != NFTA_LIST_ELEM)
3974                         return -EINVAL;
3975
3976                 err = nft_set_desc_concat_parse(attr, desc);
3977                 if (err < 0)
3978                         return err;
3979         }
3980
3981         return 0;
3982 }
3983
3984 static int nf_tables_set_desc_parse(struct nft_set_desc *desc,
3985                                     const struct nlattr *nla)
3986 {
3987         struct nlattr *da[NFTA_SET_DESC_MAX + 1];
3988         int err;
3989
3990         err = nla_parse_nested_deprecated(da, NFTA_SET_DESC_MAX, nla,
3991                                           nft_set_desc_policy, NULL);
3992         if (err < 0)
3993                 return err;
3994
3995         if (da[NFTA_SET_DESC_SIZE] != NULL)
3996                 desc->size = ntohl(nla_get_be32(da[NFTA_SET_DESC_SIZE]));
3997         if (da[NFTA_SET_DESC_CONCAT])
3998                 err = nft_set_desc_concat(desc, da[NFTA_SET_DESC_CONCAT]);
3999
4000         return err;
4001 }
4002
4003 static int nf_tables_newset(struct net *net, struct sock *nlsk,
4004                             struct sk_buff *skb, const struct nlmsghdr *nlh,
4005                             const struct nlattr * const nla[],
4006                             struct netlink_ext_ack *extack)
4007 {
4008         const struct nfgenmsg *nfmsg = nlmsg_data(nlh);
4009         u8 genmask = nft_genmask_next(net);
4010         int family = nfmsg->nfgen_family;
4011         const struct nft_set_ops *ops;
4012         struct nft_expr *expr = NULL;
4013         struct nft_table *table;
4014         struct nft_set *set;
4015         struct nft_ctx ctx;
4016         char *name;
4017         u64 size;
4018         u64 timeout;
4019         u32 ktype, dtype, flags, policy, gc_int, objtype;
4020         struct nft_set_desc desc;
4021         unsigned char *udata;
4022         u16 udlen;
4023         int err;
4024         int i;
4025
4026         if (nla[NFTA_SET_TABLE] == NULL ||
4027             nla[NFTA_SET_NAME] == NULL ||
4028             nla[NFTA_SET_KEY_LEN] == NULL ||
4029             nla[NFTA_SET_ID] == NULL)
4030                 return -EINVAL;
4031
4032         memset(&desc, 0, sizeof(desc));
4033
4034         ktype = NFT_DATA_VALUE;
4035         if (nla[NFTA_SET_KEY_TYPE] != NULL) {
4036                 ktype = ntohl(nla_get_be32(nla[NFTA_SET_KEY_TYPE]));
4037                 if ((ktype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK)
4038                         return -EINVAL;
4039         }
4040
4041         desc.klen = ntohl(nla_get_be32(nla[NFTA_SET_KEY_LEN]));
4042         if (desc.klen == 0 || desc.klen > NFT_DATA_VALUE_MAXLEN)
4043                 return -EINVAL;
4044
4045         flags = 0;
4046         if (nla[NFTA_SET_FLAGS] != NULL) {
4047                 flags = ntohl(nla_get_be32(nla[NFTA_SET_FLAGS]));
4048                 if (flags & ~(NFT_SET_ANONYMOUS | NFT_SET_CONSTANT |
4049                               NFT_SET_INTERVAL | NFT_SET_TIMEOUT |
4050                               NFT_SET_MAP | NFT_SET_EVAL |
4051                               NFT_SET_OBJECT | NFT_SET_CONCAT))
4052                         return -EOPNOTSUPP;
4053                 /* Only one of these operations is supported */
4054                 if ((flags & (NFT_SET_MAP | NFT_SET_OBJECT)) ==
4055                              (NFT_SET_MAP | NFT_SET_OBJECT))
4056                         return -EOPNOTSUPP;
4057                 if ((flags & (NFT_SET_EVAL | NFT_SET_OBJECT)) ==
4058                              (NFT_SET_EVAL | NFT_SET_OBJECT))
4059                         return -EOPNOTSUPP;
4060         }
4061
4062         dtype = 0;
4063         if (nla[NFTA_SET_DATA_TYPE] != NULL) {
4064                 if (!(flags & NFT_SET_MAP))
4065                         return -EINVAL;
4066
4067                 dtype = ntohl(nla_get_be32(nla[NFTA_SET_DATA_TYPE]));
4068                 if ((dtype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK &&
4069                     dtype != NFT_DATA_VERDICT)
4070                         return -EINVAL;
4071
4072                 if (dtype != NFT_DATA_VERDICT) {
4073                         if (nla[NFTA_SET_DATA_LEN] == NULL)
4074                                 return -EINVAL;
4075                         desc.dlen = ntohl(nla_get_be32(nla[NFTA_SET_DATA_LEN]));
4076                         if (desc.dlen == 0 || desc.dlen > NFT_DATA_VALUE_MAXLEN)
4077                                 return -EINVAL;
4078                 } else
4079                         desc.dlen = sizeof(struct nft_verdict);
4080         } else if (flags & NFT_SET_MAP)
4081                 return -EINVAL;
4082
4083         if (nla[NFTA_SET_OBJ_TYPE] != NULL) {
4084                 if (!(flags & NFT_SET_OBJECT))
4085                         return -EINVAL;
4086
4087                 objtype = ntohl(nla_get_be32(nla[NFTA_SET_OBJ_TYPE]));
4088                 if (objtype == NFT_OBJECT_UNSPEC ||
4089                     objtype > NFT_OBJECT_MAX)
4090                         return -EOPNOTSUPP;
4091         } else if (flags & NFT_SET_OBJECT)
4092                 return -EINVAL;
4093         else
4094                 objtype = NFT_OBJECT_UNSPEC;
4095
4096         timeout = 0;
4097         if (nla[NFTA_SET_TIMEOUT] != NULL) {
4098                 if (!(flags & NFT_SET_TIMEOUT))
4099                         return -EINVAL;
4100
4101                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_TIMEOUT], &timeout);
4102                 if (err)
4103                         return err;
4104         }
4105         gc_int = 0;
4106         if (nla[NFTA_SET_GC_INTERVAL] != NULL) {
4107                 if (!(flags & NFT_SET_TIMEOUT))
4108                         return -EINVAL;
4109                 gc_int = ntohl(nla_get_be32(nla[NFTA_SET_GC_INTERVAL]));
4110         }
4111
4112         policy = NFT_SET_POL_PERFORMANCE;
4113         if (nla[NFTA_SET_POLICY] != NULL)
4114                 policy = ntohl(nla_get_be32(nla[NFTA_SET_POLICY]));
4115
4116         if (nla[NFTA_SET_DESC] != NULL) {
4117                 err = nf_tables_set_desc_parse(&desc, nla[NFTA_SET_DESC]);
4118                 if (err < 0)
4119                         return err;
4120         }
4121
4122         if (nla[NFTA_SET_EXPR])
4123                 desc.expr = true;
4124
4125         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family, genmask);
4126         if (IS_ERR(table)) {
4127                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
4128                 return PTR_ERR(table);
4129         }
4130
4131         nft_ctx_init(&ctx, net, skb, nlh, family, table, NULL, nla);
4132
4133         set = nft_set_lookup(table, nla[NFTA_SET_NAME], genmask);
4134         if (IS_ERR(set)) {
4135                 if (PTR_ERR(set) != -ENOENT) {
4136                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
4137                         return PTR_ERR(set);
4138                 }
4139         } else {
4140                 if (nlh->nlmsg_flags & NLM_F_EXCL) {
4141                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
4142                         return -EEXIST;
4143                 }
4144                 if (nlh->nlmsg_flags & NLM_F_REPLACE)
4145                         return -EOPNOTSUPP;
4146
4147                 return 0;
4148         }
4149
4150         if (!(nlh->nlmsg_flags & NLM_F_CREATE))
4151                 return -ENOENT;
4152
4153         ops = nft_select_set_ops(&ctx, nla, &desc, policy);
4154         if (IS_ERR(ops))
4155                 return PTR_ERR(ops);
4156
4157         udlen = 0;
4158         if (nla[NFTA_SET_USERDATA])
4159                 udlen = nla_len(nla[NFTA_SET_USERDATA]);
4160
4161         size = 0;
4162         if (ops->privsize != NULL)
4163                 size = ops->privsize(nla, &desc);
4164
4165         set = kvzalloc(sizeof(*set) + size + udlen, GFP_KERNEL);
4166         if (!set)
4167                 return -ENOMEM;
4168
4169         name = nla_strdup(nla[NFTA_SET_NAME], GFP_KERNEL);
4170         if (!name) {
4171                 err = -ENOMEM;
4172                 goto err_set_name;
4173         }
4174
4175         err = nf_tables_set_alloc_name(&ctx, set, name);
4176         kfree(name);
4177         if (err < 0)
4178                 goto err_set_alloc_name;
4179
4180         if (nla[NFTA_SET_EXPR]) {
4181                 expr = nft_set_elem_expr_alloc(&ctx, set, nla[NFTA_SET_EXPR]);
4182                 if (IS_ERR(expr)) {
4183                         err = PTR_ERR(expr);
4184                         goto err_set_alloc_name;
4185                 }
4186         }
4187
4188         udata = NULL;
4189         if (udlen) {
4190                 udata = set->data + size;
4191                 nla_memcpy(udata, nla[NFTA_SET_USERDATA], udlen);
4192         }
4193
4194         INIT_LIST_HEAD(&set->bindings);
4195         set->table = table;
4196         write_pnet(&set->net, net);
4197         set->ops   = ops;
4198         set->ktype = ktype;
4199         set->klen  = desc.klen;
4200         set->dtype = dtype;
4201         set->objtype = objtype;
4202         set->dlen  = desc.dlen;
4203         set->expr = expr;
4204         set->flags = flags;
4205         set->size  = desc.size;
4206         set->policy = policy;
4207         set->udlen  = udlen;
4208         set->udata  = udata;
4209         set->timeout = timeout;
4210         set->gc_int = gc_int;
4211         set->handle = nf_tables_alloc_handle(table);
4212
4213         set->field_count = desc.field_count;
4214         for (i = 0; i < desc.field_count; i++)
4215                 set->field_len[i] = desc.field_len[i];
4216
4217         err = ops->init(set, &desc, nla);
4218         if (err < 0)
4219                 goto err_set_init;
4220
4221         err = nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set);
4222         if (err < 0)
4223                 goto err_set_trans;
4224
4225         list_add_tail_rcu(&set->list, &table->sets);
4226         table->use++;
4227         return 0;
4228
4229 err_set_trans:
4230         ops->destroy(set);
4231 err_set_init:
4232         if (expr)
4233                 nft_expr_destroy(&ctx, expr);
4234 err_set_alloc_name:
4235         kfree(set->name);
4236 err_set_name:
4237         kvfree(set);
4238         return err;
4239 }
4240
4241 static void nft_set_destroy(const struct nft_ctx *ctx, struct nft_set *set)
4242 {
4243         if (WARN_ON(set->use > 0))
4244                 return;
4245
4246         if (set->expr)
4247                 nft_expr_destroy(ctx, set->expr);
4248
4249         set->ops->destroy(set);
4250         kfree(set->name);
4251         kvfree(set);
4252 }
4253
4254 static int nf_tables_delset(struct net *net, struct sock *nlsk,
4255                             struct sk_buff *skb, const struct nlmsghdr *nlh,
4256                             const struct nlattr * const nla[],
4257                             struct netlink_ext_ack *extack)
4258 {
4259         const struct nfgenmsg *nfmsg = nlmsg_data(nlh);
4260         u8 genmask = nft_genmask_next(net);
4261         const struct nlattr *attr;
4262         struct nft_set *set;
4263         struct nft_ctx ctx;
4264         int err;
4265
4266         if (nfmsg->nfgen_family == NFPROTO_UNSPEC)
4267                 return -EAFNOSUPPORT;
4268         if (nla[NFTA_SET_TABLE] == NULL)
4269                 return -EINVAL;
4270
4271         err = nft_ctx_init_from_setattr(&ctx, net, skb, nlh, nla, extack,
4272                                         genmask);
4273         if (err < 0)
4274                 return err;
4275
4276         if (nla[NFTA_SET_HANDLE]) {
4277                 attr = nla[NFTA_SET_HANDLE];
4278                 set = nft_set_lookup_byhandle(ctx.table, attr, genmask);
4279         } else {
4280                 attr = nla[NFTA_SET_NAME];
4281                 set = nft_set_lookup(ctx.table, attr, genmask);
4282         }
4283
4284         if (IS_ERR(set)) {
4285                 NL_SET_BAD_ATTR(extack, attr);
4286                 return PTR_ERR(set);
4287         }
4288         if (set->use ||
4289             (nlh->nlmsg_flags & NLM_F_NONREC && atomic_read(&set->nelems) > 0)) {
4290                 NL_SET_BAD_ATTR(extack, attr);
4291                 return -EBUSY;
4292         }
4293
4294         return nft_delset(&ctx, set);
4295 }
4296
4297 static int nf_tables_bind_check_setelem(const struct nft_ctx *ctx,
4298                                         struct nft_set *set,
4299                                         const struct nft_set_iter *iter,
4300                                         struct nft_set_elem *elem)
4301 {
4302         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
4303         enum nft_registers dreg;
4304
4305         dreg = nft_type_to_reg(set->dtype);
4306         return nft_validate_register_store(ctx, dreg, nft_set_ext_data(ext),
4307                                            set->dtype == NFT_DATA_VERDICT ?
4308                                            NFT_DATA_VERDICT : NFT_DATA_VALUE,
4309                                            set->dlen);
4310 }
4311
4312 int nf_tables_bind_set(const struct nft_ctx *ctx, struct nft_set *set,
4313                        struct nft_set_binding *binding)
4314 {
4315         struct nft_set_binding *i;
4316         struct nft_set_iter iter;
4317
4318         if (set->use == UINT_MAX)
4319                 return -EOVERFLOW;
4320
4321         if (!list_empty(&set->bindings) && nft_set_is_anonymous(set))
4322                 return -EBUSY;
4323
4324         if (binding->flags & NFT_SET_MAP) {
4325                 /* If the set is already bound to the same chain all
4326                  * jumps are already validated for that chain.
4327                  */
4328                 list_for_each_entry(i, &set->bindings, list) {
4329                         if (i->flags & NFT_SET_MAP &&
4330                             i->chain == binding->chain)
4331                                 goto bind;
4332                 }
4333
4334                 iter.genmask    = nft_genmask_next(ctx->net);
4335                 iter.skip       = 0;
4336                 iter.count      = 0;
4337                 iter.err        = 0;
4338                 iter.fn         = nf_tables_bind_check_setelem;
4339
4340                 set->ops->walk(ctx, set, &iter);
4341                 if (iter.err < 0)
4342                         return iter.err;
4343         }
4344 bind:
4345         binding->chain = ctx->chain;
4346         list_add_tail_rcu(&binding->list, &set->bindings);
4347         nft_set_trans_bind(ctx, set);
4348         set->use++;
4349
4350         return 0;
4351 }
4352 EXPORT_SYMBOL_GPL(nf_tables_bind_set);
4353
4354 static void nf_tables_unbind_set(const struct nft_ctx *ctx, struct nft_set *set,
4355                                  struct nft_set_binding *binding, bool event)
4356 {
4357         list_del_rcu(&binding->list);
4358
4359         if (list_empty(&set->bindings) && nft_set_is_anonymous(set)) {
4360                 list_del_rcu(&set->list);
4361                 if (event)
4362                         nf_tables_set_notify(ctx, set, NFT_MSG_DELSET,
4363                                              GFP_KERNEL);
4364         }
4365 }
4366
4367 void nf_tables_deactivate_set(const struct nft_ctx *ctx, struct nft_set *set,
4368                               struct nft_set_binding *binding,
4369                               enum nft_trans_phase phase)
4370 {
4371         switch (phase) {
4372         case NFT_TRANS_PREPARE:
4373                 set->use--;
4374                 return;
4375         case NFT_TRANS_ABORT:
4376         case NFT_TRANS_RELEASE:
4377                 set->use--;
4378                 /* fall through */
4379         default:
4380                 nf_tables_unbind_set(ctx, set, binding,
4381                                      phase == NFT_TRANS_COMMIT);
4382         }
4383 }
4384 EXPORT_SYMBOL_GPL(nf_tables_deactivate_set);
4385
4386 void nf_tables_destroy_set(const struct nft_ctx *ctx, struct nft_set *set)
4387 {
4388         if (list_empty(&set->bindings) && nft_set_is_anonymous(set))
4389                 nft_set_destroy(ctx, set);
4390 }
4391 EXPORT_SYMBOL_GPL(nf_tables_destroy_set);
4392
4393 const struct nft_set_ext_type nft_set_ext_types[] = {
4394         [NFT_SET_EXT_KEY]               = {
4395                 .align  = __alignof__(u32),
4396         },
4397         [NFT_SET_EXT_DATA]              = {
4398                 .align  = __alignof__(u32),
4399         },
4400         [NFT_SET_EXT_EXPR]              = {
4401                 .align  = __alignof__(struct nft_expr),
4402         },
4403         [NFT_SET_EXT_OBJREF]            = {
4404                 .len    = sizeof(struct nft_object *),
4405                 .align  = __alignof__(struct nft_object *),
4406         },
4407         [NFT_SET_EXT_FLAGS]             = {
4408                 .len    = sizeof(u8),
4409                 .align  = __alignof__(u8),
4410         },
4411         [NFT_SET_EXT_TIMEOUT]           = {
4412                 .len    = sizeof(u64),
4413                 .align  = __alignof__(u64),
4414         },
4415         [NFT_SET_EXT_EXPIRATION]        = {
4416                 .len    = sizeof(u64),
4417                 .align  = __alignof__(u64),
4418         },
4419         [NFT_SET_EXT_USERDATA]          = {
4420                 .len    = sizeof(struct nft_userdata),
4421                 .align  = __alignof__(struct nft_userdata),
4422         },
4423         [NFT_SET_EXT_KEY_END]           = {
4424                 .align  = __alignof__(u32),
4425         },
4426 };
4427
4428 /*
4429  * Set elements
4430  */
4431
4432 static const struct nla_policy nft_set_elem_policy[NFTA_SET_ELEM_MAX + 1] = {
4433         [NFTA_SET_ELEM_KEY]             = { .type = NLA_NESTED },
4434         [NFTA_SET_ELEM_DATA]            = { .type = NLA_NESTED },
4435         [NFTA_SET_ELEM_FLAGS]           = { .type = NLA_U32 },
4436         [NFTA_SET_ELEM_TIMEOUT]         = { .type = NLA_U64 },
4437         [NFTA_SET_ELEM_EXPIRATION]      = { .type = NLA_U64 },
4438         [NFTA_SET_ELEM_USERDATA]        = { .type = NLA_BINARY,
4439                                             .len = NFT_USERDATA_MAXLEN },
4440         [NFTA_SET_ELEM_EXPR]            = { .type = NLA_NESTED },
4441         [NFTA_SET_ELEM_OBJREF]          = { .type = NLA_STRING,
4442                                             .len = NFT_OBJ_MAXNAMELEN - 1 },
4443         [NFTA_SET_ELEM_KEY_END]         = { .type = NLA_NESTED },
4444 };
4445
4446 static const struct nla_policy nft_set_elem_list_policy[NFTA_SET_ELEM_LIST_MAX + 1] = {
4447         [NFTA_SET_ELEM_LIST_TABLE]      = { .type = NLA_STRING,
4448                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
4449         [NFTA_SET_ELEM_LIST_SET]        = { .type = NLA_STRING,
4450                                             .len = NFT_SET_MAXNAMELEN - 1 },
4451         [NFTA_SET_ELEM_LIST_ELEMENTS]   = { .type = NLA_NESTED },
4452         [NFTA_SET_ELEM_LIST_SET_ID]     = { .type = NLA_U32 },
4453 };
4454
4455 static int nft_ctx_init_from_elemattr(struct nft_ctx *ctx, struct net *net,
4456                                       const struct sk_buff *skb,
4457                                       const struct nlmsghdr *nlh,
4458                                       const struct nlattr * const nla[],
4459                                       struct netlink_ext_ack *extack,
4460                                       u8 genmask)
4461 {
4462         const struct nfgenmsg *nfmsg = nlmsg_data(nlh);
4463         int family = nfmsg->nfgen_family;
4464         struct nft_table *table;
4465
4466         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
4467                                  genmask);
4468         if (IS_ERR(table)) {
4469                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
4470                 return PTR_ERR(table);
4471         }
4472
4473         nft_ctx_init(ctx, net, skb, nlh, family, table, NULL, nla);
4474         return 0;
4475 }
4476
4477 static int nf_tables_fill_setelem(struct sk_buff *skb,
4478                                   const struct nft_set *set,
4479                                   const struct nft_set_elem *elem)
4480 {
4481         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
4482         unsigned char *b = skb_tail_pointer(skb);
4483         struct nlattr *nest;
4484
4485         nest = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
4486         if (nest == NULL)
4487                 goto nla_put_failure;
4488
4489         if (nft_data_dump(skb, NFTA_SET_ELEM_KEY, nft_set_ext_key(ext),
4490                           NFT_DATA_VALUE, set->klen) < 0)
4491                 goto nla_put_failure;
4492
4493         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
4494             nft_data_dump(skb, NFTA_SET_ELEM_KEY_END, nft_set_ext_key_end(ext),
4495                           NFT_DATA_VALUE, set->klen) < 0)
4496                 goto nla_put_failure;
4497
4498         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
4499             nft_data_dump(skb, NFTA_SET_ELEM_DATA, nft_set_ext_data(ext),
4500                           set->dtype == NFT_DATA_VERDICT ? NFT_DATA_VERDICT : NFT_DATA_VALUE,
4501                           set->dlen) < 0)
4502                 goto nla_put_failure;
4503
4504         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPR) &&
4505             nft_expr_dump(skb, NFTA_SET_ELEM_EXPR, nft_set_ext_expr(ext)) < 0)
4506                 goto nla_put_failure;
4507
4508         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
4509             nla_put_string(skb, NFTA_SET_ELEM_OBJREF,
4510                            (*nft_set_ext_obj(ext))->key.name) < 0)
4511                 goto nla_put_failure;
4512
4513         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
4514             nla_put_be32(skb, NFTA_SET_ELEM_FLAGS,
4515                          htonl(*nft_set_ext_flags(ext))))
4516                 goto nla_put_failure;
4517
4518         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT) &&
4519             nla_put_be64(skb, NFTA_SET_ELEM_TIMEOUT,
4520                          nf_jiffies64_to_msecs(*nft_set_ext_timeout(ext)),
4521                          NFTA_SET_ELEM_PAD))
4522                 goto nla_put_failure;
4523
4524         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
4525                 u64 expires, now = get_jiffies_64();
4526
4527                 expires = *nft_set_ext_expiration(ext);
4528                 if (time_before64(now, expires))
4529                         expires -= now;
4530                 else
4531                         expires = 0;
4532
4533                 if (nla_put_be64(skb, NFTA_SET_ELEM_EXPIRATION,
4534                                  nf_jiffies64_to_msecs(expires),
4535                                  NFTA_SET_ELEM_PAD))
4536                         goto nla_put_failure;
4537         }
4538
4539         if (nft_set_ext_exists(ext, NFT_SET_EXT_USERDATA)) {
4540                 struct nft_userdata *udata;
4541
4542                 udata = nft_set_ext_userdata(ext);
4543                 if (nla_put(skb, NFTA_SET_ELEM_USERDATA,
4544                             udata->len + 1, udata->data))
4545                         goto nla_put_failure;
4546         }
4547
4548         nla_nest_end(skb, nest);
4549         return 0;
4550
4551 nla_put_failure:
4552         nlmsg_trim(skb, b);
4553         return -EMSGSIZE;
4554 }
4555
4556 struct nft_set_dump_args {
4557         const struct netlink_callback   *cb;
4558         struct nft_set_iter             iter;
4559         struct sk_buff                  *skb;
4560 };
4561
4562 static int nf_tables_dump_setelem(const struct nft_ctx *ctx,
4563                                   struct nft_set *set,
4564                                   const struct nft_set_iter *iter,
4565                                   struct nft_set_elem *elem)
4566 {
4567         struct nft_set_dump_args *args;
4568
4569         args = container_of(iter, struct nft_set_dump_args, iter);
4570         return nf_tables_fill_setelem(args->skb, set, elem);
4571 }
4572
4573 struct nft_set_dump_ctx {
4574         const struct nft_set    *set;
4575         struct nft_ctx          ctx;
4576 };
4577
4578 static int nf_tables_dump_set(struct sk_buff *skb, struct netlink_callback *cb)
4579 {
4580         struct nft_set_dump_ctx *dump_ctx = cb->data;
4581         struct net *net = sock_net(skb->sk);
4582         struct nft_table *table;
4583         struct nft_set *set;
4584         struct nft_set_dump_args args;
4585         bool set_found = false;
4586         struct nfgenmsg *nfmsg;
4587         struct nlmsghdr *nlh;
4588         struct nlattr *nest;
4589         u32 portid, seq;
4590         int event;
4591
4592         rcu_read_lock();
4593         list_for_each_entry_rcu(table, &net->nft.tables, list) {
4594                 if (dump_ctx->ctx.family != NFPROTO_UNSPEC &&
4595                     dump_ctx->ctx.family != table->family)
4596                         continue;
4597
4598                 if (table != dump_ctx->ctx.table)
4599                         continue;
4600
4601                 list_for_each_entry_rcu(set, &table->sets, list) {
4602                         if (set == dump_ctx->set) {
4603                                 set_found = true;
4604                                 break;
4605                         }
4606                 }
4607                 break;
4608         }
4609
4610         if (!set_found) {
4611                 rcu_read_unlock();
4612                 return -ENOENT;
4613         }
4614
4615         event  = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWSETELEM);
4616         portid = NETLINK_CB(cb->skb).portid;
4617         seq    = cb->nlh->nlmsg_seq;
4618
4619         nlh = nlmsg_put(skb, portid, seq, event, sizeof(struct nfgenmsg),
4620                         NLM_F_MULTI);
4621         if (nlh == NULL)
4622                 goto nla_put_failure;
4623
4624         nfmsg = nlmsg_data(nlh);
4625         nfmsg->nfgen_family = table->family;
4626         nfmsg->version      = NFNETLINK_V0;
4627         nfmsg->res_id       = htons(net->nft.base_seq & 0xffff);
4628
4629         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_TABLE, table->name))
4630                 goto nla_put_failure;
4631         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_SET, set->name))
4632                 goto nla_put_failure;
4633
4634         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
4635         if (nest == NULL)
4636                 goto nla_put_failure;
4637
4638         args.cb                 = cb;
4639         args.skb                = skb;
4640         args.iter.genmask       = nft_genmask_cur(net);
4641         args.iter.skip          = cb->args[0];
4642         args.iter.count         = 0;
4643         args.iter.err           = 0;
4644         args.iter.fn            = nf_tables_dump_setelem;
4645         set->ops->walk(&dump_ctx->ctx, set, &args.iter);
4646         rcu_read_unlock();
4647
4648         nla_nest_end(skb, nest);
4649         nlmsg_end(skb, nlh);
4650
4651         if (args.iter.err && args.iter.err != -EMSGSIZE)
4652                 return args.iter.err;
4653         if (args.iter.count == cb->args[0])
4654                 return 0;
4655
4656         cb->args[0] = args.iter.count;
4657         return skb->len;
4658
4659 nla_put_failure:
4660         rcu_read_unlock();
4661         return -ENOSPC;
4662 }
4663
4664 static int nf_tables_dump_set_start(struct netlink_callback *cb)
4665 {
4666         struct nft_set_dump_ctx *dump_ctx = cb->data;
4667
4668         cb->data = kmemdup(dump_ctx, sizeof(*dump_ctx), GFP_ATOMIC);
4669
4670         return cb->data ? 0 : -ENOMEM;
4671 }
4672
4673 static int nf_tables_dump_set_done(struct netlink_callback *cb)
4674 {
4675         kfree(cb->data);
4676         return 0;
4677 }
4678
4679 static int nf_tables_fill_setelem_info(struct sk_buff *skb,
4680                                        const struct nft_ctx *ctx, u32 seq,
4681                                        u32 portid, int event, u16 flags,
4682                                        const struct nft_set *set,
4683                                        const struct nft_set_elem *elem)
4684 {
4685         struct nfgenmsg *nfmsg;
4686         struct nlmsghdr *nlh;
4687         struct nlattr *nest;
4688         int err;
4689
4690         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
4691         nlh = nlmsg_put(skb, portid, seq, event, sizeof(struct nfgenmsg),
4692                         flags);
4693         if (nlh == NULL)
4694                 goto nla_put_failure;
4695
4696         nfmsg = nlmsg_data(nlh);
4697         nfmsg->nfgen_family     = ctx->family;
4698         nfmsg->version          = NFNETLINK_V0;
4699         nfmsg->res_id           = htons(ctx->net->nft.base_seq & 0xffff);
4700
4701         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
4702                 goto nla_put_failure;
4703         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
4704                 goto nla_put_failure;
4705
4706         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
4707         if (nest == NULL)
4708                 goto nla_put_failure;
4709
4710         err = nf_tables_fill_setelem(skb, set, elem);
4711         if (err < 0)
4712                 goto nla_put_failure;
4713
4714         nla_nest_end(skb, nest);
4715
4716         nlmsg_end(skb, nlh);
4717         return 0;
4718
4719 nla_put_failure:
4720         nlmsg_trim(skb, nlh);
4721         return -1;
4722 }
4723
4724 static int nft_setelem_parse_flags(const struct nft_set *set,
4725                                    const struct nlattr *attr, u32 *flags)
4726 {
4727         if (attr == NULL)
4728                 return 0;
4729
4730         *flags = ntohl(nla_get_be32(attr));
4731         if (*flags & ~NFT_SET_ELEM_INTERVAL_END)
4732                 return -EINVAL;
4733         if (!(set->flags & NFT_SET_INTERVAL) &&
4734             *flags & NFT_SET_ELEM_INTERVAL_END)
4735                 return -EINVAL;
4736
4737         return 0;
4738 }
4739
4740 static int nft_setelem_parse_key(struct nft_ctx *ctx, struct nft_set *set,
4741                                  struct nft_data *key, struct nlattr *attr)
4742 {
4743         struct nft_data_desc desc;
4744         int err;
4745
4746         err = nft_data_init(ctx, key, NFT_DATA_VALUE_MAXLEN, &desc, attr);
4747         if (err < 0)
4748                 return err;
4749
4750         if (desc.type != NFT_DATA_VALUE || desc.len != set->klen) {
4751                 nft_data_release(key, desc.type);
4752                 return -EINVAL;
4753         }
4754
4755         return 0;
4756 }
4757
4758 static int nft_setelem_parse_data(struct nft_ctx *ctx, struct nft_set *set,
4759                                   struct nft_data_desc *desc,
4760                                   struct nft_data *data,
4761                                   struct nlattr *attr)
4762 {
4763         int err;
4764
4765         err = nft_data_init(ctx, data, NFT_DATA_VALUE_MAXLEN, desc, attr);
4766         if (err < 0)
4767                 return err;
4768
4769         if (desc->type != NFT_DATA_VERDICT && desc->len != set->dlen) {
4770                 nft_data_release(data, desc->type);
4771                 return -EINVAL;
4772         }
4773
4774         return 0;
4775 }
4776
4777 static int nft_get_set_elem(struct nft_ctx *ctx, struct nft_set *set,
4778                             const struct nlattr *attr)
4779 {
4780         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
4781         struct nft_set_elem elem;
4782         struct sk_buff *skb;
4783         uint32_t flags = 0;
4784         void *priv;
4785         int err;
4786
4787         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
4788                                           nft_set_elem_policy, NULL);
4789         if (err < 0)
4790                 return err;
4791
4792         if (!nla[NFTA_SET_ELEM_KEY])
4793                 return -EINVAL;
4794
4795         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
4796         if (err < 0)
4797                 return err;
4798
4799         err = nft_setelem_parse_key(ctx, set, &elem.key.val,
4800                                     nla[NFTA_SET_ELEM_KEY]);
4801         if (err < 0)
4802                 return err;
4803
4804         if (nla[NFTA_SET_ELEM_KEY_END]) {
4805                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
4806                                             nla[NFTA_SET_ELEM_KEY_END]);
4807                 if (err < 0)
4808                         return err;
4809         }
4810
4811         priv = set->ops->get(ctx->net, set, &elem, flags);
4812         if (IS_ERR(priv))
4813                 return PTR_ERR(priv);
4814
4815         elem.priv = priv;
4816
4817         err = -ENOMEM;
4818         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_ATOMIC);
4819         if (skb == NULL)
4820                 goto err1;
4821
4822         err = nf_tables_fill_setelem_info(skb, ctx, ctx->seq, ctx->portid,
4823                                           NFT_MSG_NEWSETELEM, 0, set, &elem);
4824         if (err < 0)
4825                 goto err2;
4826
4827         err = nfnetlink_unicast(skb, ctx->net, ctx->portid, MSG_DONTWAIT);
4828         /* This avoids a loop in nfnetlink. */
4829         if (err < 0)
4830                 goto err1;
4831
4832         return 0;
4833 err2:
4834         kfree_skb(skb);
4835 err1:
4836         /* this avoids a loop in nfnetlink. */
4837         return err == -EAGAIN ? -ENOBUFS : err;
4838 }
4839
4840 /* called with rcu_read_lock held */
4841 static int nf_tables_getsetelem(struct net *net, struct sock *nlsk,
4842                                 struct sk_buff *skb, const struct nlmsghdr *nlh,
4843                                 const struct nlattr * const nla[],
4844                                 struct netlink_ext_ack *extack)
4845 {
4846         u8 genmask = nft_genmask_cur(net);
4847         struct nft_set *set;
4848         struct nlattr *attr;
4849         struct nft_ctx ctx;
4850         int rem, err = 0;
4851
4852         err = nft_ctx_init_from_elemattr(&ctx, net, skb, nlh, nla, extack,
4853                                          genmask);
4854         if (err < 0)
4855                 return err;
4856
4857         set = nft_set_lookup(ctx.table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
4858         if (IS_ERR(set))
4859                 return PTR_ERR(set);
4860
4861         if (nlh->nlmsg_flags & NLM_F_DUMP) {
4862                 struct netlink_dump_control c = {
4863                         .start = nf_tables_dump_set_start,
4864                         .dump = nf_tables_dump_set,
4865                         .done = nf_tables_dump_set_done,
4866                         .module = THIS_MODULE,
4867                 };
4868                 struct nft_set_dump_ctx dump_ctx = {
4869                         .set = set,
4870                         .ctx = ctx,
4871                 };
4872
4873                 c.data = &dump_ctx;
4874                 return nft_netlink_dump_start_rcu(nlsk, skb, nlh, &c);
4875         }
4876
4877         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
4878                 return -EINVAL;
4879
4880         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
4881                 err = nft_get_set_elem(&ctx, set, attr);
4882                 if (err < 0)
4883                         break;
4884         }
4885
4886         return err;
4887 }
4888
4889 static void nf_tables_setelem_notify(const struct nft_ctx *ctx,
4890                                      const struct nft_set *set,
4891                                      const struct nft_set_elem *elem,
4892                                      int event, u16 flags)
4893 {
4894         struct net *net = ctx->net;
4895         u32 portid = ctx->portid;
4896         struct sk_buff *skb;
4897         int err;
4898
4899         if (!ctx->report && !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
4900                 return;
4901
4902         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
4903         if (skb == NULL)
4904                 goto err;
4905
4906         err = nf_tables_fill_setelem_info(skb, ctx, 0, portid, event, flags,
4907                                           set, elem);
4908         if (err < 0) {
4909                 kfree_skb(skb);
4910                 goto err;
4911         }
4912
4913         nfnetlink_send(skb, net, portid, NFNLGRP_NFTABLES, ctx->report,
4914                        GFP_KERNEL);
4915         return;
4916 err:
4917         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
4918 }
4919
4920 static struct nft_trans *nft_trans_elem_alloc(struct nft_ctx *ctx,
4921                                               int msg_type,
4922                                               struct nft_set *set)
4923 {
4924         struct nft_trans *trans;
4925
4926         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_elem));
4927         if (trans == NULL)
4928                 return NULL;
4929
4930         nft_trans_elem_set(trans) = set;
4931         return trans;
4932 }
4933
4934 struct nft_expr *nft_set_elem_expr_alloc(const struct nft_ctx *ctx,
4935                                          const struct nft_set *set,
4936                                          const struct nlattr *attr)
4937 {
4938         struct nft_expr *expr;
4939         int err;
4940
4941         expr = nft_expr_init(ctx, attr);
4942         if (IS_ERR(expr))
4943                 return expr;
4944
4945         err = -EOPNOTSUPP;
4946         if (!(expr->ops->type->flags & NFT_EXPR_STATEFUL))
4947                 goto err_set_elem_expr;
4948
4949         if (expr->ops->type->flags & NFT_EXPR_GC) {
4950                 if (set->flags & NFT_SET_TIMEOUT)
4951                         goto err_set_elem_expr;
4952                 if (!set->ops->gc_init)
4953                         goto err_set_elem_expr;
4954                 set->ops->gc_init(set);
4955         }
4956
4957         return expr;
4958
4959 err_set_elem_expr:
4960         nft_expr_destroy(ctx, expr);
4961         return ERR_PTR(err);
4962 }
4963
4964 void *nft_set_elem_init(const struct nft_set *set,
4965                         const struct nft_set_ext_tmpl *tmpl,
4966                         const u32 *key, const u32 *key_end,
4967                         const u32 *data, u64 timeout, u64 expiration, gfp_t gfp)
4968 {
4969         struct nft_set_ext *ext;
4970         void *elem;
4971
4972         elem = kzalloc(set->ops->elemsize + tmpl->len, gfp);
4973         if (elem == NULL)
4974                 return NULL;
4975
4976         ext = nft_set_elem_ext(set, elem);
4977         nft_set_ext_init(ext, tmpl);
4978
4979         memcpy(nft_set_ext_key(ext), key, set->klen);
4980         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END))
4981                 memcpy(nft_set_ext_key_end(ext), key_end, set->klen);
4982         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
4983                 memcpy(nft_set_ext_data(ext), data, set->dlen);
4984         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
4985                 *nft_set_ext_expiration(ext) = get_jiffies_64() + expiration;
4986                 if (expiration == 0)
4987                         *nft_set_ext_expiration(ext) += timeout;
4988         }
4989         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT))
4990                 *nft_set_ext_timeout(ext) = timeout;
4991
4992         return elem;
4993 }
4994
4995 static void nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
4996                                       struct nft_expr *expr)
4997 {
4998         if (expr->ops->destroy_clone) {
4999                 expr->ops->destroy_clone(ctx, expr);
5000                 module_put(expr->ops->type->owner);
5001         } else {
5002                 nf_tables_expr_destroy(ctx, expr);
5003         }
5004 }
5005
5006 void nft_set_elem_destroy(const struct nft_set *set, void *elem,
5007                           bool destroy_expr)
5008 {
5009         struct nft_set_ext *ext = nft_set_elem_ext(set, elem);
5010         struct nft_ctx ctx = {
5011                 .net    = read_pnet(&set->net),
5012                 .family = set->table->family,
5013         };
5014
5015         nft_data_release(nft_set_ext_key(ext), NFT_DATA_VALUE);
5016         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
5017                 nft_data_release(nft_set_ext_data(ext), set->dtype);
5018         if (destroy_expr && nft_set_ext_exists(ext, NFT_SET_EXT_EXPR))
5019                 nft_set_elem_expr_destroy(&ctx, nft_set_ext_expr(ext));
5020
5021         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
5022                 (*nft_set_ext_obj(ext))->use--;
5023         kfree(elem);
5024 }
5025 EXPORT_SYMBOL_GPL(nft_set_elem_destroy);
5026
5027 /* Only called from commit path, nft_set_elem_deactivate() already deals with
5028  * the refcounting from the preparation phase.
5029  */
5030 static void nf_tables_set_elem_destroy(const struct nft_ctx *ctx,
5031                                        const struct nft_set *set, void *elem)
5032 {
5033         struct nft_set_ext *ext = nft_set_elem_ext(set, elem);
5034
5035         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPR))
5036                 nft_set_elem_expr_destroy(ctx, nft_set_ext_expr(ext));
5037
5038         kfree(elem);
5039 }
5040
5041 static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set,
5042                             const struct nlattr *attr, u32 nlmsg_flags)
5043 {
5044         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
5045         u8 genmask = nft_genmask_next(ctx->net);
5046         struct nft_set_ext_tmpl tmpl;
5047         struct nft_set_ext *ext, *ext2;
5048         struct nft_set_elem elem;
5049         struct nft_set_binding *binding;
5050         struct nft_object *obj = NULL;
5051         struct nft_expr *expr = NULL;
5052         struct nft_userdata *udata;
5053         struct nft_data_desc desc;
5054         enum nft_registers dreg;
5055         struct nft_trans *trans;
5056         u32 flags = 0;
5057         u64 timeout;
5058         u64 expiration;
5059         u8 ulen;
5060         int err;
5061
5062         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
5063                                           nft_set_elem_policy, NULL);
5064         if (err < 0)
5065                 return err;
5066
5067         if (nla[NFTA_SET_ELEM_KEY] == NULL)
5068                 return -EINVAL;
5069
5070         nft_set_ext_prepare(&tmpl);
5071
5072         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
5073         if (err < 0)
5074                 return err;
5075         if (flags != 0)
5076                 nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
5077
5078         if (set->flags & NFT_SET_MAP) {
5079                 if (nla[NFTA_SET_ELEM_DATA] == NULL &&
5080                     !(flags & NFT_SET_ELEM_INTERVAL_END))
5081                         return -EINVAL;
5082         } else {
5083                 if (nla[NFTA_SET_ELEM_DATA] != NULL)
5084                         return -EINVAL;
5085         }
5086
5087         if ((flags & NFT_SET_ELEM_INTERVAL_END) &&
5088              (nla[NFTA_SET_ELEM_DATA] ||
5089               nla[NFTA_SET_ELEM_OBJREF] ||
5090               nla[NFTA_SET_ELEM_TIMEOUT] ||
5091               nla[NFTA_SET_ELEM_EXPIRATION] ||
5092               nla[NFTA_SET_ELEM_USERDATA] ||
5093               nla[NFTA_SET_ELEM_EXPR]))
5094                 return -EINVAL;
5095
5096         timeout = 0;
5097         if (nla[NFTA_SET_ELEM_TIMEOUT] != NULL) {
5098                 if (!(set->flags & NFT_SET_TIMEOUT))
5099                         return -EINVAL;
5100                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_TIMEOUT],
5101                                             &timeout);
5102                 if (err)
5103                         return err;
5104         } else if (set->flags & NFT_SET_TIMEOUT) {
5105                 timeout = set->timeout;
5106         }
5107
5108         expiration = 0;
5109         if (nla[NFTA_SET_ELEM_EXPIRATION] != NULL) {
5110                 if (!(set->flags & NFT_SET_TIMEOUT))
5111                         return -EINVAL;
5112                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_EXPIRATION],
5113                                             &expiration);
5114                 if (err)
5115                         return err;
5116         }
5117
5118         if (nla[NFTA_SET_ELEM_EXPR] != NULL) {
5119                 expr = nft_set_elem_expr_alloc(ctx, set,
5120                                                nla[NFTA_SET_ELEM_EXPR]);
5121                 if (IS_ERR(expr))
5122                         return PTR_ERR(expr);
5123
5124                 err = -EOPNOTSUPP;
5125                 if (set->expr && set->expr->ops != expr->ops)
5126                         goto err_set_elem_expr;
5127         } else if (set->expr) {
5128                 expr = kzalloc(set->expr->ops->size, GFP_KERNEL);
5129                 if (!expr)
5130                         return -ENOMEM;
5131
5132                 err = nft_expr_clone(expr, set->expr);
5133                 if (err < 0)
5134                         goto err_set_elem_expr;
5135         }
5136
5137         err = nft_setelem_parse_key(ctx, set, &elem.key.val,
5138                                     nla[NFTA_SET_ELEM_KEY]);
5139         if (err < 0)
5140                 goto err_set_elem_expr;
5141
5142         nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
5143
5144         if (nla[NFTA_SET_ELEM_KEY_END]) {
5145                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
5146                                             nla[NFTA_SET_ELEM_KEY_END]);
5147                 if (err < 0)
5148                         goto err_parse_key;
5149
5150                 nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
5151         }
5152
5153         if (timeout > 0) {
5154                 nft_set_ext_add(&tmpl, NFT_SET_EXT_EXPIRATION);
5155                 if (timeout != set->timeout)
5156                         nft_set_ext_add(&tmpl, NFT_SET_EXT_TIMEOUT);
5157         }
5158
5159         if (expr)
5160                 nft_set_ext_add_length(&tmpl, NFT_SET_EXT_EXPR,
5161                                        expr->ops->size);
5162
5163         if (nla[NFTA_SET_ELEM_OBJREF] != NULL) {
5164                 if (!(set->flags & NFT_SET_OBJECT)) {
5165                         err = -EINVAL;
5166                         goto err_parse_key_end;
5167                 }
5168                 obj = nft_obj_lookup(ctx->net, ctx->table,
5169                                      nla[NFTA_SET_ELEM_OBJREF],
5170                                      set->objtype, genmask);
5171                 if (IS_ERR(obj)) {
5172                         err = PTR_ERR(obj);
5173                         goto err_parse_key_end;
5174                 }
5175                 nft_set_ext_add(&tmpl, NFT_SET_EXT_OBJREF);
5176         }
5177
5178         if (nla[NFTA_SET_ELEM_DATA] != NULL) {
5179                 err = nft_setelem_parse_data(ctx, set, &desc, &elem.data.val,
5180                                              nla[NFTA_SET_ELEM_DATA]);
5181                 if (err < 0)
5182                         goto err_parse_key_end;
5183
5184                 dreg = nft_type_to_reg(set->dtype);
5185                 list_for_each_entry(binding, &set->bindings, list) {
5186                         struct nft_ctx bind_ctx = {
5187                                 .net    = ctx->net,
5188                                 .family = ctx->family,
5189                                 .table  = ctx->table,
5190                                 .chain  = (struct nft_chain *)binding->chain,
5191                         };
5192
5193                         if (!(binding->flags & NFT_SET_MAP))
5194                                 continue;
5195
5196                         err = nft_validate_register_store(&bind_ctx, dreg,
5197                                                           &elem.data.val,
5198                                                           desc.type, desc.len);
5199                         if (err < 0)
5200                                 goto err_parse_data;
5201
5202                         if (desc.type == NFT_DATA_VERDICT &&
5203                             (elem.data.val.verdict.code == NFT_GOTO ||
5204                              elem.data.val.verdict.code == NFT_JUMP))
5205                                 nft_validate_state_update(ctx->net,
5206                                                           NFT_VALIDATE_NEED);
5207                 }
5208
5209                 nft_set_ext_add_length(&tmpl, NFT_SET_EXT_DATA, desc.len);
5210         }
5211
5212         /* The full maximum length of userdata can exceed the maximum
5213          * offset value (U8_MAX) for following extensions, therefor it
5214          * must be the last extension added.
5215          */
5216         ulen = 0;
5217         if (nla[NFTA_SET_ELEM_USERDATA] != NULL) {
5218                 ulen = nla_len(nla[NFTA_SET_ELEM_USERDATA]);
5219                 if (ulen > 0)
5220                         nft_set_ext_add_length(&tmpl, NFT_SET_EXT_USERDATA,
5221                                                ulen);
5222         }
5223
5224         err = -ENOMEM;
5225         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
5226                                       elem.key_end.val.data, elem.data.val.data,
5227                                       timeout, expiration, GFP_KERNEL);
5228         if (elem.priv == NULL)
5229                 goto err_parse_data;
5230
5231         ext = nft_set_elem_ext(set, elem.priv);
5232         if (flags)
5233                 *nft_set_ext_flags(ext) = flags;
5234         if (ulen > 0) {
5235                 udata = nft_set_ext_userdata(ext);
5236                 udata->len = ulen - 1;
5237                 nla_memcpy(&udata->data, nla[NFTA_SET_ELEM_USERDATA], ulen);
5238         }
5239         if (obj) {
5240                 *nft_set_ext_obj(ext) = obj;
5241                 obj->use++;
5242         }
5243         if (expr) {
5244                 memcpy(nft_set_ext_expr(ext), expr, expr->ops->size);
5245                 kfree(expr);
5246                 expr = NULL;
5247         }
5248
5249         trans = nft_trans_elem_alloc(ctx, NFT_MSG_NEWSETELEM, set);
5250         if (trans == NULL)
5251                 goto err_trans;
5252
5253         ext->genmask = nft_genmask_cur(ctx->net) | NFT_SET_ELEM_BUSY_MASK;
5254         err = set->ops->insert(ctx->net, set, &elem, &ext2);
5255         if (err) {
5256                 if (err == -EEXIST) {
5257                         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) ^
5258                             nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) ||
5259                             nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) ^
5260                             nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF)) {
5261                                 err = -EBUSY;
5262                                 goto err_element_clash;
5263                         }
5264                         if ((nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
5265                              nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) &&
5266                              memcmp(nft_set_ext_data(ext),
5267                                     nft_set_ext_data(ext2), set->dlen) != 0) ||
5268                             (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
5269                              nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF) &&
5270                              *nft_set_ext_obj(ext) != *nft_set_ext_obj(ext2)))
5271                                 err = -EBUSY;
5272                         else if (!(nlmsg_flags & NLM_F_EXCL))
5273                                 err = 0;
5274                 } else if (err == -ENOTEMPTY) {
5275                         /* ENOTEMPTY reports overlapping between this element
5276                          * and an existing one.
5277                          */
5278                         err = -EEXIST;
5279                 }
5280                 goto err_element_clash;
5281         }
5282
5283         if (set->size &&
5284             !atomic_add_unless(&set->nelems, 1, set->size + set->ndeact)) {
5285                 err = -ENFILE;
5286                 goto err_set_full;
5287         }
5288
5289         nft_trans_elem(trans) = elem;
5290         list_add_tail(&trans->list, &ctx->net->nft.commit_list);
5291         return 0;
5292
5293 err_set_full:
5294         set->ops->remove(ctx->net, set, &elem);
5295 err_element_clash:
5296         kfree(trans);
5297 err_trans:
5298         if (obj)
5299                 obj->use--;
5300
5301         nf_tables_set_elem_destroy(ctx, set, elem.priv);
5302 err_parse_data:
5303         if (nla[NFTA_SET_ELEM_DATA] != NULL)
5304                 nft_data_release(&elem.data.val, desc.type);
5305 err_parse_key_end:
5306         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
5307 err_parse_key:
5308         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
5309 err_set_elem_expr:
5310         if (expr != NULL)
5311                 nft_expr_destroy(ctx, expr);
5312
5313         return err;
5314 }
5315
5316 static int nf_tables_newsetelem(struct net *net, struct sock *nlsk,
5317                                 struct sk_buff *skb, const struct nlmsghdr *nlh,
5318                                 const struct nlattr * const nla[],
5319                                 struct netlink_ext_ack *extack)
5320 {
5321         u8 genmask = nft_genmask_next(net);
5322         const struct nlattr *attr;
5323         struct nft_set *set;
5324         struct nft_ctx ctx;
5325         int rem, err;
5326
5327         if (nla[NFTA_SET_ELEM_LIST_ELEMENTS] == NULL)
5328                 return -EINVAL;
5329
5330         err = nft_ctx_init_from_elemattr(&ctx, net, skb, nlh, nla, extack,
5331                                          genmask);
5332         if (err < 0)
5333                 return err;
5334
5335         set = nft_set_lookup_global(net, ctx.table, nla[NFTA_SET_ELEM_LIST_SET],
5336                                     nla[NFTA_SET_ELEM_LIST_SET_ID], genmask);
5337         if (IS_ERR(set))
5338                 return PTR_ERR(set);
5339
5340         if (!list_empty(&set->bindings) && set->flags & NFT_SET_CONSTANT)
5341                 return -EBUSY;
5342
5343         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
5344                 err = nft_add_set_elem(&ctx, set, attr, nlh->nlmsg_flags);
5345                 if (err < 0)
5346                         return err;
5347         }
5348
5349         if (net->nft.validate_state == NFT_VALIDATE_DO)
5350                 return nft_table_validate(net, ctx.table);
5351
5352         return 0;
5353 }
5354
5355 /**
5356  *      nft_data_hold - hold a nft_data item
5357  *
5358  *      @data: struct nft_data to release
5359  *      @type: type of data
5360  *
5361  *      Hold a nft_data item. NFT_DATA_VALUE types can be silently discarded,
5362  *      NFT_DATA_VERDICT bumps the reference to chains in case of NFT_JUMP and
5363  *      NFT_GOTO verdicts. This function must be called on active data objects
5364  *      from the second phase of the commit protocol.
5365  */
5366 void nft_data_hold(const struct nft_data *data, enum nft_data_types type)
5367 {
5368         struct nft_chain *chain;
5369         struct nft_rule *rule;
5370
5371         if (type == NFT_DATA_VERDICT) {
5372                 switch (data->verdict.code) {
5373                 case NFT_JUMP:
5374                 case NFT_GOTO:
5375                         chain = data->verdict.chain;
5376                         chain->use++;
5377
5378                         if (!nft_chain_is_bound(chain))
5379                                 break;
5380
5381                         chain->table->use++;
5382                         list_for_each_entry(rule, &chain->rules, list)
5383                                 chain->use++;
5384
5385                         nft_chain_add(chain->table, chain);
5386                         break;
5387                 }
5388         }
5389 }
5390
5391 static void nft_set_elem_activate(const struct net *net,
5392                                   const struct nft_set *set,
5393                                   struct nft_set_elem *elem)
5394 {
5395         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
5396
5397         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
5398                 nft_data_hold(nft_set_ext_data(ext), set->dtype);
5399         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
5400                 (*nft_set_ext_obj(ext))->use++;
5401 }
5402
5403 static void nft_set_elem_deactivate(const struct net *net,
5404                                     const struct nft_set *set,
5405                                     struct nft_set_elem *elem)
5406 {
5407         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
5408
5409         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
5410                 nft_data_release(nft_set_ext_data(ext), set->dtype);
5411         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
5412                 (*nft_set_ext_obj(ext))->use--;
5413 }
5414
5415 static int nft_del_setelem(struct nft_ctx *ctx, struct nft_set *set,
5416                            const struct nlattr *attr)
5417 {
5418         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
5419         struct nft_set_ext_tmpl tmpl;
5420         struct nft_set_elem elem;
5421         struct nft_set_ext *ext;
5422         struct nft_trans *trans;
5423         u32 flags = 0;
5424         void *priv;
5425         int err;
5426
5427         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
5428                                           nft_set_elem_policy, NULL);
5429         if (err < 0)
5430                 return err;
5431
5432         if (nla[NFTA_SET_ELEM_KEY] == NULL)
5433                 return -EINVAL;
5434
5435         nft_set_ext_prepare(&tmpl);
5436
5437         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
5438         if (err < 0)
5439                 return err;
5440         if (flags != 0)
5441                 nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
5442
5443         err = nft_setelem_parse_key(ctx, set, &elem.key.val,
5444                                     nla[NFTA_SET_ELEM_KEY]);
5445         if (err < 0)
5446                 return err;
5447
5448         nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
5449
5450         if (nla[NFTA_SET_ELEM_KEY_END]) {
5451                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
5452                                             nla[NFTA_SET_ELEM_KEY_END]);
5453                 if (err < 0)
5454                         return err;
5455
5456                 nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
5457         }
5458
5459         err = -ENOMEM;
5460         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
5461                                       elem.key_end.val.data, NULL, 0, 0,
5462                                       GFP_KERNEL);
5463         if (elem.priv == NULL)
5464                 goto fail_elem;
5465
5466         ext = nft_set_elem_ext(set, elem.priv);
5467         if (flags)
5468                 *nft_set_ext_flags(ext) = flags;
5469
5470         trans = nft_trans_elem_alloc(ctx, NFT_MSG_DELSETELEM, set);
5471         if (trans == NULL)
5472                 goto fail_trans;
5473
5474         priv = set->ops->deactivate(ctx->net, set, &elem);
5475         if (priv == NULL) {
5476                 err = -ENOENT;
5477                 goto fail_ops;
5478         }
5479         kfree(elem.priv);
5480         elem.priv = priv;
5481
5482         nft_set_elem_deactivate(ctx->net, set, &elem);
5483
5484         nft_trans_elem(trans) = elem;
5485         list_add_tail(&trans->list, &ctx->net->nft.commit_list);
5486         return 0;
5487
5488 fail_ops:
5489         kfree(trans);
5490 fail_trans:
5491         kfree(elem.priv);
5492 fail_elem:
5493         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
5494         return err;
5495 }
5496
5497 static int nft_flush_set(const struct nft_ctx *ctx,
5498                          struct nft_set *set,
5499                          const struct nft_set_iter *iter,
5500                          struct nft_set_elem *elem)
5501 {
5502         struct nft_trans *trans;
5503         int err;
5504
5505         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
5506                                     sizeof(struct nft_trans_elem), GFP_ATOMIC);
5507         if (!trans)
5508                 return -ENOMEM;
5509
5510         if (!set->ops->flush(ctx->net, set, elem->priv)) {
5511                 err = -ENOENT;
5512                 goto err1;
5513         }
5514         set->ndeact++;
5515
5516         nft_set_elem_deactivate(ctx->net, set, elem);
5517         nft_trans_elem_set(trans) = set;
5518         nft_trans_elem(trans) = *elem;
5519         list_add_tail(&trans->list, &ctx->net->nft.commit_list);
5520
5521         return 0;
5522 err1:
5523         kfree(trans);
5524         return err;
5525 }
5526
5527 static int nf_tables_delsetelem(struct net *net, struct sock *nlsk,
5528                                 struct sk_buff *skb, const struct nlmsghdr *nlh,
5529                                 const struct nlattr * const nla[],
5530                                 struct netlink_ext_ack *extack)
5531 {
5532         u8 genmask = nft_genmask_next(net);
5533         const struct nlattr *attr;
5534         struct nft_set *set;
5535         struct nft_ctx ctx;
5536         int rem, err = 0;
5537
5538         err = nft_ctx_init_from_elemattr(&ctx, net, skb, nlh, nla, extack,
5539                                          genmask);
5540         if (err < 0)
5541                 return err;
5542
5543         set = nft_set_lookup(ctx.table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
5544         if (IS_ERR(set))
5545                 return PTR_ERR(set);
5546         if (!list_empty(&set->bindings) && set->flags & NFT_SET_CONSTANT)
5547                 return -EBUSY;
5548
5549         if (nla[NFTA_SET_ELEM_LIST_ELEMENTS] == NULL) {
5550                 struct nft_set_iter iter = {
5551                         .genmask        = genmask,
5552                         .fn             = nft_flush_set,
5553                 };
5554                 set->ops->walk(&ctx, set, &iter);
5555
5556                 return iter.err;
5557         }
5558
5559         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
5560                 err = nft_del_setelem(&ctx, set, attr);
5561                 if (err < 0)
5562                         break;
5563
5564                 set->ndeact++;
5565         }
5566         return err;
5567 }
5568
5569 void nft_set_gc_batch_release(struct rcu_head *rcu)
5570 {
5571         struct nft_set_gc_batch *gcb;
5572         unsigned int i;
5573
5574         gcb = container_of(rcu, struct nft_set_gc_batch, head.rcu);
5575         for (i = 0; i < gcb->head.cnt; i++)
5576                 nft_set_elem_destroy(gcb->head.set, gcb->elems[i], true);
5577         kfree(gcb);
5578 }
5579
5580 struct nft_set_gc_batch *nft_set_gc_batch_alloc(const struct nft_set *set,
5581                                                 gfp_t gfp)
5582 {
5583         struct nft_set_gc_batch *gcb;
5584
5585         gcb = kzalloc(sizeof(*gcb), gfp);
5586         if (gcb == NULL)
5587                 return gcb;
5588         gcb->head.set = set;
5589         return gcb;
5590 }
5591
5592 /*
5593  * Stateful objects
5594  */
5595
5596 /**
5597  *      nft_register_obj- register nf_tables stateful object type
5598  *      @obj_type: object type
5599  *
5600  *      Registers the object type for use with nf_tables. Returns zero on
5601  *      success or a negative errno code otherwise.
5602  */
5603 int nft_register_obj(struct nft_object_type *obj_type)
5604 {
5605         if (obj_type->type == NFT_OBJECT_UNSPEC)
5606                 return -EINVAL;
5607
5608         nfnl_lock(NFNL_SUBSYS_NFTABLES);
5609         list_add_rcu(&obj_type->list, &nf_tables_objects);
5610         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
5611         return 0;
5612 }
5613 EXPORT_SYMBOL_GPL(nft_register_obj);
5614
5615 /**
5616  *      nft_unregister_obj - unregister nf_tables object type
5617  *      @obj_type: object type
5618  *
5619  *      Unregisters the object type for use with nf_tables.
5620  */
5621 void nft_unregister_obj(struct nft_object_type *obj_type)
5622 {
5623         nfnl_lock(NFNL_SUBSYS_NFTABLES);
5624         list_del_rcu(&obj_type->list);
5625         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
5626 }
5627 EXPORT_SYMBOL_GPL(nft_unregister_obj);
5628
5629 struct nft_object *nft_obj_lookup(const struct net *net,
5630                                   const struct nft_table *table,
5631                                   const struct nlattr *nla, u32 objtype,
5632                                   u8 genmask)
5633 {
5634         struct nft_object_hash_key k = { .table = table };
5635         char search[NFT_OBJ_MAXNAMELEN];
5636         struct rhlist_head *tmp, *list;
5637         struct nft_object *obj;
5638
5639         nla_strlcpy(search, nla, sizeof(search));
5640         k.name = search;
5641
5642         WARN_ON_ONCE(!rcu_read_lock_held() &&
5643                      !lockdep_commit_lock_is_held(net));
5644
5645         rcu_read_lock();
5646         list = rhltable_lookup(&nft_objname_ht, &k, nft_objname_ht_params);
5647         if (!list)
5648                 goto out;
5649
5650         rhl_for_each_entry_rcu(obj, tmp, list, rhlhead) {
5651                 if (objtype == obj->ops->type->type &&
5652                     nft_active_genmask(obj, genmask)) {
5653                         rcu_read_unlock();
5654                         return obj;
5655                 }
5656         }
5657 out:
5658         rcu_read_unlock();
5659         return ERR_PTR(-ENOENT);
5660 }
5661 EXPORT_SYMBOL_GPL(nft_obj_lookup);
5662
5663 static struct nft_object *nft_obj_lookup_byhandle(const struct nft_table *table,
5664                                                   const struct nlattr *nla,
5665                                                   u32 objtype, u8 genmask)
5666 {
5667         struct nft_object *obj;
5668
5669         list_for_each_entry(obj, &table->objects, list) {
5670                 if (be64_to_cpu(nla_get_be64(nla)) == obj->handle &&
5671                     objtype == obj->ops->type->type &&
5672                     nft_active_genmask(obj, genmask))
5673                         return obj;
5674         }
5675         return ERR_PTR(-ENOENT);
5676 }
5677
5678 static const struct nla_policy nft_obj_policy[NFTA_OBJ_MAX + 1] = {
5679         [NFTA_OBJ_TABLE]        = { .type = NLA_STRING,
5680                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
5681         [NFTA_OBJ_NAME]         = { .type = NLA_STRING,
5682                                     .len = NFT_OBJ_MAXNAMELEN - 1 },
5683         [NFTA_OBJ_TYPE]         = { .type = NLA_U32 },
5684         [NFTA_OBJ_DATA]         = { .type = NLA_NESTED },
5685         [NFTA_OBJ_HANDLE]       = { .type = NLA_U64},
5686 };
5687
5688 static struct nft_object *nft_obj_init(const struct nft_ctx *ctx,
5689                                        const struct nft_object_type *type,
5690                                        const struct nlattr *attr)
5691 {
5692         struct nlattr **tb;
5693         const struct nft_object_ops *ops;
5694         struct nft_object *obj;
5695         int err = -ENOMEM;
5696
5697         tb = kmalloc_array(type->maxattr + 1, sizeof(*tb), GFP_KERNEL);
5698         if (!tb)
5699                 goto err1;
5700
5701         if (attr) {
5702                 err = nla_parse_nested_deprecated(tb, type->maxattr, attr,
5703                                                   type->policy, NULL);
5704                 if (err < 0)
5705                         goto err2;
5706         } else {
5707                 memset(tb, 0, sizeof(tb[0]) * (type->maxattr + 1));
5708         }
5709
5710         if (type->select_ops) {
5711                 ops = type->select_ops(ctx, (const struct nlattr * const *)tb);
5712                 if (IS_ERR(ops)) {
5713                         err = PTR_ERR(ops);
5714                         goto err2;
5715                 }
5716         } else {
5717                 ops = type->ops;
5718         }
5719
5720         err = -ENOMEM;
5721         obj = kzalloc(sizeof(*obj) + ops->size, GFP_KERNEL);
5722         if (!obj)
5723                 goto err2;
5724
5725         err = ops->init(ctx, (const struct nlattr * const *)tb, obj);
5726         if (err < 0)
5727                 goto err3;
5728
5729         obj->ops = ops;
5730
5731         kfree(tb);
5732         return obj;
5733 err3:
5734         kfree(obj);
5735 err2:
5736         kfree(tb);
5737 err1:
5738         return ERR_PTR(err);
5739 }
5740
5741 static int nft_object_dump(struct sk_buff *skb, unsigned int attr,
5742                            struct nft_object *obj, bool reset)
5743 {
5744         struct nlattr *nest;
5745
5746         nest = nla_nest_start_noflag(skb, attr);
5747         if (!nest)
5748                 goto nla_put_failure;
5749         if (obj->ops->dump(skb, obj, reset) < 0)
5750                 goto nla_put_failure;
5751         nla_nest_end(skb, nest);
5752         return 0;
5753
5754 nla_put_failure:
5755         return -1;
5756 }
5757
5758 static const struct nft_object_type *__nft_obj_type_get(u32 objtype)
5759 {
5760         const struct nft_object_type *type;
5761
5762         list_for_each_entry(type, &nf_tables_objects, list) {
5763                 if (objtype == type->type)
5764                         return type;
5765         }
5766         return NULL;
5767 }
5768
5769 static const struct nft_object_type *
5770 nft_obj_type_get(struct net *net, u32 objtype)
5771 {
5772         const struct nft_object_type *type;
5773
5774         type = __nft_obj_type_get(objtype);
5775         if (type != NULL && try_module_get(type->owner))
5776                 return type;
5777
5778         lockdep_nfnl_nft_mutex_not_held();
5779 #ifdef CONFIG_MODULES
5780         if (type == NULL) {
5781                 if (nft_request_module(net, "nft-obj-%u", objtype) == -EAGAIN)
5782                         return ERR_PTR(-EAGAIN);
5783         }
5784 #endif
5785         return ERR_PTR(-ENOENT);
5786 }
5787
5788 static int nf_tables_updobj(const struct nft_ctx *ctx,
5789                             const struct nft_object_type *type,
5790                             const struct nlattr *attr,
5791                             struct nft_object *obj)
5792 {
5793         struct nft_object *newobj;
5794         struct nft_trans *trans;
5795         int err;
5796
5797         trans = nft_trans_alloc(ctx, NFT_MSG_NEWOBJ,
5798                                 sizeof(struct nft_trans_obj));
5799         if (!trans)
5800                 return -ENOMEM;
5801
5802         newobj = nft_obj_init(ctx, type, attr);
5803         if (IS_ERR(newobj)) {
5804                 err = PTR_ERR(newobj);
5805                 goto err_free_trans;
5806         }
5807
5808         nft_trans_obj(trans) = obj;
5809         nft_trans_obj_update(trans) = true;
5810         nft_trans_obj_newobj(trans) = newobj;
5811         list_add_tail(&trans->list, &ctx->net->nft.commit_list);
5812
5813         return 0;
5814
5815 err_free_trans:
5816         kfree(trans);
5817         return err;
5818 }
5819
5820 static int nf_tables_newobj(struct net *net, struct sock *nlsk,
5821                             struct sk_buff *skb, const struct nlmsghdr *nlh,
5822                             const struct nlattr * const nla[],
5823                             struct netlink_ext_ack *extack)
5824 {
5825         const struct nfgenmsg *nfmsg = nlmsg_data(nlh);
5826         const struct nft_object_type *type;
5827         u8 genmask = nft_genmask_next(net);
5828         int family = nfmsg->nfgen_family;
5829         struct nft_table *table;
5830         struct nft_object *obj;
5831         struct nft_ctx ctx;
5832         u32 objtype;
5833         int err;
5834
5835         if (!nla[NFTA_OBJ_TYPE] ||
5836             !nla[NFTA_OBJ_NAME] ||
5837             !nla[NFTA_OBJ_DATA])
5838                 return -EINVAL;
5839
5840         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask);
5841         if (IS_ERR(table)) {
5842                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
5843                 return PTR_ERR(table);
5844         }
5845
5846         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
5847         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
5848         if (IS_ERR(obj)) {
5849                 err = PTR_ERR(obj);
5850                 if (err != -ENOENT) {
5851                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
5852                         return err;
5853                 }
5854         } else {
5855                 if (nlh->nlmsg_flags & NLM_F_EXCL) {
5856                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
5857                         return -EEXIST;
5858                 }
5859                 if (nlh->nlmsg_flags & NLM_F_REPLACE)
5860                         return -EOPNOTSUPP;
5861
5862                 type = __nft_obj_type_get(objtype);
5863                 nft_ctx_init(&ctx, net, skb, nlh, family, table, NULL, nla);
5864
5865                 return nf_tables_updobj(&ctx, type, nla[NFTA_OBJ_DATA], obj);
5866         }
5867
5868         nft_ctx_init(&ctx, net, skb, nlh, family, table, NULL, nla);
5869
5870         type = nft_obj_type_get(net, objtype);
5871         if (IS_ERR(type))
5872                 return PTR_ERR(type);
5873
5874         obj = nft_obj_init(&ctx, type, nla[NFTA_OBJ_DATA]);
5875         if (IS_ERR(obj)) {
5876                 err = PTR_ERR(obj);
5877                 goto err1;
5878         }
5879         obj->key.table = table;
5880         obj->handle = nf_tables_alloc_handle(table);
5881
5882         obj->key.name = nla_strdup(nla[NFTA_OBJ_NAME], GFP_KERNEL);
5883         if (!obj->key.name) {
5884                 err = -ENOMEM;
5885                 goto err2;
5886         }
5887
5888         err = nft_trans_obj_add(&ctx, NFT_MSG_NEWOBJ, obj);
5889         if (err < 0)
5890                 goto err3;
5891
5892         err = rhltable_insert(&nft_objname_ht, &obj->rhlhead,
5893                               nft_objname_ht_params);
5894         if (err < 0)
5895                 goto err4;
5896
5897         list_add_tail_rcu(&obj->list, &table->objects);
5898         table->use++;
5899         return 0;
5900 err4:
5901         /* queued in transaction log */
5902         INIT_LIST_HEAD(&obj->list);
5903         return err;
5904 err3:
5905         kfree(obj->key.name);
5906 err2:
5907         if (obj->ops->destroy)
5908                 obj->ops->destroy(&ctx, obj);
5909         kfree(obj);
5910 err1:
5911         module_put(type->owner);
5912         return err;
5913 }
5914
5915 static int nf_tables_fill_obj_info(struct sk_buff *skb, struct net *net,
5916                                    u32 portid, u32 seq, int event, u32 flags,
5917                                    int family, const struct nft_table *table,
5918                                    struct nft_object *obj, bool reset)
5919 {
5920         struct nfgenmsg *nfmsg;
5921         struct nlmsghdr *nlh;
5922
5923         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
5924         nlh = nlmsg_put(skb, portid, seq, event, sizeof(struct nfgenmsg), flags);
5925         if (nlh == NULL)
5926                 goto nla_put_failure;
5927
5928         nfmsg = nlmsg_data(nlh);
5929         nfmsg->nfgen_family     = family;
5930         nfmsg->version          = NFNETLINK_V0;
5931         nfmsg->res_id           = htons(net->nft.base_seq & 0xffff);
5932
5933         if (nla_put_string(skb, NFTA_OBJ_TABLE, table->name) ||
5934             nla_put_string(skb, NFTA_OBJ_NAME, obj->key.name) ||
5935             nla_put_be32(skb, NFTA_OBJ_TYPE, htonl(obj->ops->type->type)) ||
5936             nla_put_be32(skb, NFTA_OBJ_USE, htonl(obj->use)) ||
5937             nft_object_dump(skb, NFTA_OBJ_DATA, obj, reset) ||
5938             nla_put_be64(skb, NFTA_OBJ_HANDLE, cpu_to_be64(obj->handle),
5939                          NFTA_OBJ_PAD))
5940                 goto nla_put_failure;
5941
5942         nlmsg_end(skb, nlh);
5943         return 0;
5944
5945 nla_put_failure:
5946         nlmsg_trim(skb, nlh);
5947         return -1;
5948 }
5949
5950 struct nft_obj_filter {
5951         char            *table;
5952         u32             type;
5953 };
5954
5955 static int nf_tables_dump_obj(struct sk_buff *skb, struct netlink_callback *cb)
5956 {
5957         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
5958         const struct nft_table *table;
5959         unsigned int idx = 0, s_idx = cb->args[0];
5960         struct nft_obj_filter *filter = cb->data;
5961         struct net *net = sock_net(skb->sk);
5962         int family = nfmsg->nfgen_family;
5963         struct nft_object *obj;
5964         bool reset = false;
5965
5966         if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
5967                 reset = true;
5968
5969         rcu_read_lock();
5970         cb->seq = net->nft.base_seq;
5971
5972         list_for_each_entry_rcu(table, &net->nft.tables, list) {
5973                 if (family != NFPROTO_UNSPEC && family != table->family)
5974                         continue;
5975
5976                 list_for_each_entry_rcu(obj, &table->objects, list) {
5977                         if (!nft_is_active(net, obj))
5978                                 goto cont;
5979                         if (idx < s_idx)
5980                                 goto cont;
5981                         if (idx > s_idx)
5982                                 memset(&cb->args[1], 0,
5983                                        sizeof(cb->args) - sizeof(cb->args[0]));
5984                         if (filter && filter->table &&
5985                             strcmp(filter->table, table->name))
5986                                 goto cont;
5987                         if (filter &&
5988                             filter->type != NFT_OBJECT_UNSPEC &&
5989                             obj->ops->type->type != filter->type)
5990                                 goto cont;
5991
5992                         if (nf_tables_fill_obj_info(skb, net, NETLINK_CB(cb->skb).portid,
5993                                                     cb->nlh->nlmsg_seq,
5994                                                     NFT_MSG_NEWOBJ,
5995                                                     NLM_F_MULTI | NLM_F_APPEND,
5996                                                     table->family, table,
5997                                                     obj, reset) < 0)
5998                                 goto done;
5999
6000                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
6001 cont:
6002                         idx++;
6003                 }
6004         }
6005 done:
6006         rcu_read_unlock();
6007
6008         cb->args[0] = idx;
6009         return skb->len;
6010 }
6011
6012 static int nf_tables_dump_obj_start(struct netlink_callback *cb)
6013 {
6014         const struct nlattr * const *nla = cb->data;
6015         struct nft_obj_filter *filter = NULL;
6016
6017         if (nla[NFTA_OBJ_TABLE] || nla[NFTA_OBJ_TYPE]) {
6018                 filter = kzalloc(sizeof(*filter), GFP_ATOMIC);
6019                 if (!filter)
6020                         return -ENOMEM;
6021
6022                 if (nla[NFTA_OBJ_TABLE]) {
6023                         filter->table = nla_strdup(nla[NFTA_OBJ_TABLE], GFP_ATOMIC);
6024                         if (!filter->table) {
6025                                 kfree(filter);
6026                                 return -ENOMEM;
6027                         }
6028                 }
6029
6030                 if (nla[NFTA_OBJ_TYPE])
6031                         filter->type = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
6032         }
6033
6034         cb->data = filter;
6035         return 0;
6036 }
6037
6038 static int nf_tables_dump_obj_done(struct netlink_callback *cb)
6039 {
6040         struct nft_obj_filter *filter = cb->data;
6041
6042         if (filter) {
6043                 kfree(filter->table);
6044                 kfree(filter);
6045         }
6046
6047         return 0;
6048 }
6049
6050 /* called with rcu_read_lock held */
6051 static int nf_tables_getobj(struct net *net, struct sock *nlsk,
6052                             struct sk_buff *skb, const struct nlmsghdr *nlh,
6053                             const struct nlattr * const nla[],
6054                             struct netlink_ext_ack *extack)
6055 {
6056         const struct nfgenmsg *nfmsg = nlmsg_data(nlh);
6057         u8 genmask = nft_genmask_cur(net);
6058         int family = nfmsg->nfgen_family;
6059         const struct nft_table *table;
6060         struct nft_object *obj;
6061         struct sk_buff *skb2;
6062         bool reset = false;
6063         u32 objtype;
6064         int err;
6065
6066         if (nlh->nlmsg_flags & NLM_F_DUMP) {
6067                 struct netlink_dump_control c = {
6068                         .start = nf_tables_dump_obj_start,
6069                         .dump = nf_tables_dump_obj,
6070                         .done = nf_tables_dump_obj_done,
6071                         .module = THIS_MODULE,
6072                         .data = (void *)nla,
6073                 };
6074
6075                 return nft_netlink_dump_start_rcu(nlsk, skb, nlh, &c);
6076         }
6077
6078         if (!nla[NFTA_OBJ_NAME] ||
6079             !nla[NFTA_OBJ_TYPE])
6080                 return -EINVAL;
6081
6082         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask);
6083         if (IS_ERR(table)) {
6084                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
6085                 return PTR_ERR(table);
6086         }
6087
6088         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
6089         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
6090         if (IS_ERR(obj)) {
6091                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
6092                 return PTR_ERR(obj);
6093         }
6094
6095         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
6096         if (!skb2)
6097                 return -ENOMEM;
6098
6099         if (NFNL_MSG_TYPE(nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
6100                 reset = true;
6101
6102         err = nf_tables_fill_obj_info(skb2, net, NETLINK_CB(skb).portid,
6103                                       nlh->nlmsg_seq, NFT_MSG_NEWOBJ, 0,
6104                                       family, table, obj, reset);
6105         if (err < 0)
6106                 goto err;
6107
6108         return nlmsg_unicast(nlsk, skb2, NETLINK_CB(skb).portid);
6109 err:
6110         kfree_skb(skb2);
6111         return err;
6112 }
6113
6114 static void nft_obj_destroy(const struct nft_ctx *ctx, struct nft_object *obj)
6115 {
6116         if (obj->ops->destroy)
6117                 obj->ops->destroy(ctx, obj);
6118
6119         module_put(obj->ops->type->owner);
6120         kfree(obj->key.name);
6121         kfree(obj);
6122 }
6123
6124 static int nf_tables_delobj(struct net *net, struct sock *nlsk,
6125                             struct sk_buff *skb, const struct nlmsghdr *nlh,
6126                             const struct nlattr * const nla[],
6127                             struct netlink_ext_ack *extack)
6128 {
6129         const struct nfgenmsg *nfmsg = nlmsg_data(nlh);
6130         u8 genmask = nft_genmask_next(net);
6131         int family = nfmsg->nfgen_family;
6132         const struct nlattr *attr;
6133         struct nft_table *table;
6134         struct nft_object *obj;
6135         struct nft_ctx ctx;
6136         u32 objtype;
6137
6138         if (!nla[NFTA_OBJ_TYPE] ||
6139             (!nla[NFTA_OBJ_NAME] && !nla[NFTA_OBJ_HANDLE]))
6140                 return -EINVAL;
6141
6142         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask);
6143         if (IS_ERR(table)) {
6144                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
6145                 return PTR_ERR(table);
6146         }
6147
6148         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
6149         if (nla[NFTA_OBJ_HANDLE]) {
6150                 attr = nla[NFTA_OBJ_HANDLE];
6151                 obj = nft_obj_lookup_byhandle(table, attr, objtype, genmask);
6152         } else {
6153                 attr = nla[NFTA_OBJ_NAME];
6154                 obj = nft_obj_lookup(net, table, attr, objtype, genmask);
6155         }
6156
6157         if (IS_ERR(obj)) {
6158                 NL_SET_BAD_ATTR(extack, attr);
6159                 return PTR_ERR(obj);
6160         }
6161         if (obj->use > 0) {
6162                 NL_SET_BAD_ATTR(extack, attr);
6163                 return -EBUSY;
6164         }
6165
6166         nft_ctx_init(&ctx, net, skb, nlh, family, table, NULL, nla);
6167
6168         return nft_delobj(&ctx, obj);
6169 }
6170
6171 void nft_obj_notify(struct net *net, const struct nft_table *table,
6172                     struct nft_object *obj, u32 portid, u32 seq, int event,
6173                     int family, int report, gfp_t gfp)
6174 {
6175         struct sk_buff *skb;
6176         int err;
6177
6178         if (!report &&
6179             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
6180                 return;
6181
6182         skb = nlmsg_new(NLMSG_GOODSIZE, gfp);
6183         if (skb == NULL)
6184                 goto err;
6185
6186         err = nf_tables_fill_obj_info(skb, net, portid, seq, event, 0, family,
6187                                       table, obj, false);
6188         if (err < 0) {
6189                 kfree_skb(skb);
6190                 goto err;
6191         }
6192
6193         nfnetlink_send(skb, net, portid, NFNLGRP_NFTABLES, report, gfp);
6194         return;
6195 err:
6196         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
6197 }
6198 EXPORT_SYMBOL_GPL(nft_obj_notify);
6199
6200 static void nf_tables_obj_notify(const struct nft_ctx *ctx,
6201                                  struct nft_object *obj, int event)
6202 {
6203         nft_obj_notify(ctx->net, ctx->table, obj, ctx->portid, ctx->seq, event,
6204                        ctx->family, ctx->report, GFP_KERNEL);
6205 }
6206
6207 /*
6208  * Flow tables
6209  */
6210 void nft_register_flowtable_type(struct nf_flowtable_type *type)
6211 {
6212         nfnl_lock(NFNL_SUBSYS_NFTABLES);
6213         list_add_tail_rcu(&type->list, &nf_tables_flowtables);
6214         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
6215 }
6216 EXPORT_SYMBOL_GPL(nft_register_flowtable_type);
6217
6218 void nft_unregister_flowtable_type(struct nf_flowtable_type *type)
6219 {
6220         nfnl_lock(NFNL_SUBSYS_NFTABLES);
6221         list_del_rcu(&type->list);
6222         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
6223 }
6224 EXPORT_SYMBOL_GPL(nft_unregister_flowtable_type);
6225
6226 static const struct nla_policy nft_flowtable_policy[NFTA_FLOWTABLE_MAX + 1] = {
6227         [NFTA_FLOWTABLE_TABLE]          = { .type = NLA_STRING,
6228                                             .len = NFT_NAME_MAXLEN - 1 },
6229         [NFTA_FLOWTABLE_NAME]           = { .type = NLA_STRING,
6230                                             .len = NFT_NAME_MAXLEN - 1 },
6231         [NFTA_FLOWTABLE_HOOK]           = { .type = NLA_NESTED },
6232         [NFTA_FLOWTABLE_HANDLE]         = { .type = NLA_U64 },
6233         [NFTA_FLOWTABLE_FLAGS]          = { .type = NLA_U32 },
6234 };
6235
6236 struct nft_flowtable *nft_flowtable_lookup(const struct nft_table *table,
6237                                            const struct nlattr *nla, u8 genmask)
6238 {
6239         struct nft_flowtable *flowtable;
6240
6241         list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
6242                 if (!nla_strcmp(nla, flowtable->name) &&
6243                     nft_active_genmask(flowtable, genmask))
6244                         return flowtable;
6245         }
6246         return ERR_PTR(-ENOENT);
6247 }
6248 EXPORT_SYMBOL_GPL(nft_flowtable_lookup);
6249
6250 void nf_tables_deactivate_flowtable(const struct nft_ctx *ctx,
6251                                     struct nft_flowtable *flowtable,
6252                                     enum nft_trans_phase phase)
6253 {
6254         switch (phase) {
6255         case NFT_TRANS_PREPARE:
6256         case NFT_TRANS_ABORT:
6257         case NFT_TRANS_RELEASE:
6258                 flowtable->use--;
6259                 /* fall through */
6260         default:
6261                 return;
6262         }
6263 }
6264 EXPORT_SYMBOL_GPL(nf_tables_deactivate_flowtable);
6265
6266 static struct nft_flowtable *
6267 nft_flowtable_lookup_byhandle(const struct nft_table *table,
6268                               const struct nlattr *nla, u8 genmask)
6269 {
6270        struct nft_flowtable *flowtable;
6271
6272        list_for_each_entry(flowtable, &table->flowtables, list) {
6273                if (be64_to_cpu(nla_get_be64(nla)) == flowtable->handle &&
6274                    nft_active_genmask(flowtable, genmask))
6275                        return flowtable;
6276        }
6277        return ERR_PTR(-ENOENT);
6278 }
6279
6280 struct nft_flowtable_hook {
6281         u32                     num;
6282         int                     priority;
6283         struct list_head        list;
6284 };
6285
6286 static const struct nla_policy nft_flowtable_hook_policy[NFTA_FLOWTABLE_HOOK_MAX + 1] = {
6287         [NFTA_FLOWTABLE_HOOK_NUM]       = { .type = NLA_U32 },
6288         [NFTA_FLOWTABLE_HOOK_PRIORITY]  = { .type = NLA_U32 },
6289         [NFTA_FLOWTABLE_HOOK_DEVS]      = { .type = NLA_NESTED },
6290 };
6291
6292 static int nft_flowtable_parse_hook(const struct nft_ctx *ctx,
6293                                     const struct nlattr *attr,
6294                                     struct nft_flowtable_hook *flowtable_hook,
6295                                     struct nft_flowtable *flowtable, bool add)
6296 {
6297         struct nlattr *tb[NFTA_FLOWTABLE_HOOK_MAX + 1];
6298         struct nft_hook *hook;
6299         int hooknum, priority;
6300         int err;
6301
6302         INIT_LIST_HEAD(&flowtable_hook->list);
6303
6304         err = nla_parse_nested_deprecated(tb, NFTA_FLOWTABLE_HOOK_MAX, attr,
6305                                           nft_flowtable_hook_policy, NULL);
6306         if (err < 0)
6307                 return err;
6308
6309         if (add) {
6310                 if (!tb[NFTA_FLOWTABLE_HOOK_NUM] ||
6311                     !tb[NFTA_FLOWTABLE_HOOK_PRIORITY])
6312                         return -EINVAL;
6313
6314                 hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
6315                 if (hooknum != NF_NETDEV_INGRESS)
6316                         return -EOPNOTSUPP;
6317
6318                 priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
6319
6320                 flowtable_hook->priority        = priority;
6321                 flowtable_hook->num             = hooknum;
6322         } else {
6323                 if (tb[NFTA_FLOWTABLE_HOOK_NUM]) {
6324                         hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
6325                         if (hooknum != flowtable->hooknum)
6326                                 return -EOPNOTSUPP;
6327                 }
6328
6329                 if (tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
6330                         priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
6331                         if (priority != flowtable->data.priority)
6332                                 return -EOPNOTSUPP;
6333                 }
6334
6335                 flowtable_hook->priority        = flowtable->data.priority;
6336                 flowtable_hook->num             = flowtable->hooknum;
6337         }
6338
6339         if (tb[NFTA_FLOWTABLE_HOOK_DEVS]) {
6340                 err = nf_tables_parse_netdev_hooks(ctx->net,
6341                                                    tb[NFTA_FLOWTABLE_HOOK_DEVS],
6342                                                    &flowtable_hook->list);
6343                 if (err < 0)
6344                         return err;
6345         }
6346
6347         list_for_each_entry(hook, &flowtable_hook->list, list) {
6348                 hook->ops.pf            = NFPROTO_NETDEV;
6349                 hook->ops.hooknum       = flowtable_hook->num;
6350                 hook->ops.priority      = flowtable_hook->priority;
6351                 hook->ops.priv          = &flowtable->data;
6352                 hook->ops.hook          = flowtable->data.type->hook;
6353         }
6354
6355         return err;
6356 }
6357
6358 static const struct nf_flowtable_type *__nft_flowtable_type_get(u8 family)
6359 {
6360         const struct nf_flowtable_type *type;
6361
6362         list_for_each_entry(type, &nf_tables_flowtables, list) {
6363                 if (family == type->family)
6364                         return type;
6365         }
6366         return NULL;
6367 }
6368
6369 static const struct nf_flowtable_type *
6370 nft_flowtable_type_get(struct net *net, u8 family)
6371 {
6372         const struct nf_flowtable_type *type;
6373
6374         type = __nft_flowtable_type_get(family);
6375         if (type != NULL && try_module_get(type->owner))
6376                 return type;
6377
6378         lockdep_nfnl_nft_mutex_not_held();
6379 #ifdef CONFIG_MODULES
6380         if (type == NULL) {
6381                 if (nft_request_module(net, "nf-flowtable-%u", family) == -EAGAIN)
6382                         return ERR_PTR(-EAGAIN);
6383         }
6384 #endif
6385         return ERR_PTR(-ENOENT);
6386 }
6387
6388 /* Only called from error and netdev event paths. */
6389 static void nft_unregister_flowtable_hook(struct net *net,
6390                                           struct nft_flowtable *flowtable,
6391                                           struct nft_hook *hook)
6392 {
6393         nf_unregister_net_hook(net, &hook->ops);
6394         flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
6395                                     FLOW_BLOCK_UNBIND);
6396 }
6397
6398 static void nft_unregister_flowtable_net_hooks(struct net *net,
6399                                                struct list_head *hook_list)
6400 {
6401         struct nft_hook *hook;
6402
6403         list_for_each_entry(hook, hook_list, list)
6404                 nf_unregister_net_hook(net, &hook->ops);
6405 }
6406
6407 static int nft_register_flowtable_net_hooks(struct net *net,
6408                                             struct nft_table *table,
6409                                             struct list_head *hook_list,
6410                                             struct nft_flowtable *flowtable)
6411 {
6412         struct nft_hook *hook, *hook2, *next;
6413         struct nft_flowtable *ft;
6414         int err, i = 0;
6415
6416         list_for_each_entry(hook, hook_list, list) {
6417                 list_for_each_entry(ft, &table->flowtables, list) {
6418                         list_for_each_entry(hook2, &ft->hook_list, list) {
6419                                 if (hook->ops.dev == hook2->ops.dev &&
6420                                     hook->ops.pf == hook2->ops.pf) {
6421                                         err = -EBUSY;
6422                                         goto err_unregister_net_hooks;
6423                                 }
6424                         }
6425                 }
6426
6427                 err = flowtable->data.type->setup(&flowtable->data,
6428                                                   hook->ops.dev,
6429                                                   FLOW_BLOCK_BIND);
6430                 if (err < 0)
6431                         goto err_unregister_net_hooks;
6432
6433                 err = nf_register_net_hook(net, &hook->ops);
6434                 if (err < 0) {
6435                         flowtable->data.type->setup(&flowtable->data,
6436                                                     hook->ops.dev,
6437                                                     FLOW_BLOCK_UNBIND);
6438                         goto err_unregister_net_hooks;
6439                 }
6440
6441                 i++;
6442         }
6443
6444         return 0;
6445
6446 err_unregister_net_hooks:
6447         list_for_each_entry_safe(hook, next, hook_list, list) {
6448                 if (i-- <= 0)
6449                         break;
6450
6451                 nft_unregister_flowtable_hook(net, flowtable, hook);
6452                 list_del_rcu(&hook->list);
6453                 kfree_rcu(hook, rcu);
6454         }
6455
6456         return err;
6457 }
6458
6459 static void nft_flowtable_hooks_destroy(struct list_head *hook_list)
6460 {
6461         struct nft_hook *hook, *next;
6462
6463         list_for_each_entry_safe(hook, next, hook_list, list) {
6464                 list_del_rcu(&hook->list);
6465                 kfree_rcu(hook, rcu);
6466         }
6467 }
6468
6469 static int nft_flowtable_update(struct nft_ctx *ctx, const struct nlmsghdr *nlh,
6470                                 struct nft_flowtable *flowtable)
6471 {
6472         const struct nlattr * const *nla = ctx->nla;
6473         struct nft_flowtable_hook flowtable_hook;
6474         struct nft_hook *hook, *next;
6475         struct nft_trans *trans;
6476         bool unregister = false;
6477         int err;
6478
6479         err = nft_flowtable_parse_hook(ctx, nla[NFTA_FLOWTABLE_HOOK],
6480                                        &flowtable_hook, flowtable, false);
6481         if (err < 0)
6482                 return err;
6483
6484         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
6485                 if (nft_hook_list_find(&flowtable->hook_list, hook)) {
6486                         list_del(&hook->list);
6487                         kfree(hook);
6488                 }
6489         }
6490
6491         err = nft_register_flowtable_net_hooks(ctx->net, ctx->table,
6492                                                &flowtable_hook.list, flowtable);
6493         if (err < 0)
6494                 goto err_flowtable_update_hook;
6495
6496         trans = nft_trans_alloc(ctx, NFT_MSG_NEWFLOWTABLE,
6497                                 sizeof(struct nft_trans_flowtable));
6498         if (!trans) {
6499                 unregister = true;
6500                 err = -ENOMEM;
6501                 goto err_flowtable_update_hook;
6502         }
6503
6504         nft_trans_flowtable(trans) = flowtable;
6505         nft_trans_flowtable_update(trans) = true;
6506         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
6507         list_splice(&flowtable_hook.list, &nft_trans_flowtable_hooks(trans));
6508
6509         list_add_tail(&trans->list, &ctx->net->nft.commit_list);
6510
6511         return 0;
6512
6513 err_flowtable_update_hook:
6514         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
6515                 if (unregister)
6516                         nft_unregister_flowtable_hook(ctx->net, flowtable, hook);
6517                 list_del_rcu(&hook->list);
6518                 kfree_rcu(hook, rcu);
6519         }
6520
6521         return err;
6522
6523 }
6524
6525 static int nf_tables_newflowtable(struct net *net, struct sock *nlsk,
6526                                   struct sk_buff *skb,
6527                                   const struct nlmsghdr *nlh,
6528                                   const struct nlattr * const nla[],
6529                                   struct netlink_ext_ack *extack)
6530 {
6531         const struct nfgenmsg *nfmsg = nlmsg_data(nlh);
6532         struct nft_flowtable_hook flowtable_hook;
6533         const struct nf_flowtable_type *type;
6534         u8 genmask = nft_genmask_next(net);
6535         int family = nfmsg->nfgen_family;
6536         struct nft_flowtable *flowtable;
6537         struct nft_hook *hook, *next;
6538         struct nft_table *table;
6539         struct nft_ctx ctx;
6540         int err;
6541
6542         if (!nla[NFTA_FLOWTABLE_TABLE] ||
6543             !nla[NFTA_FLOWTABLE_NAME] ||
6544             !nla[NFTA_FLOWTABLE_HOOK])
6545                 return -EINVAL;
6546
6547         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
6548                                  genmask);
6549         if (IS_ERR(table)) {
6550                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
6551                 return PTR_ERR(table);
6552         }
6553
6554         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
6555                                          genmask);
6556         if (IS_ERR(flowtable)) {
6557                 err = PTR_ERR(flowtable);
6558                 if (err != -ENOENT) {
6559                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
6560                         return err;
6561                 }
6562         } else {
6563                 if (nlh->nlmsg_flags & NLM_F_EXCL) {
6564                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
6565                         return -EEXIST;
6566                 }
6567
6568                 nft_ctx_init(&ctx, net, skb, nlh, family, table, NULL, nla);
6569
6570                 return nft_flowtable_update(&ctx, nlh, flowtable);
6571         }
6572
6573         nft_ctx_init(&ctx, net, skb, nlh, family, table, NULL, nla);
6574
6575         flowtable = kzalloc(sizeof(*flowtable), GFP_KERNEL);
6576         if (!flowtable)
6577                 return -ENOMEM;
6578
6579         flowtable->table = table;
6580         flowtable->handle = nf_tables_alloc_handle(table);
6581         INIT_LIST_HEAD(&flowtable->hook_list);
6582
6583         flowtable->name = nla_strdup(nla[NFTA_FLOWTABLE_NAME], GFP_KERNEL);
6584         if (!flowtable->name) {
6585                 err = -ENOMEM;
6586                 goto err1;
6587         }
6588
6589         type = nft_flowtable_type_get(net, family);
6590         if (IS_ERR(type)) {
6591                 err = PTR_ERR(type);
6592                 goto err2;
6593         }
6594
6595         if (nla[NFTA_FLOWTABLE_FLAGS]) {
6596                 flowtable->data.flags =
6597                         ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
6598                 if (flowtable->data.flags & ~NFT_FLOWTABLE_MASK)
6599                         goto err3;
6600         }
6601
6602         write_pnet(&flowtable->data.net, net);
6603         flowtable->data.type = type;
6604         err = type->init(&flowtable->data);
6605         if (err < 0)
6606                 goto err3;
6607
6608         err = nft_flowtable_parse_hook(&ctx, nla[NFTA_FLOWTABLE_HOOK],
6609                                        &flowtable_hook, flowtable, true);
6610         if (err < 0)
6611                 goto err4;
6612
6613         list_splice(&flowtable_hook.list, &flowtable->hook_list);
6614         flowtable->data.priority = flowtable_hook.priority;
6615         flowtable->hooknum = flowtable_hook.num;
6616
6617         err = nft_register_flowtable_net_hooks(ctx.net, table,
6618                                                &flowtable->hook_list,
6619                                                flowtable);
6620         if (err < 0) {
6621                 nft_flowtable_hooks_destroy(&flowtable->hook_list);
6622                 goto err4;
6623         }
6624
6625         err = nft_trans_flowtable_add(&ctx, NFT_MSG_NEWFLOWTABLE, flowtable);
6626         if (err < 0)
6627                 goto err5;
6628
6629         list_add_tail_rcu(&flowtable->list, &table->flowtables);
6630         table->use++;
6631
6632         return 0;
6633 err5:
6634         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
6635                 nft_unregister_flowtable_hook(net, flowtable, hook);
6636                 list_del_rcu(&hook->list);
6637                 kfree_rcu(hook, rcu);
6638         }
6639 err4:
6640         flowtable->data.type->free(&flowtable->data);
6641 err3:
6642         module_put(type->owner);
6643 err2:
6644         kfree(flowtable->name);
6645 err1:
6646         kfree(flowtable);
6647         return err;
6648 }
6649
6650 static void nft_flowtable_hook_release(struct nft_flowtable_hook *flowtable_hook)
6651 {
6652         struct nft_hook *this, *next;
6653
6654         list_for_each_entry_safe(this, next, &flowtable_hook->list, list) {
6655                 list_del(&this->list);
6656                 kfree(this);
6657         }
6658 }
6659
6660 static int nft_delflowtable_hook(struct nft_ctx *ctx,
6661                                  struct nft_flowtable *flowtable)
6662 {
6663         const struct nlattr * const *nla = ctx->nla;
6664         struct nft_flowtable_hook flowtable_hook;
6665         struct nft_hook *this, *hook;
6666         struct nft_trans *trans;
6667         int err;
6668
6669         err = nft_flowtable_parse_hook(ctx, nla[NFTA_FLOWTABLE_HOOK],
6670                                        &flowtable_hook, flowtable, false);
6671         if (err < 0)
6672                 return err;
6673
6674         list_for_each_entry(this, &flowtable_hook.list, list) {
6675                 hook = nft_hook_list_find(&flowtable->hook_list, this);
6676                 if (!hook) {
6677                         err = -ENOENT;
6678                         goto err_flowtable_del_hook;
6679                 }
6680                 hook->inactive = true;
6681         }
6682
6683         trans = nft_trans_alloc(ctx, NFT_MSG_DELFLOWTABLE,
6684                                 sizeof(struct nft_trans_flowtable));
6685         if (!trans) {
6686                 err = -ENOMEM;
6687                 goto err_flowtable_del_hook;
6688         }
6689
6690         nft_trans_flowtable(trans) = flowtable;
6691         nft_trans_flowtable_update(trans) = true;
6692         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
6693         nft_flowtable_hook_release(&flowtable_hook);
6694
6695         list_add_tail(&trans->list, &ctx->net->nft.commit_list);
6696
6697         return 0;
6698
6699 err_flowtable_del_hook:
6700         list_for_each_entry(this, &flowtable_hook.list, list) {
6701                 hook = nft_hook_list_find(&flowtable->hook_list, this);
6702                 if (!hook)
6703                         break;
6704
6705                 hook->inactive = false;
6706         }
6707         nft_flowtable_hook_release(&flowtable_hook);
6708
6709         return err;
6710 }
6711
6712 static int nf_tables_delflowtable(struct net *net, struct sock *nlsk,
6713                                   struct sk_buff *skb,
6714                                   const struct nlmsghdr *nlh,
6715                                   const struct nlattr * const nla[],
6716                                   struct netlink_ext_ack *extack)
6717 {
6718         const struct nfgenmsg *nfmsg = nlmsg_data(nlh);
6719         u8 genmask = nft_genmask_next(net);
6720         int family = nfmsg->nfgen_family;
6721         struct nft_flowtable *flowtable;
6722         const struct nlattr *attr;
6723         struct nft_table *table;
6724         struct nft_ctx ctx;
6725
6726         if (!nla[NFTA_FLOWTABLE_TABLE] ||
6727             (!nla[NFTA_FLOWTABLE_NAME] &&
6728              !nla[NFTA_FLOWTABLE_HANDLE]))
6729                 return -EINVAL;
6730
6731         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
6732                                  genmask);
6733         if (IS_ERR(table)) {
6734                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
6735                 return PTR_ERR(table);
6736         }
6737
6738         if (nla[NFTA_FLOWTABLE_HANDLE]) {
6739                 attr = nla[NFTA_FLOWTABLE_HANDLE];
6740                 flowtable = nft_flowtable_lookup_byhandle(table, attr, genmask);
6741         } else {
6742                 attr = nla[NFTA_FLOWTABLE_NAME];
6743                 flowtable = nft_flowtable_lookup(table, attr, genmask);
6744         }
6745
6746         if (IS_ERR(flowtable)) {
6747                 NL_SET_BAD_ATTR(extack, attr);
6748                 return PTR_ERR(flowtable);
6749         }
6750
6751         nft_ctx_init(&ctx, net, skb, nlh, family, table, NULL, nla);
6752
6753         if (nla[NFTA_FLOWTABLE_HOOK])
6754                 return nft_delflowtable_hook(&ctx, flowtable);
6755
6756         if (flowtable->use > 0) {
6757                 NL_SET_BAD_ATTR(extack, attr);
6758                 return -EBUSY;
6759         }
6760
6761         return nft_delflowtable(&ctx, flowtable);
6762 }
6763
6764 static int nf_tables_fill_flowtable_info(struct sk_buff *skb, struct net *net,
6765                                          u32 portid, u32 seq, int event,
6766                                          u32 flags, int family,
6767                                          struct nft_flowtable *flowtable,
6768                                          struct list_head *hook_list)
6769 {
6770         struct nlattr *nest, *nest_devs;
6771         struct nfgenmsg *nfmsg;
6772         struct nft_hook *hook;
6773         struct nlmsghdr *nlh;
6774
6775         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
6776         nlh = nlmsg_put(skb, portid, seq, event, sizeof(struct nfgenmsg), flags);
6777         if (nlh == NULL)
6778                 goto nla_put_failure;
6779
6780         nfmsg = nlmsg_data(nlh);
6781         nfmsg->nfgen_family     = family;
6782         nfmsg->version          = NFNETLINK_V0;
6783         nfmsg->res_id           = htons(net->nft.base_seq & 0xffff);
6784
6785         if (nla_put_string(skb, NFTA_FLOWTABLE_TABLE, flowtable->table->name) ||
6786             nla_put_string(skb, NFTA_FLOWTABLE_NAME, flowtable->name) ||
6787             nla_put_be32(skb, NFTA_FLOWTABLE_USE, htonl(flowtable->use)) ||
6788             nla_put_be64(skb, NFTA_FLOWTABLE_HANDLE, cpu_to_be64(flowtable->handle),
6789                          NFTA_FLOWTABLE_PAD) ||
6790             nla_put_be32(skb, NFTA_FLOWTABLE_FLAGS, htonl(flowtable->data.flags)))
6791                 goto nla_put_failure;
6792
6793         nest = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK);
6794         if (!nest)
6795                 goto nla_put_failure;
6796         if (nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_NUM, htonl(flowtable->hooknum)) ||
6797             nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_PRIORITY, htonl(flowtable->data.priority)))
6798                 goto nla_put_failure;
6799
6800         nest_devs = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK_DEVS);
6801         if (!nest_devs)
6802                 goto nla_put_failure;
6803
6804         list_for_each_entry_rcu(hook, hook_list, list) {
6805                 if (nla_put_string(skb, NFTA_DEVICE_NAME, hook->ops.dev->name))
6806                         goto nla_put_failure;
6807         }
6808         nla_nest_end(skb, nest_devs);
6809         nla_nest_end(skb, nest);
6810
6811         nlmsg_end(skb, nlh);
6812         return 0;
6813
6814 nla_put_failure:
6815         nlmsg_trim(skb, nlh);
6816         return -1;
6817 }
6818
6819 struct nft_flowtable_filter {
6820         char            *table;
6821 };
6822
6823 static int nf_tables_dump_flowtable(struct sk_buff *skb,
6824                                     struct netlink_callback *cb)
6825 {
6826         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
6827         struct nft_flowtable_filter *filter = cb->data;
6828         unsigned int idx = 0, s_idx = cb->args[0];
6829         struct net *net = sock_net(skb->sk);
6830         int family = nfmsg->nfgen_family;
6831         struct nft_flowtable *flowtable;
6832         const struct nft_table *table;
6833
6834         rcu_read_lock();
6835         cb->seq = net->nft.base_seq;
6836
6837         list_for_each_entry_rcu(table, &net->nft.tables, list) {
6838                 if (family != NFPROTO_UNSPEC && family != table->family)
6839                         continue;
6840
6841                 list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
6842                         if (!nft_is_active(net, flowtable))
6843                                 goto cont;
6844                         if (idx < s_idx)
6845                                 goto cont;
6846                         if (idx > s_idx)
6847                                 memset(&cb->args[1], 0,
6848                                        sizeof(cb->args) - sizeof(cb->args[0]));
6849                         if (filter && filter->table &&
6850                             strcmp(filter->table, table->name))
6851                                 goto cont;
6852
6853                         if (nf_tables_fill_flowtable_info(skb, net, NETLINK_CB(cb->skb).portid,
6854                                                           cb->nlh->nlmsg_seq,
6855                                                           NFT_MSG_NEWFLOWTABLE,
6856                                                           NLM_F_MULTI | NLM_F_APPEND,
6857                                                           table->family,
6858                                                           flowtable,
6859                                                           &flowtable->hook_list) < 0)
6860                                 goto done;
6861
6862                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
6863 cont:
6864                         idx++;
6865                 }
6866         }
6867 done:
6868         rcu_read_unlock();
6869
6870         cb->args[0] = idx;
6871         return skb->len;
6872 }
6873
6874 static int nf_tables_dump_flowtable_start(struct netlink_callback *cb)
6875 {
6876         const struct nlattr * const *nla = cb->data;
6877         struct nft_flowtable_filter *filter = NULL;
6878
6879         if (nla[NFTA_FLOWTABLE_TABLE]) {
6880                 filter = kzalloc(sizeof(*filter), GFP_ATOMIC);
6881                 if (!filter)
6882                         return -ENOMEM;
6883
6884                 filter->table = nla_strdup(nla[NFTA_FLOWTABLE_TABLE],
6885                                            GFP_ATOMIC);
6886                 if (!filter->table) {
6887                         kfree(filter);
6888                         return -ENOMEM;
6889                 }
6890         }
6891
6892         cb->data = filter;
6893         return 0;
6894 }
6895
6896 static int nf_tables_dump_flowtable_done(struct netlink_callback *cb)
6897 {
6898         struct nft_flowtable_filter *filter = cb->data;
6899
6900         if (!filter)
6901                 return 0;
6902
6903         kfree(filter->table);
6904         kfree(filter);
6905
6906         return 0;
6907 }
6908
6909 /* called with rcu_read_lock held */
6910 static int nf_tables_getflowtable(struct net *net, struct sock *nlsk,
6911                                   struct sk_buff *skb,
6912                                   const struct nlmsghdr *nlh,
6913                                   const struct nlattr * const nla[],
6914                                   struct netlink_ext_ack *extack)
6915 {
6916         const struct nfgenmsg *nfmsg = nlmsg_data(nlh);
6917         u8 genmask = nft_genmask_cur(net);
6918         int family = nfmsg->nfgen_family;
6919         struct nft_flowtable *flowtable;
6920         const struct nft_table *table;
6921         struct sk_buff *skb2;
6922         int err;
6923
6924         if (nlh->nlmsg_flags & NLM_F_DUMP) {
6925                 struct netlink_dump_control c = {
6926                         .start = nf_tables_dump_flowtable_start,
6927                         .dump = nf_tables_dump_flowtable,
6928                         .done = nf_tables_dump_flowtable_done,
6929                         .module = THIS_MODULE,
6930                         .data = (void *)nla,
6931                 };
6932
6933                 return nft_netlink_dump_start_rcu(nlsk, skb, nlh, &c);
6934         }
6935
6936         if (!nla[NFTA_FLOWTABLE_NAME])
6937                 return -EINVAL;
6938
6939         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
6940                                  genmask);
6941         if (IS_ERR(table))
6942                 return PTR_ERR(table);
6943
6944         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
6945                                          genmask);
6946         if (IS_ERR(flowtable))
6947                 return PTR_ERR(flowtable);
6948
6949         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
6950         if (!skb2)
6951                 return -ENOMEM;
6952
6953         err = nf_tables_fill_flowtable_info(skb2, net, NETLINK_CB(skb).portid,
6954                                             nlh->nlmsg_seq,
6955                                             NFT_MSG_NEWFLOWTABLE, 0, family,
6956                                             flowtable, &flowtable->hook_list);
6957         if (err < 0)
6958                 goto err;
6959
6960         return nlmsg_unicast(nlsk, skb2, NETLINK_CB(skb).portid);
6961 err:
6962         kfree_skb(skb2);
6963         return err;
6964 }
6965
6966 static void nf_tables_flowtable_notify(struct nft_ctx *ctx,
6967                                        struct nft_flowtable *flowtable,
6968                                        struct list_head *hook_list,
6969                                        int event)
6970 {
6971         struct sk_buff *skb;
6972         int err;
6973
6974         if (ctx->report &&
6975             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
6976                 return;
6977
6978         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
6979         if (skb == NULL)
6980                 goto err;
6981
6982         err = nf_tables_fill_flowtable_info(skb, ctx->net, ctx->portid,
6983                                             ctx->seq, event, 0,
6984                                             ctx->family, flowtable, hook_list);
6985         if (err < 0) {
6986                 kfree_skb(skb);
6987                 goto err;
6988         }
6989
6990         nfnetlink_send(skb, ctx->net, ctx->portid, NFNLGRP_NFTABLES,
6991                        ctx->report, GFP_KERNEL);
6992         return;
6993 err:
6994         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
6995 }
6996
6997 static void nf_tables_flowtable_destroy(struct nft_flowtable *flowtable)
6998 {
6999         struct nft_hook *hook, *next;
7000
7001         flowtable->data.type->free(&flowtable->data);
7002         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
7003                 flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
7004                                             FLOW_BLOCK_UNBIND);
7005                 list_del_rcu(&hook->list);
7006                 kfree(hook);
7007         }
7008         kfree(flowtable->name);
7009         module_put(flowtable->data.type->owner);
7010         kfree(flowtable);
7011 }
7012
7013 static int nf_tables_fill_gen_info(struct sk_buff *skb, struct net *net,
7014                                    u32 portid, u32 seq)
7015 {
7016         struct nlmsghdr *nlh;
7017         struct nfgenmsg *nfmsg;
7018         char buf[TASK_COMM_LEN];
7019         int event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWGEN);
7020
7021         nlh = nlmsg_put(skb, portid, seq, event, sizeof(struct nfgenmsg), 0);
7022         if (nlh == NULL)
7023                 goto nla_put_failure;
7024
7025         nfmsg = nlmsg_data(nlh);
7026         nfmsg->nfgen_family     = AF_UNSPEC;
7027         nfmsg->version          = NFNETLINK_V0;
7028         nfmsg->res_id           = htons(net->nft.base_seq & 0xffff);
7029
7030         if (nla_put_be32(skb, NFTA_GEN_ID, htonl(net->nft.base_seq)) ||
7031             nla_put_be32(skb, NFTA_GEN_PROC_PID, htonl(task_pid_nr(current))) ||
7032             nla_put_string(skb, NFTA_GEN_PROC_NAME, get_task_comm(buf, current)))
7033                 goto nla_put_failure;
7034
7035         nlmsg_end(skb, nlh);
7036         return 0;
7037
7038 nla_put_failure:
7039         nlmsg_trim(skb, nlh);
7040         return -EMSGSIZE;
7041 }
7042
7043 static void nft_flowtable_event(unsigned long event, struct net_device *dev,
7044                                 struct nft_flowtable *flowtable)
7045 {
7046         struct nft_hook *hook;
7047
7048         list_for_each_entry(hook, &flowtable->hook_list, list) {
7049                 if (hook->ops.dev != dev)
7050                         continue;
7051
7052                 /* flow_offload_netdev_event() cleans up entries for us. */
7053                 nft_unregister_flowtable_hook(dev_net(dev), flowtable, hook);
7054                 list_del_rcu(&hook->list);
7055                 kfree_rcu(hook, rcu);
7056                 break;
7057         }
7058 }
7059
7060 static int nf_tables_flowtable_event(struct notifier_block *this,
7061                                      unsigned long event, void *ptr)
7062 {
7063         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
7064         struct nft_flowtable *flowtable;
7065         struct nft_table *table;
7066         struct net *net;
7067
7068         if (event != NETDEV_UNREGISTER)
7069                 return 0;
7070
7071         net = dev_net(dev);
7072         mutex_lock(&net->nft.commit_mutex);
7073         list_for_each_entry(table, &net->nft.tables, list) {
7074                 list_for_each_entry(flowtable, &table->flowtables, list) {
7075                         nft_flowtable_event(event, dev, flowtable);
7076                 }
7077         }
7078         mutex_unlock(&net->nft.commit_mutex);
7079
7080         return NOTIFY_DONE;
7081 }
7082
7083 static struct notifier_block nf_tables_flowtable_notifier = {
7084         .notifier_call  = nf_tables_flowtable_event,
7085 };
7086
7087 static void nf_tables_gen_notify(struct net *net, struct sk_buff *skb,
7088                                  int event)
7089 {
7090         struct nlmsghdr *nlh = nlmsg_hdr(skb);
7091         struct sk_buff *skb2;
7092         int err;
7093
7094         if (nlmsg_report(nlh) &&
7095             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
7096                 return;
7097
7098         skb2 = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
7099         if (skb2 == NULL)
7100                 goto err;
7101
7102         err = nf_tables_fill_gen_info(skb2, net, NETLINK_CB(skb).portid,
7103                                       nlh->nlmsg_seq);
7104         if (err < 0) {
7105                 kfree_skb(skb2);
7106                 goto err;
7107         }
7108
7109         nfnetlink_send(skb2, net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
7110                        nlmsg_report(nlh), GFP_KERNEL);
7111         return;
7112 err:
7113         nfnetlink_set_err(net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
7114                           -ENOBUFS);
7115 }
7116
7117 static int nf_tables_getgen(struct net *net, struct sock *nlsk,
7118                             struct sk_buff *skb, const struct nlmsghdr *nlh,
7119                             const struct nlattr * const nla[],
7120                             struct netlink_ext_ack *extack)
7121 {
7122         struct sk_buff *skb2;
7123         int err;
7124
7125         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
7126         if (skb2 == NULL)
7127                 return -ENOMEM;
7128
7129         err = nf_tables_fill_gen_info(skb2, net, NETLINK_CB(skb).portid,
7130                                       nlh->nlmsg_seq);
7131         if (err < 0)
7132                 goto err;
7133
7134         return nlmsg_unicast(nlsk, skb2, NETLINK_CB(skb).portid);
7135 err:
7136         kfree_skb(skb2);
7137         return err;
7138 }
7139
7140 static const struct nfnl_callback nf_tables_cb[NFT_MSG_MAX] = {
7141         [NFT_MSG_NEWTABLE] = {
7142                 .call_batch     = nf_tables_newtable,
7143                 .attr_count     = NFTA_TABLE_MAX,
7144                 .policy         = nft_table_policy,
7145         },
7146         [NFT_MSG_GETTABLE] = {
7147                 .call_rcu       = nf_tables_gettable,
7148                 .attr_count     = NFTA_TABLE_MAX,
7149                 .policy         = nft_table_policy,
7150         },
7151         [NFT_MSG_DELTABLE] = {
7152                 .call_batch     = nf_tables_deltable,
7153                 .attr_count     = NFTA_TABLE_MAX,
7154                 .policy         = nft_table_policy,
7155         },
7156         [NFT_MSG_NEWCHAIN] = {
7157                 .call_batch     = nf_tables_newchain,
7158                 .attr_count     = NFTA_CHAIN_MAX,
7159                 .policy         = nft_chain_policy,
7160         },
7161         [NFT_MSG_GETCHAIN] = {
7162                 .call_rcu       = nf_tables_getchain,
7163                 .attr_count     = NFTA_CHAIN_MAX,
7164                 .policy         = nft_chain_policy,
7165         },
7166         [NFT_MSG_DELCHAIN] = {
7167                 .call_batch     = nf_tables_delchain,
7168                 .attr_count     = NFTA_CHAIN_MAX,
7169                 .policy         = nft_chain_policy,
7170         },
7171         [NFT_MSG_NEWRULE] = {
7172                 .call_batch     = nf_tables_newrule,
7173                 .attr_count     = NFTA_RULE_MAX,
7174                 .policy         = nft_rule_policy,
7175         },
7176         [NFT_MSG_GETRULE] = {
7177                 .call_rcu       = nf_tables_getrule,
7178                 .attr_count     = NFTA_RULE_MAX,
7179                 .policy         = nft_rule_policy,
7180         },
7181         [NFT_MSG_DELRULE] = {
7182                 .call_batch     = nf_tables_delrule,
7183                 .attr_count     = NFTA_RULE_MAX,
7184                 .policy         = nft_rule_policy,
7185         },
7186         [NFT_MSG_NEWSET] = {
7187                 .call_batch     = nf_tables_newset,
7188                 .attr_count     = NFTA_SET_MAX,
7189                 .policy         = nft_set_policy,
7190         },
7191         [NFT_MSG_GETSET] = {
7192                 .call_rcu       = nf_tables_getset,
7193                 .attr_count     = NFTA_SET_MAX,
7194                 .policy         = nft_set_policy,
7195         },
7196         [NFT_MSG_DELSET] = {
7197                 .call_batch     = nf_tables_delset,
7198                 .attr_count     = NFTA_SET_MAX,
7199                 .policy         = nft_set_policy,
7200         },
7201         [NFT_MSG_NEWSETELEM] = {
7202                 .call_batch     = nf_tables_newsetelem,
7203                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
7204                 .policy         = nft_set_elem_list_policy,
7205         },
7206         [NFT_MSG_GETSETELEM] = {
7207                 .call_rcu       = nf_tables_getsetelem,
7208                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
7209                 .policy         = nft_set_elem_list_policy,
7210         },
7211         [NFT_MSG_DELSETELEM] = {
7212                 .call_batch     = nf_tables_delsetelem,
7213                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
7214                 .policy         = nft_set_elem_list_policy,
7215         },
7216         [NFT_MSG_GETGEN] = {
7217                 .call_rcu       = nf_tables_getgen,
7218         },
7219         [NFT_MSG_NEWOBJ] = {
7220                 .call_batch     = nf_tables_newobj,
7221                 .attr_count     = NFTA_OBJ_MAX,
7222                 .policy         = nft_obj_policy,
7223         },
7224         [NFT_MSG_GETOBJ] = {
7225                 .call_rcu       = nf_tables_getobj,
7226                 .attr_count     = NFTA_OBJ_MAX,
7227                 .policy         = nft_obj_policy,
7228         },
7229         [NFT_MSG_DELOBJ] = {
7230                 .call_batch     = nf_tables_delobj,
7231                 .attr_count     = NFTA_OBJ_MAX,
7232                 .policy         = nft_obj_policy,
7233         },
7234         [NFT_MSG_GETOBJ_RESET] = {
7235                 .call_rcu       = nf_tables_getobj,
7236                 .attr_count     = NFTA_OBJ_MAX,
7237                 .policy         = nft_obj_policy,
7238         },
7239         [NFT_MSG_NEWFLOWTABLE] = {
7240                 .call_batch     = nf_tables_newflowtable,
7241                 .attr_count     = NFTA_FLOWTABLE_MAX,
7242                 .policy         = nft_flowtable_policy,
7243         },
7244         [NFT_MSG_GETFLOWTABLE] = {
7245                 .call_rcu       = nf_tables_getflowtable,
7246                 .attr_count     = NFTA_FLOWTABLE_MAX,
7247                 .policy         = nft_flowtable_policy,
7248         },
7249         [NFT_MSG_DELFLOWTABLE] = {
7250                 .call_batch     = nf_tables_delflowtable,
7251                 .attr_count     = NFTA_FLOWTABLE_MAX,
7252                 .policy         = nft_flowtable_policy,
7253         },
7254 };
7255
7256 static int nf_tables_validate(struct net *net)
7257 {
7258         struct nft_table *table;
7259
7260         switch (net->nft.validate_state) {
7261         case NFT_VALIDATE_SKIP:
7262                 break;
7263         case NFT_VALIDATE_NEED:
7264                 nft_validate_state_update(net, NFT_VALIDATE_DO);
7265                 /* fall through */
7266         case NFT_VALIDATE_DO:
7267                 list_for_each_entry(table, &net->nft.tables, list) {
7268                         if (nft_table_validate(net, table) < 0)
7269                                 return -EAGAIN;
7270                 }
7271                 break;
7272         }
7273
7274         return 0;
7275 }
7276
7277 /* a drop policy has to be deferred until all rules have been activated,
7278  * otherwise a large ruleset that contains a drop-policy base chain will
7279  * cause all packets to get dropped until the full transaction has been
7280  * processed.
7281  *
7282  * We defer the drop policy until the transaction has been finalized.
7283  */
7284 static void nft_chain_commit_drop_policy(struct nft_trans *trans)
7285 {
7286         struct nft_base_chain *basechain;
7287
7288         if (nft_trans_chain_policy(trans) != NF_DROP)
7289                 return;
7290
7291         if (!nft_is_base_chain(trans->ctx.chain))
7292                 return;
7293
7294         basechain = nft_base_chain(trans->ctx.chain);
7295         basechain->policy = NF_DROP;
7296 }
7297
7298 static void nft_chain_commit_update(struct nft_trans *trans)
7299 {
7300         struct nft_base_chain *basechain;
7301
7302         if (nft_trans_chain_name(trans)) {
7303                 rhltable_remove(&trans->ctx.table->chains_ht,
7304                                 &trans->ctx.chain->rhlhead,
7305                                 nft_chain_ht_params);
7306                 swap(trans->ctx.chain->name, nft_trans_chain_name(trans));
7307                 rhltable_insert_key(&trans->ctx.table->chains_ht,
7308                                     trans->ctx.chain->name,
7309                                     &trans->ctx.chain->rhlhead,
7310                                     nft_chain_ht_params);
7311         }
7312
7313         if (!nft_is_base_chain(trans->ctx.chain))
7314                 return;
7315
7316         nft_chain_stats_replace(trans);
7317
7318         basechain = nft_base_chain(trans->ctx.chain);
7319
7320         switch (nft_trans_chain_policy(trans)) {
7321         case NF_DROP:
7322         case NF_ACCEPT:
7323                 basechain->policy = nft_trans_chain_policy(trans);
7324                 break;
7325         }
7326 }
7327
7328 static void nft_obj_commit_update(struct nft_trans *trans)
7329 {
7330         struct nft_object *newobj;
7331         struct nft_object *obj;
7332
7333         obj = nft_trans_obj(trans);
7334         newobj = nft_trans_obj_newobj(trans);
7335
7336         if (obj->ops->update)
7337                 obj->ops->update(obj, newobj);
7338
7339         kfree(newobj);
7340 }
7341
7342 static void nft_commit_release(struct nft_trans *trans)
7343 {
7344         switch (trans->msg_type) {
7345         case NFT_MSG_DELTABLE:
7346                 nf_tables_table_destroy(&trans->ctx);
7347                 break;
7348         case NFT_MSG_NEWCHAIN:
7349                 free_percpu(nft_trans_chain_stats(trans));
7350                 kfree(nft_trans_chain_name(trans));
7351                 break;
7352         case NFT_MSG_DELCHAIN:
7353                 nf_tables_chain_destroy(&trans->ctx);
7354                 break;
7355         case NFT_MSG_DELRULE:
7356                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
7357                 break;
7358         case NFT_MSG_DELSET:
7359                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
7360                 break;
7361         case NFT_MSG_DELSETELEM:
7362                 nf_tables_set_elem_destroy(&trans->ctx,
7363                                            nft_trans_elem_set(trans),
7364                                            nft_trans_elem(trans).priv);
7365                 break;
7366         case NFT_MSG_DELOBJ:
7367                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
7368                 break;
7369         case NFT_MSG_DELFLOWTABLE:
7370                 if (nft_trans_flowtable_update(trans))
7371                         nft_flowtable_hooks_destroy(&nft_trans_flowtable_hooks(trans));
7372                 else
7373                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
7374                 break;
7375         }
7376
7377         if (trans->put_net)
7378                 put_net(trans->ctx.net);
7379
7380         kfree(trans);
7381 }
7382
7383 static void nf_tables_trans_destroy_work(struct work_struct *w)
7384 {
7385         struct nft_trans *trans, *next;
7386         LIST_HEAD(head);
7387
7388         spin_lock(&nf_tables_destroy_list_lock);
7389         list_splice_init(&nf_tables_destroy_list, &head);
7390         spin_unlock(&nf_tables_destroy_list_lock);
7391
7392         if (list_empty(&head))
7393                 return;
7394
7395         synchronize_rcu();
7396
7397         list_for_each_entry_safe(trans, next, &head, list) {
7398                 list_del(&trans->list);
7399                 nft_commit_release(trans);
7400         }
7401 }
7402
7403 static int nf_tables_commit_chain_prepare(struct net *net, struct nft_chain *chain)
7404 {
7405         struct nft_rule *rule;
7406         unsigned int alloc = 0;
7407         int i;
7408
7409         /* already handled or inactive chain? */
7410         if (chain->rules_next || !nft_is_active_next(net, chain))
7411                 return 0;
7412
7413         rule = list_entry(&chain->rules, struct nft_rule, list);
7414         i = 0;
7415
7416         list_for_each_entry_continue(rule, &chain->rules, list) {
7417                 if (nft_is_active_next(net, rule))
7418                         alloc++;
7419         }
7420
7421         chain->rules_next = nf_tables_chain_alloc_rules(chain, alloc);
7422         if (!chain->rules_next)
7423                 return -ENOMEM;
7424
7425         list_for_each_entry_continue(rule, &chain->rules, list) {
7426                 if (nft_is_active_next(net, rule))
7427                         chain->rules_next[i++] = rule;
7428         }
7429
7430         chain->rules_next[i] = NULL;
7431         return 0;
7432 }
7433
7434 static void nf_tables_commit_chain_prepare_cancel(struct net *net)
7435 {
7436         struct nft_trans *trans, *next;
7437
7438         list_for_each_entry_safe(trans, next, &net->nft.commit_list, list) {
7439                 struct nft_chain *chain = trans->ctx.chain;
7440
7441                 if (trans->msg_type == NFT_MSG_NEWRULE ||
7442                     trans->msg_type == NFT_MSG_DELRULE) {
7443                         kvfree(chain->rules_next);
7444                         chain->rules_next = NULL;
7445                 }
7446         }
7447 }
7448
7449 static void __nf_tables_commit_chain_free_rules_old(struct rcu_head *h)
7450 {
7451         struct nft_rules_old *o = container_of(h, struct nft_rules_old, h);
7452
7453         kvfree(o->start);
7454 }
7455
7456 static void nf_tables_commit_chain_free_rules_old(struct nft_rule **rules)
7457 {
7458         struct nft_rule **r = rules;
7459         struct nft_rules_old *old;
7460
7461         while (*r)
7462                 r++;
7463
7464         r++;    /* rcu_head is after end marker */
7465         old = (void *) r;
7466         old->start = rules;
7467
7468         call_rcu(&old->h, __nf_tables_commit_chain_free_rules_old);
7469 }
7470
7471 static void nf_tables_commit_chain(struct net *net, struct nft_chain *chain)
7472 {
7473         struct nft_rule **g0, **g1;
7474         bool next_genbit;
7475
7476         next_genbit = nft_gencursor_next(net);
7477
7478         g0 = rcu_dereference_protected(chain->rules_gen_0,
7479                                        lockdep_commit_lock_is_held(net));
7480         g1 = rcu_dereference_protected(chain->rules_gen_1,
7481                                        lockdep_commit_lock_is_held(net));
7482
7483         /* No changes to this chain? */
7484         if (chain->rules_next == NULL) {
7485                 /* chain had no change in last or next generation */
7486                 if (g0 == g1)
7487                         return;
7488                 /*
7489                  * chain had no change in this generation; make sure next
7490                  * one uses same rules as current generation.
7491                  */
7492                 if (next_genbit) {
7493                         rcu_assign_pointer(chain->rules_gen_1, g0);
7494                         nf_tables_commit_chain_free_rules_old(g1);
7495                 } else {
7496                         rcu_assign_pointer(chain->rules_gen_0, g1);
7497                         nf_tables_commit_chain_free_rules_old(g0);
7498                 }
7499
7500                 return;
7501         }
7502
7503         if (next_genbit)
7504                 rcu_assign_pointer(chain->rules_gen_1, chain->rules_next);
7505         else
7506                 rcu_assign_pointer(chain->rules_gen_0, chain->rules_next);
7507
7508         chain->rules_next = NULL;
7509
7510         if (g0 == g1)
7511                 return;
7512
7513         if (next_genbit)
7514                 nf_tables_commit_chain_free_rules_old(g1);
7515         else
7516                 nf_tables_commit_chain_free_rules_old(g0);
7517 }
7518
7519 static void nft_obj_del(struct nft_object *obj)
7520 {
7521         rhltable_remove(&nft_objname_ht, &obj->rhlhead, nft_objname_ht_params);
7522         list_del_rcu(&obj->list);
7523 }
7524
7525 void nft_chain_del(struct nft_chain *chain)
7526 {
7527         struct nft_table *table = chain->table;
7528
7529         WARN_ON_ONCE(rhltable_remove(&table->chains_ht, &chain->rhlhead,
7530                                      nft_chain_ht_params));
7531         list_del_rcu(&chain->list);
7532 }
7533
7534 static void nft_flowtable_hooks_del(struct nft_flowtable *flowtable,
7535                                     struct list_head *hook_list)
7536 {
7537         struct nft_hook *hook, *next;
7538
7539         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
7540                 if (hook->inactive)
7541                         list_move(&hook->list, hook_list);
7542         }
7543 }
7544
7545 static void nf_tables_module_autoload_cleanup(struct net *net)
7546 {
7547         struct nft_module_request *req, *next;
7548
7549         WARN_ON_ONCE(!list_empty(&net->nft.commit_list));
7550         list_for_each_entry_safe(req, next, &net->nft.module_list, list) {
7551                 WARN_ON_ONCE(!req->done);
7552                 list_del(&req->list);
7553                 kfree(req);
7554         }
7555 }
7556
7557 static void nf_tables_commit_release(struct net *net)
7558 {
7559         struct nft_trans *trans;
7560
7561         /* all side effects have to be made visible.
7562          * For example, if a chain named 'foo' has been deleted, a
7563          * new transaction must not find it anymore.
7564          *
7565          * Memory reclaim happens asynchronously from work queue
7566          * to prevent expensive synchronize_rcu() in commit phase.
7567          */
7568         if (list_empty(&net->nft.commit_list)) {
7569                 nf_tables_module_autoload_cleanup(net);
7570                 mutex_unlock(&net->nft.commit_mutex);
7571                 return;
7572         }
7573
7574         trans = list_last_entry(&net->nft.commit_list,
7575                                 struct nft_trans, list);
7576         get_net(trans->ctx.net);
7577         WARN_ON_ONCE(trans->put_net);
7578
7579         trans->put_net = true;
7580         spin_lock(&nf_tables_destroy_list_lock);
7581         list_splice_tail_init(&net->nft.commit_list, &nf_tables_destroy_list);
7582         spin_unlock(&nf_tables_destroy_list_lock);
7583
7584         nf_tables_module_autoload_cleanup(net);
7585         mutex_unlock(&net->nft.commit_mutex);
7586
7587         schedule_work(&trans_destroy_work);
7588 }
7589
7590 static int nf_tables_commit(struct net *net, struct sk_buff *skb)
7591 {
7592         struct nft_trans *trans, *next;
7593         struct nft_trans_elem *te;
7594         struct nft_chain *chain;
7595         struct nft_table *table;
7596         int err;
7597
7598         if (list_empty(&net->nft.commit_list)) {
7599                 mutex_unlock(&net->nft.commit_mutex);
7600                 return 0;
7601         }
7602
7603         /* 0. Validate ruleset, otherwise roll back for error reporting. */
7604         if (nf_tables_validate(net) < 0)
7605                 return -EAGAIN;
7606
7607         err = nft_flow_rule_offload_commit(net);
7608         if (err < 0)
7609                 return err;
7610
7611         /* 1.  Allocate space for next generation rules_gen_X[] */
7612         list_for_each_entry_safe(trans, next, &net->nft.commit_list, list) {
7613                 int ret;
7614
7615                 if (trans->msg_type == NFT_MSG_NEWRULE ||
7616                     trans->msg_type == NFT_MSG_DELRULE) {
7617                         chain = trans->ctx.chain;
7618
7619                         ret = nf_tables_commit_chain_prepare(net, chain);
7620                         if (ret < 0) {
7621                                 nf_tables_commit_chain_prepare_cancel(net);
7622                                 return ret;
7623                         }
7624                 }
7625         }
7626
7627         /* step 2.  Make rules_gen_X visible to packet path */
7628         list_for_each_entry(table, &net->nft.tables, list) {
7629                 list_for_each_entry(chain, &table->chains, list)
7630                         nf_tables_commit_chain(net, chain);
7631         }
7632
7633         /*
7634          * Bump generation counter, invalidate any dump in progress.
7635          * Cannot fail after this point.
7636          */
7637         while (++net->nft.base_seq == 0);
7638
7639         /* step 3. Start new generation, rules_gen_X now in use. */
7640         net->nft.gencursor = nft_gencursor_next(net);
7641
7642         list_for_each_entry_safe(trans, next, &net->nft.commit_list, list) {
7643                 switch (trans->msg_type) {
7644                 case NFT_MSG_NEWTABLE:
7645                         if (nft_trans_table_update(trans)) {
7646                                 if (!nft_trans_table_enable(trans)) {
7647                                         nf_tables_table_disable(net,
7648                                                                 trans->ctx.table);
7649                                         trans->ctx.table->flags |= NFT_TABLE_F_DORMANT;
7650                                 }
7651                         } else {
7652                                 nft_clear(net, trans->ctx.table);
7653                         }
7654                         nf_tables_table_notify(&trans->ctx, NFT_MSG_NEWTABLE);
7655                         nft_trans_destroy(trans);
7656                         break;
7657                 case NFT_MSG_DELTABLE:
7658                         list_del_rcu(&trans->ctx.table->list);
7659                         nf_tables_table_notify(&trans->ctx, NFT_MSG_DELTABLE);
7660                         break;
7661                 case NFT_MSG_NEWCHAIN:
7662                         if (nft_trans_chain_update(trans)) {
7663                                 nft_chain_commit_update(trans);
7664                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN);
7665                                 /* trans destroyed after rcu grace period */
7666                         } else {
7667                                 nft_chain_commit_drop_policy(trans);
7668                                 nft_clear(net, trans->ctx.chain);
7669                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN);
7670                                 nft_trans_destroy(trans);
7671                         }
7672                         break;
7673                 case NFT_MSG_DELCHAIN:
7674                         nft_chain_del(trans->ctx.chain);
7675                         nf_tables_chain_notify(&trans->ctx, NFT_MSG_DELCHAIN);
7676                         nf_tables_unregister_hook(trans->ctx.net,
7677                                                   trans->ctx.table,
7678                                                   trans->ctx.chain);
7679                         break;
7680                 case NFT_MSG_NEWRULE:
7681                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
7682                         nf_tables_rule_notify(&trans->ctx,
7683                                               nft_trans_rule(trans),
7684                                               NFT_MSG_NEWRULE);
7685                         nft_trans_destroy(trans);
7686                         break;
7687                 case NFT_MSG_DELRULE:
7688                         list_del_rcu(&nft_trans_rule(trans)->list);
7689                         nf_tables_rule_notify(&trans->ctx,
7690                                               nft_trans_rule(trans),
7691                                               NFT_MSG_DELRULE);
7692                         nft_rule_expr_deactivate(&trans->ctx,
7693                                                  nft_trans_rule(trans),
7694                                                  NFT_TRANS_COMMIT);
7695                         break;
7696                 case NFT_MSG_NEWSET:
7697                         nft_clear(net, nft_trans_set(trans));
7698                         /* This avoids hitting -EBUSY when deleting the table
7699                          * from the transaction.
7700                          */
7701                         if (nft_set_is_anonymous(nft_trans_set(trans)) &&
7702                             !list_empty(&nft_trans_set(trans)->bindings))
7703                                 trans->ctx.table->use--;
7704
7705                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
7706                                              NFT_MSG_NEWSET, GFP_KERNEL);
7707                         nft_trans_destroy(trans);
7708                         break;
7709                 case NFT_MSG_DELSET:
7710                         list_del_rcu(&nft_trans_set(trans)->list);
7711                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
7712                                              NFT_MSG_DELSET, GFP_KERNEL);
7713                         break;
7714                 case NFT_MSG_NEWSETELEM:
7715                         te = (struct nft_trans_elem *)trans->data;
7716
7717                         te->set->ops->activate(net, te->set, &te->elem);
7718                         nf_tables_setelem_notify(&trans->ctx, te->set,
7719                                                  &te->elem,
7720                                                  NFT_MSG_NEWSETELEM, 0);
7721                         nft_trans_destroy(trans);
7722                         break;
7723                 case NFT_MSG_DELSETELEM:
7724                         te = (struct nft_trans_elem *)trans->data;
7725
7726                         nf_tables_setelem_notify(&trans->ctx, te->set,
7727                                                  &te->elem,
7728                                                  NFT_MSG_DELSETELEM, 0);
7729                         te->set->ops->remove(net, te->set, &te->elem);
7730                         atomic_dec(&te->set->nelems);
7731                         te->set->ndeact--;
7732                         break;
7733                 case NFT_MSG_NEWOBJ:
7734                         if (nft_trans_obj_update(trans)) {
7735                                 nft_obj_commit_update(trans);
7736                                 nf_tables_obj_notify(&trans->ctx,
7737                                                      nft_trans_obj(trans),
7738                                                      NFT_MSG_NEWOBJ);
7739                         } else {
7740                                 nft_clear(net, nft_trans_obj(trans));
7741                                 nf_tables_obj_notify(&trans->ctx,
7742                                                      nft_trans_obj(trans),
7743                                                      NFT_MSG_NEWOBJ);
7744                                 nft_trans_destroy(trans);
7745                         }
7746                         break;
7747                 case NFT_MSG_DELOBJ:
7748                         nft_obj_del(nft_trans_obj(trans));
7749                         nf_tables_obj_notify(&trans->ctx, nft_trans_obj(trans),
7750                                              NFT_MSG_DELOBJ);
7751                         break;
7752                 case NFT_MSG_NEWFLOWTABLE:
7753                         if (nft_trans_flowtable_update(trans)) {
7754                                 nf_tables_flowtable_notify(&trans->ctx,
7755                                                            nft_trans_flowtable(trans),
7756                                                            &nft_trans_flowtable_hooks(trans),
7757                                                            NFT_MSG_NEWFLOWTABLE);
7758                                 list_splice(&nft_trans_flowtable_hooks(trans),
7759                                             &nft_trans_flowtable(trans)->hook_list);
7760                         } else {
7761                                 nft_clear(net, nft_trans_flowtable(trans));
7762                                 nf_tables_flowtable_notify(&trans->ctx,
7763                                                            nft_trans_flowtable(trans),
7764                                                            &nft_trans_flowtable(trans)->hook_list,
7765                                                            NFT_MSG_NEWFLOWTABLE);
7766                         }
7767                         nft_trans_destroy(trans);
7768                         break;
7769                 case NFT_MSG_DELFLOWTABLE:
7770                         if (nft_trans_flowtable_update(trans)) {
7771                                 nft_flowtable_hooks_del(nft_trans_flowtable(trans),
7772                                                         &nft_trans_flowtable_hooks(trans));
7773                                 nf_tables_flowtable_notify(&trans->ctx,
7774                                                            nft_trans_flowtable(trans),
7775                                                            &nft_trans_flowtable_hooks(trans),
7776                                                            NFT_MSG_DELFLOWTABLE);
7777                                 nft_unregister_flowtable_net_hooks(net,
7778                                                                    &nft_trans_flowtable_hooks(trans));
7779                         } else {
7780                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
7781                                 nf_tables_flowtable_notify(&trans->ctx,
7782                                                            nft_trans_flowtable(trans),
7783                                                            &nft_trans_flowtable(trans)->hook_list,
7784                                                            NFT_MSG_DELFLOWTABLE);
7785                                 nft_unregister_flowtable_net_hooks(net,
7786                                                 &nft_trans_flowtable(trans)->hook_list);
7787                         }
7788                         break;
7789                 }
7790         }
7791
7792         nf_tables_gen_notify(net, skb, NFT_MSG_NEWGEN);
7793         nf_tables_commit_release(net);
7794
7795         return 0;
7796 }
7797
7798 static void nf_tables_module_autoload(struct net *net)
7799 {
7800         struct nft_module_request *req, *next;
7801         LIST_HEAD(module_list);
7802
7803         list_splice_init(&net->nft.module_list, &module_list);
7804         mutex_unlock(&net->nft.commit_mutex);
7805         list_for_each_entry_safe(req, next, &module_list, list) {
7806                 request_module("%s", req->module);
7807                 req->done = true;
7808         }
7809         mutex_lock(&net->nft.commit_mutex);
7810         list_splice(&module_list, &net->nft.module_list);
7811 }
7812
7813 static void nf_tables_abort_release(struct nft_trans *trans)
7814 {
7815         switch (trans->msg_type) {
7816         case NFT_MSG_NEWTABLE:
7817                 nf_tables_table_destroy(&trans->ctx);
7818                 break;
7819         case NFT_MSG_NEWCHAIN:
7820                 nf_tables_chain_destroy(&trans->ctx);
7821                 break;
7822         case NFT_MSG_NEWRULE:
7823                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
7824                 break;
7825         case NFT_MSG_NEWSET:
7826                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
7827                 break;
7828         case NFT_MSG_NEWSETELEM:
7829                 nft_set_elem_destroy(nft_trans_elem_set(trans),
7830                                      nft_trans_elem(trans).priv, true);
7831                 break;
7832         case NFT_MSG_NEWOBJ:
7833                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
7834                 break;
7835         case NFT_MSG_NEWFLOWTABLE:
7836                 if (nft_trans_flowtable_update(trans))
7837                         nft_flowtable_hooks_destroy(&nft_trans_flowtable_hooks(trans));
7838                 else
7839                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
7840                 break;
7841         }
7842         kfree(trans);
7843 }
7844
7845 static int __nf_tables_abort(struct net *net, bool autoload)
7846 {
7847         struct nft_trans *trans, *next;
7848         struct nft_trans_elem *te;
7849         struct nft_hook *hook;
7850
7851         list_for_each_entry_safe_reverse(trans, next, &net->nft.commit_list,
7852                                          list) {
7853                 switch (trans->msg_type) {
7854                 case NFT_MSG_NEWTABLE:
7855                         if (nft_trans_table_update(trans)) {
7856                                 if (nft_trans_table_enable(trans)) {
7857                                         nf_tables_table_disable(net,
7858                                                                 trans->ctx.table);
7859                                         trans->ctx.table->flags |= NFT_TABLE_F_DORMANT;
7860                                 }
7861                                 nft_trans_destroy(trans);
7862                         } else {
7863                                 list_del_rcu(&trans->ctx.table->list);
7864                         }
7865                         break;
7866                 case NFT_MSG_DELTABLE:
7867                         nft_clear(trans->ctx.net, trans->ctx.table);
7868                         nft_trans_destroy(trans);
7869                         break;
7870                 case NFT_MSG_NEWCHAIN:
7871                         if (nft_trans_chain_update(trans)) {
7872                                 free_percpu(nft_trans_chain_stats(trans));
7873                                 kfree(nft_trans_chain_name(trans));
7874                                 nft_trans_destroy(trans);
7875                         } else {
7876                                 if (nft_chain_is_bound(trans->ctx.chain)) {
7877                                         nft_trans_destroy(trans);
7878                                         break;
7879                                 }
7880                                 trans->ctx.table->use--;
7881                                 nft_chain_del(trans->ctx.chain);
7882                                 nf_tables_unregister_hook(trans->ctx.net,
7883                                                           trans->ctx.table,
7884                                                           trans->ctx.chain);
7885                         }
7886                         break;
7887                 case NFT_MSG_DELCHAIN:
7888                         trans->ctx.table->use++;
7889                         nft_clear(trans->ctx.net, trans->ctx.chain);
7890                         nft_trans_destroy(trans);
7891                         break;
7892                 case NFT_MSG_NEWRULE:
7893                         trans->ctx.chain->use--;
7894                         list_del_rcu(&nft_trans_rule(trans)->list);
7895                         nft_rule_expr_deactivate(&trans->ctx,
7896                                                  nft_trans_rule(trans),
7897                                                  NFT_TRANS_ABORT);
7898                         break;
7899                 case NFT_MSG_DELRULE:
7900                         trans->ctx.chain->use++;
7901                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
7902                         nft_rule_expr_activate(&trans->ctx, nft_trans_rule(trans));
7903                         nft_trans_destroy(trans);
7904                         break;
7905                 case NFT_MSG_NEWSET:
7906                         trans->ctx.table->use--;
7907                         if (nft_trans_set_bound(trans)) {
7908                                 nft_trans_destroy(trans);
7909                                 break;
7910                         }
7911                         list_del_rcu(&nft_trans_set(trans)->list);
7912                         break;
7913                 case NFT_MSG_DELSET:
7914                         trans->ctx.table->use++;
7915                         nft_clear(trans->ctx.net, nft_trans_set(trans));
7916                         nft_trans_destroy(trans);
7917                         break;
7918                 case NFT_MSG_NEWSETELEM:
7919                         if (nft_trans_elem_set_bound(trans)) {
7920                                 nft_trans_destroy(trans);
7921                                 break;
7922                         }
7923                         te = (struct nft_trans_elem *)trans->data;
7924                         te->set->ops->remove(net, te->set, &te->elem);
7925                         atomic_dec(&te->set->nelems);
7926                         break;
7927                 case NFT_MSG_DELSETELEM:
7928                         te = (struct nft_trans_elem *)trans->data;
7929
7930                         nft_set_elem_activate(net, te->set, &te->elem);
7931                         te->set->ops->activate(net, te->set, &te->elem);
7932                         te->set->ndeact--;
7933
7934                         nft_trans_destroy(trans);
7935                         break;
7936                 case NFT_MSG_NEWOBJ:
7937                         if (nft_trans_obj_update(trans)) {
7938                                 kfree(nft_trans_obj_newobj(trans));
7939                                 nft_trans_destroy(trans);
7940                         } else {
7941                                 trans->ctx.table->use--;
7942                                 nft_obj_del(nft_trans_obj(trans));
7943                         }
7944                         break;
7945                 case NFT_MSG_DELOBJ:
7946                         trans->ctx.table->use++;
7947                         nft_clear(trans->ctx.net, nft_trans_obj(trans));
7948                         nft_trans_destroy(trans);
7949                         break;
7950                 case NFT_MSG_NEWFLOWTABLE:
7951                         if (nft_trans_flowtable_update(trans)) {
7952                                 nft_unregister_flowtable_net_hooks(net,
7953                                                 &nft_trans_flowtable_hooks(trans));
7954                         } else {
7955                                 trans->ctx.table->use--;
7956                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
7957                                 nft_unregister_flowtable_net_hooks(net,
7958                                                 &nft_trans_flowtable(trans)->hook_list);
7959                         }
7960                         break;
7961                 case NFT_MSG_DELFLOWTABLE:
7962                         if (nft_trans_flowtable_update(trans)) {
7963                                 list_for_each_entry(hook, &nft_trans_flowtable(trans)->hook_list, list)
7964                                         hook->inactive = false;
7965                         } else {
7966                                 trans->ctx.table->use++;
7967                                 nft_clear(trans->ctx.net, nft_trans_flowtable(trans));
7968                         }
7969                         nft_trans_destroy(trans);
7970                         break;
7971                 }
7972         }
7973
7974         synchronize_rcu();
7975
7976         list_for_each_entry_safe_reverse(trans, next,
7977                                          &net->nft.commit_list, list) {
7978                 list_del(&trans->list);
7979                 nf_tables_abort_release(trans);
7980         }
7981
7982         if (autoload)
7983                 nf_tables_module_autoload(net);
7984         else
7985                 nf_tables_module_autoload_cleanup(net);
7986
7987         return 0;
7988 }
7989
7990 static void nf_tables_cleanup(struct net *net)
7991 {
7992         nft_validate_state_update(net, NFT_VALIDATE_SKIP);
7993 }
7994
7995 static int nf_tables_abort(struct net *net, struct sk_buff *skb, bool autoload)
7996 {
7997         int ret = __nf_tables_abort(net, autoload);
7998
7999         mutex_unlock(&net->nft.commit_mutex);
8000
8001         return ret;
8002 }
8003
8004 static bool nf_tables_valid_genid(struct net *net, u32 genid)
8005 {
8006         bool genid_ok;
8007
8008         mutex_lock(&net->nft.commit_mutex);
8009
8010         genid_ok = genid == 0 || net->nft.base_seq == genid;
8011         if (!genid_ok)
8012                 mutex_unlock(&net->nft.commit_mutex);
8013
8014         /* else, commit mutex has to be released by commit or abort function */
8015         return genid_ok;
8016 }
8017
8018 static const struct nfnetlink_subsystem nf_tables_subsys = {
8019         .name           = "nf_tables",
8020         .subsys_id      = NFNL_SUBSYS_NFTABLES,
8021         .cb_count       = NFT_MSG_MAX,
8022         .cb             = nf_tables_cb,
8023         .commit         = nf_tables_commit,
8024         .abort          = nf_tables_abort,
8025         .cleanup        = nf_tables_cleanup,
8026         .valid_genid    = nf_tables_valid_genid,
8027         .owner          = THIS_MODULE,
8028 };
8029
8030 int nft_chain_validate_dependency(const struct nft_chain *chain,
8031                                   enum nft_chain_types type)
8032 {
8033         const struct nft_base_chain *basechain;
8034
8035         if (nft_is_base_chain(chain)) {
8036                 basechain = nft_base_chain(chain);
8037                 if (basechain->type->type != type)
8038                         return -EOPNOTSUPP;
8039         }
8040         return 0;
8041 }
8042 EXPORT_SYMBOL_GPL(nft_chain_validate_dependency);
8043
8044 int nft_chain_validate_hooks(const struct nft_chain *chain,
8045                              unsigned int hook_flags)
8046 {
8047         struct nft_base_chain *basechain;
8048
8049         if (nft_is_base_chain(chain)) {
8050                 basechain = nft_base_chain(chain);
8051
8052                 if ((1 << basechain->ops.hooknum) & hook_flags)
8053                         return 0;
8054
8055                 return -EOPNOTSUPP;
8056         }
8057
8058         return 0;
8059 }
8060 EXPORT_SYMBOL_GPL(nft_chain_validate_hooks);
8061
8062 /*
8063  * Loop detection - walk through the ruleset beginning at the destination chain
8064  * of a new jump until either the source chain is reached (loop) or all
8065  * reachable chains have been traversed.
8066  *
8067  * The loop check is performed whenever a new jump verdict is added to an
8068  * expression or verdict map or a verdict map is bound to a new chain.
8069  */
8070
8071 static int nf_tables_check_loops(const struct nft_ctx *ctx,
8072                                  const struct nft_chain *chain);
8073
8074 static int nf_tables_loop_check_setelem(const struct nft_ctx *ctx,
8075                                         struct nft_set *set,
8076                                         const struct nft_set_iter *iter,
8077                                         struct nft_set_elem *elem)
8078 {
8079         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
8080         const struct nft_data *data;
8081
8082         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
8083             *nft_set_ext_flags(ext) & NFT_SET_ELEM_INTERVAL_END)
8084                 return 0;
8085
8086         data = nft_set_ext_data(ext);
8087         switch (data->verdict.code) {
8088         case NFT_JUMP:
8089         case NFT_GOTO:
8090                 return nf_tables_check_loops(ctx, data->verdict.chain);
8091         default:
8092                 return 0;
8093         }
8094 }
8095
8096 static int nf_tables_check_loops(const struct nft_ctx *ctx,
8097                                  const struct nft_chain *chain)
8098 {
8099         const struct nft_rule *rule;
8100         const struct nft_expr *expr, *last;
8101         struct nft_set *set;
8102         struct nft_set_binding *binding;
8103         struct nft_set_iter iter;
8104
8105         if (ctx->chain == chain)
8106                 return -ELOOP;
8107
8108         list_for_each_entry(rule, &chain->rules, list) {
8109                 nft_rule_for_each_expr(expr, last, rule) {
8110                         struct nft_immediate_expr *priv;
8111                         const struct nft_data *data;
8112                         int err;
8113
8114                         if (strcmp(expr->ops->type->name, "immediate"))
8115                                 continue;
8116
8117                         priv = nft_expr_priv(expr);
8118                         if (priv->dreg != NFT_REG_VERDICT)
8119                                 continue;
8120
8121                         data = &priv->data;
8122                         switch (data->verdict.code) {
8123                         case NFT_JUMP:
8124                         case NFT_GOTO:
8125                                 err = nf_tables_check_loops(ctx,
8126                                                         data->verdict.chain);
8127                                 if (err < 0)
8128                                         return err;
8129                         default:
8130                                 break;
8131                         }
8132                 }
8133         }
8134
8135         list_for_each_entry(set, &ctx->table->sets, list) {
8136                 if (!nft_is_active_next(ctx->net, set))
8137                         continue;
8138                 if (!(set->flags & NFT_SET_MAP) ||
8139                     set->dtype != NFT_DATA_VERDICT)
8140                         continue;
8141
8142                 list_for_each_entry(binding, &set->bindings, list) {
8143                         if (!(binding->flags & NFT_SET_MAP) ||
8144                             binding->chain != chain)
8145                                 continue;
8146
8147                         iter.genmask    = nft_genmask_next(ctx->net);
8148                         iter.skip       = 0;
8149                         iter.count      = 0;
8150                         iter.err        = 0;
8151                         iter.fn         = nf_tables_loop_check_setelem;
8152
8153                         set->ops->walk(ctx, set, &iter);
8154                         if (iter.err < 0)
8155                                 return iter.err;
8156                 }
8157         }
8158
8159         return 0;
8160 }
8161
8162 /**
8163  *      nft_parse_u32_check - fetch u32 attribute and check for maximum value
8164  *
8165  *      @attr: netlink attribute to fetch value from
8166  *      @max: maximum value to be stored in dest
8167  *      @dest: pointer to the variable
8168  *
8169  *      Parse, check and store a given u32 netlink attribute into variable.
8170  *      This function returns -ERANGE if the value goes over maximum value.
8171  *      Otherwise a 0 is returned and the attribute value is stored in the
8172  *      destination variable.
8173  */
8174 int nft_parse_u32_check(const struct nlattr *attr, int max, u32 *dest)
8175 {
8176         u32 val;
8177
8178         val = ntohl(nla_get_be32(attr));
8179         if (val > max)
8180                 return -ERANGE;
8181
8182         *dest = val;
8183         return 0;
8184 }
8185 EXPORT_SYMBOL_GPL(nft_parse_u32_check);
8186
8187 /**
8188  *      nft_parse_register - parse a register value from a netlink attribute
8189  *
8190  *      @attr: netlink attribute
8191  *
8192  *      Parse and translate a register value from a netlink attribute.
8193  *      Registers used to be 128 bit wide, these register numbers will be
8194  *      mapped to the corresponding 32 bit register numbers.
8195  */
8196 unsigned int nft_parse_register(const struct nlattr *attr)
8197 {
8198         unsigned int reg;
8199
8200         reg = ntohl(nla_get_be32(attr));
8201         switch (reg) {
8202         case NFT_REG_VERDICT...NFT_REG_4:
8203                 return reg * NFT_REG_SIZE / NFT_REG32_SIZE;
8204         default:
8205                 return reg + NFT_REG_SIZE / NFT_REG32_SIZE - NFT_REG32_00;
8206         }
8207 }
8208 EXPORT_SYMBOL_GPL(nft_parse_register);
8209
8210 /**
8211  *      nft_dump_register - dump a register value to a netlink attribute
8212  *
8213  *      @skb: socket buffer
8214  *      @attr: attribute number
8215  *      @reg: register number
8216  *
8217  *      Construct a netlink attribute containing the register number. For
8218  *      compatibility reasons, register numbers being a multiple of 4 are
8219  *      translated to the corresponding 128 bit register numbers.
8220  */
8221 int nft_dump_register(struct sk_buff *skb, unsigned int attr, unsigned int reg)
8222 {
8223         if (reg % (NFT_REG_SIZE / NFT_REG32_SIZE) == 0)
8224                 reg = reg / (NFT_REG_SIZE / NFT_REG32_SIZE);
8225         else
8226                 reg = reg - NFT_REG_SIZE / NFT_REG32_SIZE + NFT_REG32_00;
8227
8228         return nla_put_be32(skb, attr, htonl(reg));
8229 }
8230 EXPORT_SYMBOL_GPL(nft_dump_register);
8231
8232 /**
8233  *      nft_validate_register_load - validate a load from a register
8234  *
8235  *      @reg: the register number
8236  *      @len: the length of the data
8237  *
8238  *      Validate that the input register is one of the general purpose
8239  *      registers and that the length of the load is within the bounds.
8240  */
8241 int nft_validate_register_load(enum nft_registers reg, unsigned int len)
8242 {
8243         if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
8244                 return -EINVAL;
8245         if (len == 0)
8246                 return -EINVAL;
8247         if (reg * NFT_REG32_SIZE + len > sizeof_field(struct nft_regs, data))
8248                 return -ERANGE;
8249
8250         return 0;
8251 }
8252 EXPORT_SYMBOL_GPL(nft_validate_register_load);
8253
8254 /**
8255  *      nft_validate_register_store - validate an expressions' register store
8256  *
8257  *      @ctx: context of the expression performing the load
8258  *      @reg: the destination register number
8259  *      @data: the data to load
8260  *      @type: the data type
8261  *      @len: the length of the data
8262  *
8263  *      Validate that a data load uses the appropriate data type for
8264  *      the destination register and the length is within the bounds.
8265  *      A value of NULL for the data means that its runtime gathered
8266  *      data.
8267  */
8268 int nft_validate_register_store(const struct nft_ctx *ctx,
8269                                 enum nft_registers reg,
8270                                 const struct nft_data *data,
8271                                 enum nft_data_types type, unsigned int len)
8272 {
8273         int err;
8274
8275         switch (reg) {
8276         case NFT_REG_VERDICT:
8277                 if (type != NFT_DATA_VERDICT)
8278                         return -EINVAL;
8279
8280                 if (data != NULL &&
8281                     (data->verdict.code == NFT_GOTO ||
8282                      data->verdict.code == NFT_JUMP)) {
8283                         err = nf_tables_check_loops(ctx, data->verdict.chain);
8284                         if (err < 0)
8285                                 return err;
8286                 }
8287
8288                 return 0;
8289         default:
8290                 if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
8291                         return -EINVAL;
8292                 if (len == 0)
8293                         return -EINVAL;
8294                 if (reg * NFT_REG32_SIZE + len >
8295                     sizeof_field(struct nft_regs, data))
8296                         return -ERANGE;
8297
8298                 if (data != NULL && type != NFT_DATA_VALUE)
8299                         return -EINVAL;
8300                 return 0;
8301         }
8302 }
8303 EXPORT_SYMBOL_GPL(nft_validate_register_store);
8304
8305 static const struct nla_policy nft_verdict_policy[NFTA_VERDICT_MAX + 1] = {
8306         [NFTA_VERDICT_CODE]     = { .type = NLA_U32 },
8307         [NFTA_VERDICT_CHAIN]    = { .type = NLA_STRING,
8308                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
8309         [NFTA_VERDICT_CHAIN_ID] = { .type = NLA_U32 },
8310 };
8311
8312 static int nft_verdict_init(const struct nft_ctx *ctx, struct nft_data *data,
8313                             struct nft_data_desc *desc, const struct nlattr *nla)
8314 {
8315         u8 genmask = nft_genmask_next(ctx->net);
8316         struct nlattr *tb[NFTA_VERDICT_MAX + 1];
8317         struct nft_chain *chain;
8318         int err;
8319
8320         err = nla_parse_nested_deprecated(tb, NFTA_VERDICT_MAX, nla,
8321                                           nft_verdict_policy, NULL);
8322         if (err < 0)
8323                 return err;
8324
8325         if (!tb[NFTA_VERDICT_CODE])
8326                 return -EINVAL;
8327         data->verdict.code = ntohl(nla_get_be32(tb[NFTA_VERDICT_CODE]));
8328
8329         switch (data->verdict.code) {
8330         default:
8331                 switch (data->verdict.code & NF_VERDICT_MASK) {
8332                 case NF_ACCEPT:
8333                 case NF_DROP:
8334                 case NF_QUEUE:
8335                         break;
8336                 default:
8337                         return -EINVAL;
8338                 }
8339                 /* fall through */
8340         case NFT_CONTINUE:
8341         case NFT_BREAK:
8342         case NFT_RETURN:
8343                 break;
8344         case NFT_JUMP:
8345         case NFT_GOTO:
8346                 if (tb[NFTA_VERDICT_CHAIN]) {
8347                         chain = nft_chain_lookup(ctx->net, ctx->table,
8348                                                  tb[NFTA_VERDICT_CHAIN],
8349                                                  genmask);
8350                 } else if (tb[NFTA_VERDICT_CHAIN_ID]) {
8351                         chain = nft_chain_lookup_byid(ctx->net,
8352                                                       tb[NFTA_VERDICT_CHAIN_ID]);
8353                         if (IS_ERR(chain))
8354                                 return PTR_ERR(chain);
8355                 } else {
8356                         return -EINVAL;
8357                 }
8358
8359                 if (IS_ERR(chain))
8360                         return PTR_ERR(chain);
8361                 if (nft_is_base_chain(chain))
8362                         return -EOPNOTSUPP;
8363
8364                 chain->use++;
8365                 data->verdict.chain = chain;
8366                 break;
8367         }
8368
8369         desc->len = sizeof(data->verdict);
8370         desc->type = NFT_DATA_VERDICT;
8371         return 0;
8372 }
8373
8374 static void nft_verdict_uninit(const struct nft_data *data)
8375 {
8376         struct nft_chain *chain;
8377         struct nft_rule *rule;
8378
8379         switch (data->verdict.code) {
8380         case NFT_JUMP:
8381         case NFT_GOTO:
8382                 chain = data->verdict.chain;
8383                 chain->use--;
8384
8385                 if (!nft_chain_is_bound(chain))
8386                         break;
8387
8388                 chain->table->use--;
8389                 list_for_each_entry(rule, &chain->rules, list)
8390                         chain->use--;
8391
8392                 nft_chain_del(chain);
8393                 break;
8394         }
8395 }
8396
8397 int nft_verdict_dump(struct sk_buff *skb, int type, const struct nft_verdict *v)
8398 {
8399         struct nlattr *nest;
8400
8401         nest = nla_nest_start_noflag(skb, type);
8402         if (!nest)
8403                 goto nla_put_failure;
8404
8405         if (nla_put_be32(skb, NFTA_VERDICT_CODE, htonl(v->code)))
8406                 goto nla_put_failure;
8407
8408         switch (v->code) {
8409         case NFT_JUMP:
8410         case NFT_GOTO:
8411                 if (nla_put_string(skb, NFTA_VERDICT_CHAIN,
8412                                    v->chain->name))
8413                         goto nla_put_failure;
8414         }
8415         nla_nest_end(skb, nest);
8416         return 0;
8417
8418 nla_put_failure:
8419         return -1;
8420 }
8421
8422 static int nft_value_init(const struct nft_ctx *ctx,
8423                           struct nft_data *data, unsigned int size,
8424                           struct nft_data_desc *desc, const struct nlattr *nla)
8425 {
8426         unsigned int len;
8427
8428         len = nla_len(nla);
8429         if (len == 0)
8430                 return -EINVAL;
8431         if (len > size)
8432                 return -EOVERFLOW;
8433
8434         nla_memcpy(data->data, nla, len);
8435         desc->type = NFT_DATA_VALUE;
8436         desc->len  = len;
8437         return 0;
8438 }
8439
8440 static int nft_value_dump(struct sk_buff *skb, const struct nft_data *data,
8441                           unsigned int len)
8442 {
8443         return nla_put(skb, NFTA_DATA_VALUE, len, data->data);
8444 }
8445
8446 static const struct nla_policy nft_data_policy[NFTA_DATA_MAX + 1] = {
8447         [NFTA_DATA_VALUE]       = { .type = NLA_BINARY },
8448         [NFTA_DATA_VERDICT]     = { .type = NLA_NESTED },
8449 };
8450
8451 /**
8452  *      nft_data_init - parse nf_tables data netlink attributes
8453  *
8454  *      @ctx: context of the expression using the data
8455  *      @data: destination struct nft_data
8456  *      @size: maximum data length
8457  *      @desc: data description
8458  *      @nla: netlink attribute containing data
8459  *
8460  *      Parse the netlink data attributes and initialize a struct nft_data.
8461  *      The type and length of data are returned in the data description.
8462  *
8463  *      The caller can indicate that it only wants to accept data of type
8464  *      NFT_DATA_VALUE by passing NULL for the ctx argument.
8465  */
8466 int nft_data_init(const struct nft_ctx *ctx,
8467                   struct nft_data *data, unsigned int size,
8468                   struct nft_data_desc *desc, const struct nlattr *nla)
8469 {
8470         struct nlattr *tb[NFTA_DATA_MAX + 1];
8471         int err;
8472
8473         err = nla_parse_nested_deprecated(tb, NFTA_DATA_MAX, nla,
8474                                           nft_data_policy, NULL);
8475         if (err < 0)
8476                 return err;
8477
8478         if (tb[NFTA_DATA_VALUE])
8479                 return nft_value_init(ctx, data, size, desc,
8480                                       tb[NFTA_DATA_VALUE]);
8481         if (tb[NFTA_DATA_VERDICT] && ctx != NULL)
8482                 return nft_verdict_init(ctx, data, desc, tb[NFTA_DATA_VERDICT]);
8483         return -EINVAL;
8484 }
8485 EXPORT_SYMBOL_GPL(nft_data_init);
8486
8487 /**
8488  *      nft_data_release - release a nft_data item
8489  *
8490  *      @data: struct nft_data to release
8491  *      @type: type of data
8492  *
8493  *      Release a nft_data item. NFT_DATA_VALUE types can be silently discarded,
8494  *      all others need to be released by calling this function.
8495  */
8496 void nft_data_release(const struct nft_data *data, enum nft_data_types type)
8497 {
8498         if (type < NFT_DATA_VERDICT)
8499                 return;
8500         switch (type) {
8501         case NFT_DATA_VERDICT:
8502                 return nft_verdict_uninit(data);
8503         default:
8504                 WARN_ON(1);
8505         }
8506 }
8507 EXPORT_SYMBOL_GPL(nft_data_release);
8508
8509 int nft_data_dump(struct sk_buff *skb, int attr, const struct nft_data *data,
8510                   enum nft_data_types type, unsigned int len)
8511 {
8512         struct nlattr *nest;
8513         int err;
8514
8515         nest = nla_nest_start_noflag(skb, attr);
8516         if (nest == NULL)
8517                 return -1;
8518
8519         switch (type) {
8520         case NFT_DATA_VALUE:
8521                 err = nft_value_dump(skb, data, len);
8522                 break;
8523         case NFT_DATA_VERDICT:
8524                 err = nft_verdict_dump(skb, NFTA_DATA_VERDICT, &data->verdict);
8525                 break;
8526         default:
8527                 err = -EINVAL;
8528                 WARN_ON(1);
8529         }
8530
8531         nla_nest_end(skb, nest);
8532         return err;
8533 }
8534 EXPORT_SYMBOL_GPL(nft_data_dump);
8535
8536 int __nft_release_basechain(struct nft_ctx *ctx)
8537 {
8538         struct nft_rule *rule, *nr;
8539
8540         if (WARN_ON(!nft_is_base_chain(ctx->chain)))
8541                 return 0;
8542
8543         nf_tables_unregister_hook(ctx->net, ctx->chain->table, ctx->chain);
8544         list_for_each_entry_safe(rule, nr, &ctx->chain->rules, list) {
8545                 list_del(&rule->list);
8546                 ctx->chain->use--;
8547                 nf_tables_rule_release(ctx, rule);
8548         }
8549         nft_chain_del(ctx->chain);
8550         ctx->table->use--;
8551         nf_tables_chain_destroy(ctx);
8552
8553         return 0;
8554 }
8555 EXPORT_SYMBOL_GPL(__nft_release_basechain);
8556
8557 static void __nft_release_tables(struct net *net)
8558 {
8559         struct nft_flowtable *flowtable, *nf;
8560         struct nft_table *table, *nt;
8561         struct nft_chain *chain, *nc;
8562         struct nft_object *obj, *ne;
8563         struct nft_rule *rule, *nr;
8564         struct nft_set *set, *ns;
8565         struct nft_ctx ctx = {
8566                 .net    = net,
8567                 .family = NFPROTO_NETDEV,
8568         };
8569
8570         list_for_each_entry_safe(table, nt, &net->nft.tables, list) {
8571                 ctx.family = table->family;
8572
8573                 list_for_each_entry(chain, &table->chains, list)
8574                         nf_tables_unregister_hook(net, table, chain);
8575                 /* No packets are walking on these chains anymore. */
8576                 ctx.table = table;
8577                 list_for_each_entry(chain, &table->chains, list) {
8578                         ctx.chain = chain;
8579                         list_for_each_entry_safe(rule, nr, &chain->rules, list) {
8580                                 list_del(&rule->list);
8581                                 chain->use--;
8582                                 nf_tables_rule_release(&ctx, rule);
8583                         }
8584                 }
8585                 list_for_each_entry_safe(flowtable, nf, &table->flowtables, list) {
8586                         list_del(&flowtable->list);
8587                         table->use--;
8588                         nf_tables_flowtable_destroy(flowtable);
8589                 }
8590                 list_for_each_entry_safe(set, ns, &table->sets, list) {
8591                         list_del(&set->list);
8592                         table->use--;
8593                         nft_set_destroy(&ctx, set);
8594                 }
8595                 list_for_each_entry_safe(obj, ne, &table->objects, list) {
8596                         nft_obj_del(obj);
8597                         table->use--;
8598                         nft_obj_destroy(&ctx, obj);
8599                 }
8600                 list_for_each_entry_safe(chain, nc, &table->chains, list) {
8601                         ctx.chain = chain;
8602                         nft_chain_del(chain);
8603                         table->use--;
8604                         nf_tables_chain_destroy(&ctx);
8605                 }
8606                 list_del(&table->list);
8607                 nf_tables_table_destroy(&ctx);
8608         }
8609 }
8610
8611 static int __net_init nf_tables_init_net(struct net *net)
8612 {
8613         INIT_LIST_HEAD(&net->nft.tables);
8614         INIT_LIST_HEAD(&net->nft.commit_list);
8615         INIT_LIST_HEAD(&net->nft.module_list);
8616         mutex_init(&net->nft.commit_mutex);
8617         net->nft.base_seq = 1;
8618         net->nft.validate_state = NFT_VALIDATE_SKIP;
8619
8620         return 0;
8621 }
8622
8623 static void __net_exit nf_tables_exit_net(struct net *net)
8624 {
8625         mutex_lock(&net->nft.commit_mutex);
8626         if (!list_empty(&net->nft.commit_list))
8627                 __nf_tables_abort(net, false);
8628         __nft_release_tables(net);
8629         mutex_unlock(&net->nft.commit_mutex);
8630         WARN_ON_ONCE(!list_empty(&net->nft.tables));
8631         WARN_ON_ONCE(!list_empty(&net->nft.module_list));
8632 }
8633
8634 static struct pernet_operations nf_tables_net_ops = {
8635         .init   = nf_tables_init_net,
8636         .exit   = nf_tables_exit_net,
8637 };
8638
8639 static int __init nf_tables_module_init(void)
8640 {
8641         int err;
8642
8643         spin_lock_init(&nf_tables_destroy_list_lock);
8644         err = register_pernet_subsys(&nf_tables_net_ops);
8645         if (err < 0)
8646                 return err;
8647
8648         err = nft_chain_filter_init();
8649         if (err < 0)
8650                 goto err1;
8651
8652         err = nf_tables_core_module_init();
8653         if (err < 0)
8654                 goto err2;
8655
8656         err = register_netdevice_notifier(&nf_tables_flowtable_notifier);
8657         if (err < 0)
8658                 goto err3;
8659
8660         err = rhltable_init(&nft_objname_ht, &nft_objname_ht_params);
8661         if (err < 0)
8662                 goto err4;
8663
8664         err = nft_offload_init();
8665         if (err < 0)
8666                 goto err5;
8667
8668         /* must be last */
8669         err = nfnetlink_subsys_register(&nf_tables_subsys);
8670         if (err < 0)
8671                 goto err6;
8672
8673         nft_chain_route_init();
8674
8675         return err;
8676 err6:
8677         nft_offload_exit();
8678 err5:
8679         rhltable_destroy(&nft_objname_ht);
8680 err4:
8681         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
8682 err3:
8683         nf_tables_core_module_exit();
8684 err2:
8685         nft_chain_filter_fini();
8686 err1:
8687         unregister_pernet_subsys(&nf_tables_net_ops);
8688         return err;
8689 }
8690
8691 static void __exit nf_tables_module_exit(void)
8692 {
8693         nfnetlink_subsys_unregister(&nf_tables_subsys);
8694         nft_offload_exit();
8695         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
8696         nft_chain_filter_fini();
8697         nft_chain_route_fini();
8698         unregister_pernet_subsys(&nf_tables_net_ops);
8699         cancel_work_sync(&trans_destroy_work);
8700         rcu_barrier();
8701         rhltable_destroy(&nft_objname_ht);
8702         nf_tables_core_module_exit();
8703 }
8704
8705 module_init(nf_tables_module_init);
8706 module_exit(nf_tables_module_exit);
8707
8708 MODULE_LICENSE("GPL");
8709 MODULE_AUTHOR("Patrick McHardy <[email protected]>");
8710 MODULE_ALIAS_NFNL_SUBSYS(NFNL_SUBSYS_NFTABLES);