net/netfilter/nf_tables_api.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2007-2009 Patrick McHardy <kaber@trash.net>
   4  *
   5  * Development of this code funded by Astaro AG (http://www.astaro.com/)
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/init.h>
  10 #include <linux/list.h>
  11 #include <linux/skbuff.h>
  12 #include <linux/netlink.h>
  13 #include <linux/vmalloc.h>
  14 #include <linux/rhashtable.h>
  15 #include <linux/audit.h>
  16 #include <linux/netfilter.h>
  17 #include <linux/netfilter/nfnetlink.h>
  18 #include <linux/netfilter/nf_tables.h>
  19 #include <net/netfilter/nf_flow_table.h>
  20 #include <net/netfilter/nf_tables_core.h>
  21 #include <net/netfilter/nf_tables.h>
  22 #include <net/netfilter/nf_tables_offload.h>
  23 #include <net/net_namespace.h>
  24 #include <net/sock.h>
  25
  26 #define NFT_MODULE_AUTOLOAD_LIMIT (MODULE_NAME_LEN - sizeof("nft-expr-255-"))
  27
  28 unsigned int nf_tables_net_id __read_mostly;
  29
  30 static LIST_HEAD(nf_tables_expressions);
  31 static LIST_HEAD(nf_tables_objects);
  32 static LIST_HEAD(nf_tables_flowtables);
  33 static LIST_HEAD(nf_tables_destroy_list);
  34 static DEFINE_SPINLOCK(nf_tables_destroy_list_lock);
  35
  36 enum {
  37         NFT_VALIDATE_SKIP       = 0,
  38         NFT_VALIDATE_NEED,
  39         NFT_VALIDATE_DO,
  40 };
  41
  42 static struct rhltable nft_objname_ht;
  43
  44 static u32 nft_chain_hash(const void *data, u32 len, u32 seed);
  45 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed);
  46 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *, const void *);
  47
  48 static u32 nft_objname_hash(const void *data, u32 len, u32 seed);
  49 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed);
  50 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *, const void *);
  51
  52 static const struct rhashtable_params nft_chain_ht_params = {
  53         .head_offset            = offsetof(struct nft_chain, rhlhead),
  54         .key_offset             = offsetof(struct nft_chain, name),
  55         .hashfn                 = nft_chain_hash,
  56         .obj_hashfn             = nft_chain_hash_obj,
  57         .obj_cmpfn              = nft_chain_hash_cmp,
  58         .automatic_shrinking    = true,
  59 };
  60
  61 static const struct rhashtable_params nft_objname_ht_params = {
  62         .head_offset            = offsetof(struct nft_object, rhlhead),
  63         .key_offset             = offsetof(struct nft_object, key),
  64         .hashfn                 = nft_objname_hash,
  65         .obj_hashfn             = nft_objname_hash_obj,
  66         .obj_cmpfn              = nft_objname_hash_cmp,
  67         .automatic_shrinking    = true,
  68 };
  69
  70 struct nft_audit_data {
  71         struct nft_table *table;
  72         int entries;
  73         int op;
  74         struct list_head list;
  75 };
  76
  77 static const u8 nft2audit_op[NFT_MSG_MAX] = { // enum nf_tables_msg_types
  78         [NFT_MSG_NEWTABLE]      = AUDIT_NFT_OP_TABLE_REGISTER,
  79         [NFT_MSG_GETTABLE]      = AUDIT_NFT_OP_INVALID,
  80         [NFT_MSG_DELTABLE]      = AUDIT_NFT_OP_TABLE_UNREGISTER,
  81         [NFT_MSG_NEWCHAIN]      = AUDIT_NFT_OP_CHAIN_REGISTER,
  82         [NFT_MSG_GETCHAIN]      = AUDIT_NFT_OP_INVALID,
  83         [NFT_MSG_DELCHAIN]      = AUDIT_NFT_OP_CHAIN_UNREGISTER,
  84         [NFT_MSG_NEWRULE]       = AUDIT_NFT_OP_RULE_REGISTER,
  85         [NFT_MSG_GETRULE]       = AUDIT_NFT_OP_INVALID,
  86         [NFT_MSG_DELRULE]       = AUDIT_NFT_OP_RULE_UNREGISTER,
  87         [NFT_MSG_NEWSET]        = AUDIT_NFT_OP_SET_REGISTER,
  88         [NFT_MSG_GETSET]        = AUDIT_NFT_OP_INVALID,
  89         [NFT_MSG_DELSET]        = AUDIT_NFT_OP_SET_UNREGISTER,
  90         [NFT_MSG_NEWSETELEM]    = AUDIT_NFT_OP_SETELEM_REGISTER,
  91         [NFT_MSG_GETSETELEM]    = AUDIT_NFT_OP_INVALID,
  92         [NFT_MSG_DELSETELEM]    = AUDIT_NFT_OP_SETELEM_UNREGISTER,
  93         [NFT_MSG_NEWGEN]        = AUDIT_NFT_OP_GEN_REGISTER,
  94         [NFT_MSG_GETGEN]        = AUDIT_NFT_OP_INVALID,
  95         [NFT_MSG_TRACE]         = AUDIT_NFT_OP_INVALID,
  96         [NFT_MSG_NEWOBJ]        = AUDIT_NFT_OP_OBJ_REGISTER,
  97         [NFT_MSG_GETOBJ]        = AUDIT_NFT_OP_INVALID,
  98         [NFT_MSG_DELOBJ]        = AUDIT_NFT_OP_OBJ_UNREGISTER,
  99         [NFT_MSG_GETOBJ_RESET]  = AUDIT_NFT_OP_OBJ_RESET,
 100         [NFT_MSG_NEWFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_REGISTER,
 101         [NFT_MSG_GETFLOWTABLE]  = AUDIT_NFT_OP_INVALID,
 102         [NFT_MSG_DELFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_UNREGISTER,
 103 };
 104
 105 static void nft_validate_state_update(struct nft_table *table, u8 new_validate_state)
 106 {
 107         switch (table->validate_state) {
 108         case NFT_VALIDATE_SKIP:
 109                 WARN_ON_ONCE(new_validate_state == NFT_VALIDATE_DO);
 110                 break;
 111         case NFT_VALIDATE_NEED:
 112                 break;
 113         case NFT_VALIDATE_DO:
 114                 if (new_validate_state == NFT_VALIDATE_NEED)
 115                         return;
 116         }
 117
 118         table->validate_state = new_validate_state;
 119 }
 120 static void nf_tables_trans_destroy_work(struct work_struct *w);
 121 static DECLARE_WORK(trans_destroy_work, nf_tables_trans_destroy_work);
 122
 123 static void nft_ctx_init(struct nft_ctx *ctx,
 124                          struct net *net,
 125                          const struct sk_buff *skb,
 126                          const struct nlmsghdr *nlh,
 127                          u8 family,
 128                          struct nft_table *table,
 129                          struct nft_chain *chain,
 130                          const struct nlattr * const *nla)
 131 {
 132         ctx->net        = net;
 133         ctx->family     = family;
 134         ctx->level      = 0;
 135         ctx->table      = table;
 136         ctx->chain      = chain;
 137         ctx->nla        = nla;
 138         ctx->portid     = NETLINK_CB(skb).portid;
 139         ctx->report     = nlmsg_report(nlh);
 140         ctx->flags      = nlh->nlmsg_flags;
 141         ctx->seq        = nlh->nlmsg_seq;
 142 }
 143
 144 static struct nft_trans *nft_trans_alloc_gfp(const struct nft_ctx *ctx,
 145                                              int msg_type, u32 size, gfp_t gfp)
 146 {
 147         struct nft_trans *trans;
 148
 149         trans = kzalloc(sizeof(struct nft_trans) + size, gfp);
 150         if (trans == NULL)
 151                 return NULL;
 152
 153         INIT_LIST_HEAD(&trans->list);
 154         trans->msg_type = msg_type;
 155         trans->ctx      = *ctx;
 156
 157         return trans;
 158 }
 159
 160 static struct nft_trans *nft_trans_alloc(const struct nft_ctx *ctx,
 161                                          int msg_type, u32 size)
 162 {
 163         return nft_trans_alloc_gfp(ctx, msg_type, size, GFP_KERNEL);
 164 }
 165
 166 static void nft_trans_destroy(struct nft_trans *trans)
 167 {
 168         list_del(&trans->list);
 169         kfree(trans);
 170 }
 171
 172 static void nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set)
 173 {
 174         struct nftables_pernet *nft_net;
 175         struct net *net = ctx->net;
 176         struct nft_trans *trans;
 177
 178         if (!nft_set_is_anonymous(set))
 179                 return;
 180
 181         nft_net = nft_pernet(net);
 182         list_for_each_entry_reverse(trans, &nft_net->commit_list, list) {
 183                 switch (trans->msg_type) {
 184                 case NFT_MSG_NEWSET:
 185                         if (nft_trans_set(trans) == set)
 186                                 nft_trans_set_bound(trans) = true;
 187                         break;
 188                 case NFT_MSG_NEWSETELEM:
 189                         if (nft_trans_elem_set(trans) == set)
 190                                 nft_trans_elem_set_bound(trans) = true;
 191                         break;
 192                 }
 193         }
 194 }
 195
 196 static int nft_netdev_register_hooks(struct net *net,
 197                                      struct list_head *hook_list)
 198 {
 199         struct nft_hook *hook;
 200         int err, j;
 201
 202         j = 0;
 203         list_for_each_entry(hook, hook_list, list) {
 204                 err = nf_register_net_hook(net, &hook->ops);
 205                 if (err < 0)
 206                         goto err_register;
 207
 208                 j++;
 209         }
 210         return 0;
 211
 212 err_register:
 213         list_for_each_entry(hook, hook_list, list) {
 214                 if (j-- <= 0)
 215                         break;
 216
 217                 nf_unregister_net_hook(net, &hook->ops);
 218         }
 219         return err;
 220 }
 221
 222 static void nft_netdev_unregister_hooks(struct net *net,
 223                                         struct list_head *hook_list,
 224                                         bool release_netdev)
 225 {
 226         struct nft_hook *hook, *next;
 227
 228         list_for_each_entry_safe(hook, next, hook_list, list) {
 229                 nf_unregister_net_hook(net, &hook->ops);
 230                 if (release_netdev) {
 231                         list_del(&hook->list);
 232                         kfree_rcu(hook, rcu);
 233                 }
 234         }
 235 }
 236
 237 static int nf_tables_register_hook(struct net *net,
 238                                    const struct nft_table *table,
 239                                    struct nft_chain *chain)
 240 {
 241         struct nft_base_chain *basechain;
 242         const struct nf_hook_ops *ops;
 243
 244         if (table->flags & NFT_TABLE_F_DORMANT ||
 245             !nft_is_base_chain(chain))
 246                 return 0;
 247
 248         basechain = nft_base_chain(chain);
 249         ops = &basechain->ops;
 250
 251         if (basechain->type->ops_register)
 252                 return basechain->type->ops_register(net, ops);
 253
 254         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 255                 return nft_netdev_register_hooks(net, &basechain->hook_list);
 256
 257         return nf_register_net_hook(net, &basechain->ops);
 258 }
 259
 260 static void __nf_tables_unregister_hook(struct net *net,
 261                                         const struct nft_table *table,
 262                                         struct nft_chain *chain,
 263                                         bool release_netdev)
 264 {
 265         struct nft_base_chain *basechain;
 266         const struct nf_hook_ops *ops;
 267
 268         if (table->flags & NFT_TABLE_F_DORMANT ||
 269             !nft_is_base_chain(chain))
 270                 return;
 271         basechain = nft_base_chain(chain);
 272         ops = &basechain->ops;
 273
 274         if (basechain->type->ops_unregister)
 275                 return basechain->type->ops_unregister(net, ops);
 276
 277         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 278                 nft_netdev_unregister_hooks(net, &basechain->hook_list,
 279                                             release_netdev);
 280         else
 281                 nf_unregister_net_hook(net, &basechain->ops);
 282 }
 283
 284 static void nf_tables_unregister_hook(struct net *net,
 285                                       const struct nft_table *table,
 286                                       struct nft_chain *chain)
 287 {
 288         return __nf_tables_unregister_hook(net, table, chain, false);
 289 }
 290
 291 static void nft_trans_commit_list_add_tail(struct net *net, struct nft_trans *trans)
 292 {
 293         struct nftables_pernet *nft_net = nft_pernet(net);
 294
 295         list_add_tail(&trans->list, &nft_net->commit_list);
 296 }
 297
 298 static int nft_trans_table_add(struct nft_ctx *ctx, int msg_type)
 299 {
 300         struct nft_trans *trans;
 301
 302         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_table));
 303         if (trans == NULL)
 304                 return -ENOMEM;
 305
 306         if (msg_type == NFT_MSG_NEWTABLE)
 307                 nft_activate_next(ctx->net, ctx->table);
 308
 309         nft_trans_commit_list_add_tail(ctx->net, trans);
 310         return 0;
 311 }
 312
 313 static int nft_deltable(struct nft_ctx *ctx)
 314 {
 315         int err;
 316
 317         err = nft_trans_table_add(ctx, NFT_MSG_DELTABLE);
 318         if (err < 0)
 319                 return err;
 320
 321         nft_deactivate_next(ctx->net, ctx->table);
 322         return err;
 323 }
 324
 325 static struct nft_trans *nft_trans_chain_add(struct nft_ctx *ctx, int msg_type)
 326 {
 327         struct nft_trans *trans;
 328
 329         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_chain));
 330         if (trans == NULL)
 331                 return ERR_PTR(-ENOMEM);
 332
 333         if (msg_type == NFT_MSG_NEWCHAIN) {
 334                 nft_activate_next(ctx->net, ctx->chain);
 335
 336                 if (ctx->nla[NFTA_CHAIN_ID]) {
 337                         nft_trans_chain_id(trans) =
 338                                 ntohl(nla_get_be32(ctx->nla[NFTA_CHAIN_ID]));
 339                 }
 340         }
 341
 342         nft_trans_commit_list_add_tail(ctx->net, trans);
 343         return trans;
 344 }
 345
 346 static int nft_delchain(struct nft_ctx *ctx)
 347 {
 348         struct nft_trans *trans;
 349
 350         trans = nft_trans_chain_add(ctx, NFT_MSG_DELCHAIN);
 351         if (IS_ERR(trans))
 352                 return PTR_ERR(trans);
 353
 354         ctx->table->use--;
 355         nft_deactivate_next(ctx->net, ctx->chain);
 356
 357         return 0;
 358 }
 359
 360 static void nft_rule_expr_activate(const struct nft_ctx *ctx,
 361                                    struct nft_rule *rule)
 362 {
 363         struct nft_expr *expr;
 364
 365         expr = nft_expr_first(rule);
 366         while (nft_expr_more(rule, expr)) {
 367                 if (expr->ops->activate)
 368                         expr->ops->activate(ctx, expr);
 369
 370                 expr = nft_expr_next(expr);
 371         }
 372 }
 373
 374 static void nft_rule_expr_deactivate(const struct nft_ctx *ctx,
 375                                      struct nft_rule *rule,
 376                                      enum nft_trans_phase phase)
 377 {
 378         struct nft_expr *expr;
 379
 380         expr = nft_expr_first(rule);
 381         while (nft_expr_more(rule, expr)) {
 382                 if (expr->ops->deactivate)
 383                         expr->ops->deactivate(ctx, expr, phase);
 384
 385                 expr = nft_expr_next(expr);
 386         }
 387 }
 388
 389 static int
 390 nf_tables_delrule_deactivate(struct nft_ctx *ctx, struct nft_rule *rule)
 391 {
 392         /* You cannot delete the same rule twice */
 393         if (nft_is_active_next(ctx->net, rule)) {
 394                 nft_deactivate_next(ctx->net, rule);
 395                 ctx->chain->use--;
 396                 return 0;
 397         }
 398         return -ENOENT;
 399 }
 400
 401 static struct nft_trans *nft_trans_rule_add(struct nft_ctx *ctx, int msg_type,
 402                                             struct nft_rule *rule)
 403 {
 404         struct nft_trans *trans;
 405
 406         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_rule));
 407         if (trans == NULL)
 408                 return NULL;
 409
 410         if (msg_type == NFT_MSG_NEWRULE && ctx->nla[NFTA_RULE_ID] != NULL) {
 411                 nft_trans_rule_id(trans) =
 412                         ntohl(nla_get_be32(ctx->nla[NFTA_RULE_ID]));
 413         }
 414         nft_trans_rule(trans) = rule;
 415         nft_trans_commit_list_add_tail(ctx->net, trans);
 416
 417         return trans;
 418 }
 419
 420 static int nft_delrule(struct nft_ctx *ctx, struct nft_rule *rule)
 421 {
 422         struct nft_flow_rule *flow;
 423         struct nft_trans *trans;
 424         int err;
 425
 426         trans = nft_trans_rule_add(ctx, NFT_MSG_DELRULE, rule);
 427         if (trans == NULL)
 428                 return -ENOMEM;
 429
 430         if (ctx->chain->flags & NFT_CHAIN_HW_OFFLOAD) {
 431                 flow = nft_flow_rule_create(ctx->net, rule);
 432                 if (IS_ERR(flow)) {
 433                         nft_trans_destroy(trans);
 434                         return PTR_ERR(flow);
 435                 }
 436
 437                 nft_trans_flow_rule(trans) = flow;
 438         }
 439
 440         err = nf_tables_delrule_deactivate(ctx, rule);
 441         if (err < 0) {
 442                 nft_trans_destroy(trans);
 443                 return err;
 444         }
 445         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_PREPARE);
 446
 447         return 0;
 448 }
 449
 450 static int nft_delrule_by_chain(struct nft_ctx *ctx)
 451 {
 452         struct nft_rule *rule;
 453         int err;
 454
 455         list_for_each_entry(rule, &ctx->chain->rules, list) {
 456                 if (!nft_is_active_next(ctx->net, rule))
 457                         continue;
 458
 459                 err = nft_delrule(ctx, rule);
 460                 if (err < 0)
 461                         return err;
 462         }
 463         return 0;
 464 }
 465
 466 static int __nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
 467                                struct nft_set *set,
 468                                const struct nft_set_desc *desc)
 469 {
 470         struct nft_trans *trans;
 471
 472         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_set));
 473         if (trans == NULL)
 474                 return -ENOMEM;
 475
 476         if (msg_type == NFT_MSG_NEWSET && ctx->nla[NFTA_SET_ID] && !desc) {
 477                 nft_trans_set_id(trans) =
 478                         ntohl(nla_get_be32(ctx->nla[NFTA_SET_ID]));
 479                 nft_activate_next(ctx->net, set);
 480         }
 481         nft_trans_set(trans) = set;
 482         if (desc) {
 483                 nft_trans_set_update(trans) = true;
 484                 nft_trans_set_gc_int(trans) = desc->gc_int;
 485                 nft_trans_set_timeout(trans) = desc->timeout;
 486         }
 487         nft_trans_commit_list_add_tail(ctx->net, trans);
 488
 489         return 0;
 490 }
 491
 492 static int nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
 493                              struct nft_set *set)
 494 {
 495         return __nft_trans_set_add(ctx, msg_type, set, NULL);
 496 }
 497
 498 static int nft_delset(const struct nft_ctx *ctx, struct nft_set *set)
 499 {
 500         int err;
 501
 502         err = nft_trans_set_add(ctx, NFT_MSG_DELSET, set);
 503         if (err < 0)
 504                 return err;
 505
 506         nft_deactivate_next(ctx->net, set);
 507         ctx->table->use--;
 508
 509         return err;
 510 }
 511
 512 static int nft_trans_obj_add(struct nft_ctx *ctx, int msg_type,
 513                              struct nft_object *obj)
 514 {
 515         struct nft_trans *trans;
 516
 517         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_obj));
 518         if (trans == NULL)
 519                 return -ENOMEM;
 520
 521         if (msg_type == NFT_MSG_NEWOBJ)
 522                 nft_activate_next(ctx->net, obj);
 523
 524         nft_trans_obj(trans) = obj;
 525         nft_trans_commit_list_add_tail(ctx->net, trans);
 526
 527         return 0;
 528 }
 529
 530 static int nft_delobj(struct nft_ctx *ctx, struct nft_object *obj)
 531 {
 532         int err;
 533
 534         err = nft_trans_obj_add(ctx, NFT_MSG_DELOBJ, obj);
 535         if (err < 0)
 536                 return err;
 537
 538         nft_deactivate_next(ctx->net, obj);
 539         ctx->table->use--;
 540
 541         return err;
 542 }
 543
 544 static int nft_trans_flowtable_add(struct nft_ctx *ctx, int msg_type,
 545                                    struct nft_flowtable *flowtable)
 546 {
 547         struct nft_trans *trans;
 548
 549         trans = nft_trans_alloc(ctx, msg_type,
 550                                 sizeof(struct nft_trans_flowtable));
 551         if (trans == NULL)
 552                 return -ENOMEM;
 553
 554         if (msg_type == NFT_MSG_NEWFLOWTABLE)
 555                 nft_activate_next(ctx->net, flowtable);
 556
 557         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
 558         nft_trans_flowtable(trans) = flowtable;
 559         nft_trans_commit_list_add_tail(ctx->net, trans);
 560
 561         return 0;
 562 }
 563
 564 static int nft_delflowtable(struct nft_ctx *ctx,
 565                             struct nft_flowtable *flowtable)
 566 {
 567         int err;
 568
 569         err = nft_trans_flowtable_add(ctx, NFT_MSG_DELFLOWTABLE, flowtable);
 570         if (err < 0)
 571                 return err;
 572
 573         nft_deactivate_next(ctx->net, flowtable);
 574         ctx->table->use--;
 575
 576         return err;
 577 }
 578
 579 static void __nft_reg_track_clobber(struct nft_regs_track *track, u8 dreg)
 580 {
 581         int i;
 582
 583         for (i = track->regs[dreg].num_reg; i > 0; i--)
 584                 __nft_reg_track_cancel(track, dreg - i);
 585 }
 586
 587 static void __nft_reg_track_update(struct nft_regs_track *track,
 588                                    const struct nft_expr *expr,
 589                                    u8 dreg, u8 num_reg)
 590 {
 591         track->regs[dreg].selector = expr;
 592         track->regs[dreg].bitwise = NULL;
 593         track->regs[dreg].num_reg = num_reg;
 594 }
 595
 596 void nft_reg_track_update(struct nft_regs_track *track,
 597                           const struct nft_expr *expr, u8 dreg, u8 len)
 598 {
 599         unsigned int regcount;
 600         int i;
 601
 602         __nft_reg_track_clobber(track, dreg);
 603
 604         regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
 605         for (i = 0; i < regcount; i++, dreg++)
 606                 __nft_reg_track_update(track, expr, dreg, i);
 607 }
 608 EXPORT_SYMBOL_GPL(nft_reg_track_update);
 609
 610 void nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg, u8 len)
 611 {
 612         unsigned int regcount;
 613         int i;
 614
 615         __nft_reg_track_clobber(track, dreg);
 616
 617         regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
 618         for (i = 0; i < regcount; i++, dreg++)
 619                 __nft_reg_track_cancel(track, dreg);
 620 }
 621 EXPORT_SYMBOL_GPL(nft_reg_track_cancel);
 622
 623 void __nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg)
 624 {
 625         track->regs[dreg].selector = NULL;
 626         track->regs[dreg].bitwise = NULL;
 627         track->regs[dreg].num_reg = 0;
 628 }
 629 EXPORT_SYMBOL_GPL(__nft_reg_track_cancel);
 630
 631 /*
 632  * Tables
 633  */
 634
 635 static struct nft_table *nft_table_lookup(const struct net *net,
 636                                           const struct nlattr *nla,
 637                                           u8 family, u8 genmask, u32 nlpid)
 638 {
 639         struct nftables_pernet *nft_net;
 640         struct nft_table *table;
 641
 642         if (nla == NULL)
 643                 return ERR_PTR(-EINVAL);
 644
 645         nft_net = nft_pernet(net);
 646         list_for_each_entry_rcu(table, &nft_net->tables, list,
 647                                 lockdep_is_held(&nft_net->commit_mutex)) {
 648                 if (!nla_strcmp(nla, table->name) &&
 649                     table->family == family &&
 650                     nft_active_genmask(table, genmask)) {
 651                         if (nft_table_has_owner(table) &&
 652                             nlpid && table->nlpid != nlpid)
 653                                 return ERR_PTR(-EPERM);
 654
 655                         return table;
 656                 }
 657         }
 658
 659         return ERR_PTR(-ENOENT);
 660 }
 661
 662 static struct nft_table *nft_table_lookup_byhandle(const struct net *net,
 663                                                    const struct nlattr *nla,
 664                                                    u8 genmask, u32 nlpid)
 665 {
 666         struct nftables_pernet *nft_net;
 667         struct nft_table *table;
 668
 669         nft_net = nft_pernet(net);
 670         list_for_each_entry(table, &nft_net->tables, list) {
 671                 if (be64_to_cpu(nla_get_be64(nla)) == table->handle &&
 672                     nft_active_genmask(table, genmask)) {
 673                         if (nft_table_has_owner(table) &&
 674                             nlpid && table->nlpid != nlpid)
 675                                 return ERR_PTR(-EPERM);
 676
 677                         return table;
 678                 }
 679         }
 680
 681         return ERR_PTR(-ENOENT);
 682 }
 683
 684 static inline u64 nf_tables_alloc_handle(struct nft_table *table)
 685 {
 686         return ++table->hgenerator;
 687 }
 688
 689 static const struct nft_chain_type *chain_type[NFPROTO_NUMPROTO][NFT_CHAIN_T_MAX];
 690
 691 static const struct nft_chain_type *
 692 __nft_chain_type_get(u8 family, enum nft_chain_types type)
 693 {
 694         if (family >= NFPROTO_NUMPROTO ||
 695             type >= NFT_CHAIN_T_MAX)
 696                 return NULL;
 697
 698         return chain_type[family][type];
 699 }
 700
 701 static const struct nft_chain_type *
 702 __nf_tables_chain_type_lookup(const struct nlattr *nla, u8 family)
 703 {
 704         const struct nft_chain_type *type;
 705         int i;
 706
 707         for (i = 0; i < NFT_CHAIN_T_MAX; i++) {
 708                 type = __nft_chain_type_get(family, i);
 709                 if (!type)
 710                         continue;
 711                 if (!nla_strcmp(nla, type->name))
 712                         return type;
 713         }
 714         return NULL;
 715 }
 716
 717 struct nft_module_request {
 718         struct list_head        list;
 719         char                    module[MODULE_NAME_LEN];
 720         bool                    done;
 721 };
 722
 723 #ifdef CONFIG_MODULES
 724 __printf(2, 3) int nft_request_module(struct net *net, const char *fmt,
 725                                       ...)
 726 {
 727         char module_name[MODULE_NAME_LEN];
 728         struct nftables_pernet *nft_net;
 729         struct nft_module_request *req;
 730         va_list args;
 731         int ret;
 732
 733         va_start(args, fmt);
 734         ret = vsnprintf(module_name, MODULE_NAME_LEN, fmt, args);
 735         va_end(args);
 736         if (ret >= MODULE_NAME_LEN)
 737                 return 0;
 738
 739         nft_net = nft_pernet(net);
 740         list_for_each_entry(req, &nft_net->module_list, list) {
 741                 if (!strcmp(req->module, module_name)) {
 742                         if (req->done)
 743                                 return 0;
 744
 745                         /* A request to load this module already exists. */
 746                         return -EAGAIN;
 747                 }
 748         }
 749
 750         req = kmalloc(sizeof(*req), GFP_KERNEL);
 751         if (!req)
 752                 return -ENOMEM;
 753
 754         req->done = false;
 755         strscpy(req->module, module_name, MODULE_NAME_LEN);
 756         list_add_tail(&req->list, &nft_net->module_list);
 757
 758         return -EAGAIN;
 759 }
 760 EXPORT_SYMBOL_GPL(nft_request_module);
 761 #endif
 762
 763 static void lockdep_nfnl_nft_mutex_not_held(void)
 764 {
 765 #ifdef CONFIG_PROVE_LOCKING
 766         if (debug_locks)
 767                 WARN_ON_ONCE(lockdep_nfnl_is_held(NFNL_SUBSYS_NFTABLES));
 768 #endif
 769 }
 770
 771 static const struct nft_chain_type *
 772 nf_tables_chain_type_lookup(struct net *net, const struct nlattr *nla,
 773                             u8 family, bool autoload)
 774 {
 775         const struct nft_chain_type *type;
 776
 777         type = __nf_tables_chain_type_lookup(nla, family);
 778         if (type != NULL)
 779                 return type;
 780
 781         lockdep_nfnl_nft_mutex_not_held();
 782 #ifdef CONFIG_MODULES
 783         if (autoload) {
 784                 if (nft_request_module(net, "nft-chain-%u-%.*s", family,
 785                                        nla_len(nla),
 786                                        (const char *)nla_data(nla)) == -EAGAIN)
 787                         return ERR_PTR(-EAGAIN);
 788         }
 789 #endif
 790         return ERR_PTR(-ENOENT);
 791 }
 792
 793 static __be16 nft_base_seq(const struct net *net)
 794 {
 795         struct nftables_pernet *nft_net = nft_pernet(net);
 796
 797         return htons(nft_net->base_seq & 0xffff);
 798 }
 799
 800 static const struct nla_policy nft_table_policy[NFTA_TABLE_MAX + 1] = {
 801         [NFTA_TABLE_NAME]       = { .type = NLA_STRING,
 802                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
 803         [NFTA_TABLE_FLAGS]      = { .type = NLA_U32 },
 804         [NFTA_TABLE_HANDLE]     = { .type = NLA_U64 },
 805         [NFTA_TABLE_USERDATA]   = { .type = NLA_BINARY,
 806                                     .len = NFT_USERDATA_MAXLEN }
 807 };
 808
 809 static int nf_tables_fill_table_info(struct sk_buff *skb, struct net *net,
 810                                      u32 portid, u32 seq, int event, u32 flags,
 811                                      int family, const struct nft_table *table)
 812 {
 813         struct nlmsghdr *nlh;
 814
 815         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
 816         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
 817                            NFNETLINK_V0, nft_base_seq(net));
 818         if (!nlh)
 819                 goto nla_put_failure;
 820
 821         if (nla_put_string(skb, NFTA_TABLE_NAME, table->name) ||
 822             nla_put_be32(skb, NFTA_TABLE_USE, htonl(table->use)) ||
 823             nla_put_be64(skb, NFTA_TABLE_HANDLE, cpu_to_be64(table->handle),
 824                          NFTA_TABLE_PAD))
 825                 goto nla_put_failure;
 826
 827         if (event == NFT_MSG_DELTABLE) {
 828                 nlmsg_end(skb, nlh);
 829                 return 0;
 830         }
 831
 832         if (nla_put_be32(skb, NFTA_TABLE_FLAGS,
 833                          htonl(table->flags & NFT_TABLE_F_MASK)))
 834                 goto nla_put_failure;
 835
 836         if (nft_table_has_owner(table) &&
 837             nla_put_be32(skb, NFTA_TABLE_OWNER, htonl(table->nlpid)))
 838                 goto nla_put_failure;
 839
 840         if (table->udata) {
 841                 if (nla_put(skb, NFTA_TABLE_USERDATA, table->udlen, table->udata))
 842                         goto nla_put_failure;
 843         }
 844
 845         nlmsg_end(skb, nlh);
 846         return 0;
 847
 848 nla_put_failure:
 849         nlmsg_trim(skb, nlh);
 850         return -1;
 851 }
 852
 853 struct nftnl_skb_parms {
 854         bool report;
 855 };
 856 #define NFT_CB(skb)     (*(struct nftnl_skb_parms*)&((skb)->cb))
 857
 858 static void nft_notify_enqueue(struct sk_buff *skb, bool report,
 859                                struct list_head *notify_list)
 860 {
 861         NFT_CB(skb).report = report;
 862         list_add_tail(&skb->list, notify_list);
 863 }
 864
 865 static void nf_tables_table_notify(const struct nft_ctx *ctx, int event)
 866 {
 867         struct nftables_pernet *nft_net;
 868         struct sk_buff *skb;
 869         u16 flags = 0;
 870         int err;
 871
 872         if (!ctx->report &&
 873             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
 874                 return;
 875
 876         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
 877         if (skb == NULL)
 878                 goto err;
 879
 880         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
 881                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
 882
 883         err = nf_tables_fill_table_info(skb, ctx->net, ctx->portid, ctx->seq,
 884                                         event, flags, ctx->family, ctx->table);
 885         if (err < 0) {
 886                 kfree_skb(skb);
 887                 goto err;
 888         }
 889
 890         nft_net = nft_pernet(ctx->net);
 891         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
 892         return;
 893 err:
 894         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
 895 }
 896
 897 static int nf_tables_dump_tables(struct sk_buff *skb,
 898                                  struct netlink_callback *cb)
 899 {
 900         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
 901         struct nftables_pernet *nft_net;
 902         const struct nft_table *table;
 903         unsigned int idx = 0, s_idx = cb->args[0];
 904         struct net *net = sock_net(skb->sk);
 905         int family = nfmsg->nfgen_family;
 906
 907         rcu_read_lock();
 908         nft_net = nft_pernet(net);
 909         cb->seq = READ_ONCE(nft_net->base_seq);
 910
 911         list_for_each_entry_rcu(table, &nft_net->tables, list) {
 912                 if (family != NFPROTO_UNSPEC && family != table->family)
 913                         continue;
 914
 915                 if (idx < s_idx)
 916                         goto cont;
 917                 if (idx > s_idx)
 918                         memset(&cb->args[1], 0,
 919                                sizeof(cb->args) - sizeof(cb->args[0]));
 920                 if (!nft_is_active(net, table))
 921                         continue;
 922                 if (nf_tables_fill_table_info(skb, net,
 923                                               NETLINK_CB(cb->skb).portid,
 924                                               cb->nlh->nlmsg_seq,
 925                                               NFT_MSG_NEWTABLE, NLM_F_MULTI,
 926                                               table->family, table) < 0)
 927                         goto done;
 928
 929                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
 930 cont:
 931                 idx++;
 932         }
 933 done:
 934         rcu_read_unlock();
 935         cb->args[0] = idx;
 936         return skb->len;
 937 }
 938
 939 static int nft_netlink_dump_start_rcu(struct sock *nlsk, struct sk_buff *skb,
 940                                       const struct nlmsghdr *nlh,
 941                                       struct netlink_dump_control *c)
 942 {
 943         int err;
 944
 945         if (!try_module_get(THIS_MODULE))
 946                 return -EINVAL;
 947
 948         rcu_read_unlock();
 949         err = netlink_dump_start(nlsk, skb, nlh, c);
 950         rcu_read_lock();
 951         module_put(THIS_MODULE);
 952
 953         return err;
 954 }
 955
 956 /* called with rcu_read_lock held */
 957 static int nf_tables_gettable(struct sk_buff *skb, const struct nfnl_info *info,
 958                               const struct nlattr * const nla[])
 959 {
 960         struct netlink_ext_ack *extack = info->extack;
 961         u8 genmask = nft_genmask_cur(info->net);
 962         u8 family = info->nfmsg->nfgen_family;
 963         const struct nft_table *table;
 964         struct net *net = info->net;
 965         struct sk_buff *skb2;
 966         int err;
 967
 968         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
 969                 struct netlink_dump_control c = {
 970                         .dump = nf_tables_dump_tables,
 971                         .module = THIS_MODULE,
 972                 };
 973
 974                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
 975         }
 976
 977         table = nft_table_lookup(net, nla[NFTA_TABLE_NAME], family, genmask, 0);
 978         if (IS_ERR(table)) {
 979                 NL_SET_BAD_ATTR(extack, nla[NFTA_TABLE_NAME]);
 980                 return PTR_ERR(table);
 981         }
 982
 983         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
 984         if (!skb2)
 985                 return -ENOMEM;
 986
 987         err = nf_tables_fill_table_info(skb2, net, NETLINK_CB(skb).portid,
 988                                         info->nlh->nlmsg_seq, NFT_MSG_NEWTABLE,
 989                                         0, family, table);
 990         if (err < 0)
 991                 goto err_fill_table_info;
 992
 993         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
 994
 995 err_fill_table_info:
 996         kfree_skb(skb2);
 997         return err;
 998 }
 999
1000 static void nft_table_disable(struct net *net, struct nft_table *table, u32 cnt)
1001 {
1002         struct nft_chain *chain;
1003         u32 i = 0;
1004
1005         list_for_each_entry(chain, &table->chains, list) {
1006                 if (!nft_is_active_next(net, chain))
1007                         continue;
1008                 if (!nft_is_base_chain(chain))
1009                         continue;
1010
1011                 if (cnt && i++ == cnt)
1012                         break;
1013
1014                 nf_tables_unregister_hook(net, table, chain);
1015         }
1016 }
1017
1018 static int nf_tables_table_enable(struct net *net, struct nft_table *table)
1019 {
1020         struct nft_chain *chain;
1021         int err, i = 0;
1022
1023         list_for_each_entry(chain, &table->chains, list) {
1024                 if (!nft_is_active_next(net, chain))
1025                         continue;
1026                 if (!nft_is_base_chain(chain))
1027                         continue;
1028
1029                 err = nf_tables_register_hook(net, table, chain);
1030                 if (err < 0)
1031                         goto err_register_hooks;
1032
1033                 i++;
1034         }
1035         return 0;
1036
1037 err_register_hooks:
1038         if (i)
1039                 nft_table_disable(net, table, i);
1040         return err;
1041 }
1042
1043 static void nf_tables_table_disable(struct net *net, struct nft_table *table)
1044 {
1045         table->flags &= ~NFT_TABLE_F_DORMANT;
1046         nft_table_disable(net, table, 0);
1047         table->flags |= NFT_TABLE_F_DORMANT;
1048 }
1049
1050 #define __NFT_TABLE_F_INTERNAL          (NFT_TABLE_F_MASK + 1)
1051 #define __NFT_TABLE_F_WAS_DORMANT       (__NFT_TABLE_F_INTERNAL << 0)
1052 #define __NFT_TABLE_F_WAS_AWAKEN        (__NFT_TABLE_F_INTERNAL << 1)
1053 #define __NFT_TABLE_F_UPDATE            (__NFT_TABLE_F_WAS_DORMANT | \
1054                                          __NFT_TABLE_F_WAS_AWAKEN)
1055
1056 static int nf_tables_updtable(struct nft_ctx *ctx)
1057 {
1058         struct nft_trans *trans;
1059         u32 flags;
1060         int ret;
1061
1062         if (!ctx->nla[NFTA_TABLE_FLAGS])
1063                 return 0;
1064
1065         flags = ntohl(nla_get_be32(ctx->nla[NFTA_TABLE_FLAGS]));
1066         if (flags & ~NFT_TABLE_F_MASK)
1067                 return -EOPNOTSUPP;
1068
1069         if (flags == ctx->table->flags)
1070                 return 0;
1071
1072         if ((nft_table_has_owner(ctx->table) &&
1073              !(flags & NFT_TABLE_F_OWNER)) ||
1074             (!nft_table_has_owner(ctx->table) &&
1075              flags & NFT_TABLE_F_OWNER))
1076                 return -EOPNOTSUPP;
1077
1078         trans = nft_trans_alloc(ctx, NFT_MSG_NEWTABLE,
1079                                 sizeof(struct nft_trans_table));
1080         if (trans == NULL)
1081                 return -ENOMEM;
1082
1083         if ((flags & NFT_TABLE_F_DORMANT) &&
1084             !(ctx->table->flags & NFT_TABLE_F_DORMANT)) {
1085                 ctx->table->flags |= NFT_TABLE_F_DORMANT;
1086                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE))
1087                         ctx->table->flags |= __NFT_TABLE_F_WAS_AWAKEN;
1088         } else if (!(flags & NFT_TABLE_F_DORMANT) &&
1089                    ctx->table->flags & NFT_TABLE_F_DORMANT) {
1090                 ctx->table->flags &= ~NFT_TABLE_F_DORMANT;
1091                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE)) {
1092                         ret = nf_tables_table_enable(ctx->net, ctx->table);
1093                         if (ret < 0)
1094                                 goto err_register_hooks;
1095
1096                         ctx->table->flags |= __NFT_TABLE_F_WAS_DORMANT;
1097                 }
1098         }
1099
1100         nft_trans_table_update(trans) = true;
1101         nft_trans_commit_list_add_tail(ctx->net, trans);
1102
1103         return 0;
1104
1105 err_register_hooks:
1106         nft_trans_destroy(trans);
1107         return ret;
1108 }
1109
1110 static u32 nft_chain_hash(const void *data, u32 len, u32 seed)
1111 {
1112         const char *name = data;
1113
1114         return jhash(name, strlen(name), seed);
1115 }
1116
1117 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed)
1118 {
1119         const struct nft_chain *chain = data;
1120
1121         return nft_chain_hash(chain->name, 0, seed);
1122 }
1123
1124 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *arg,
1125                               const void *ptr)
1126 {
1127         const struct nft_chain *chain = ptr;
1128         const char *name = arg->key;
1129
1130         return strcmp(chain->name, name);
1131 }
1132
1133 static u32 nft_objname_hash(const void *data, u32 len, u32 seed)
1134 {
1135         const struct nft_object_hash_key *k = data;
1136
1137         seed ^= hash_ptr(k->table, 32);
1138
1139         return jhash(k->name, strlen(k->name), seed);
1140 }
1141
1142 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed)
1143 {
1144         const struct nft_object *obj = data;
1145
1146         return nft_objname_hash(&obj->key, 0, seed);
1147 }
1148
1149 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *arg,
1150                                 const void *ptr)
1151 {
1152         const struct nft_object_hash_key *k = arg->key;
1153         const struct nft_object *obj = ptr;
1154
1155         if (obj->key.table != k->table)
1156                 return -1;
1157
1158         return strcmp(obj->key.name, k->name);
1159 }
1160
1161 static bool nft_supported_family(u8 family)
1162 {
1163         return false
1164 #ifdef CONFIG_NF_TABLES_INET
1165                 || family == NFPROTO_INET
1166 #endif
1167 #ifdef CONFIG_NF_TABLES_IPV4
1168                 || family == NFPROTO_IPV4
1169 #endif
1170 #ifdef CONFIG_NF_TABLES_ARP
1171                 || family == NFPROTO_ARP
1172 #endif
1173 #ifdef CONFIG_NF_TABLES_NETDEV
1174                 || family == NFPROTO_NETDEV
1175 #endif
1176 #if IS_ENABLED(CONFIG_NF_TABLES_BRIDGE)
1177                 || family == NFPROTO_BRIDGE
1178 #endif
1179 #ifdef CONFIG_NF_TABLES_IPV6
1180                 || family == NFPROTO_IPV6
1181 #endif
1182                 ;
1183 }
1184
1185 static int nf_tables_newtable(struct sk_buff *skb, const struct nfnl_info *info,
1186                               const struct nlattr * const nla[])
1187 {
1188         struct nftables_pernet *nft_net = nft_pernet(info->net);
1189         struct netlink_ext_ack *extack = info->extack;
1190         u8 genmask = nft_genmask_next(info->net);
1191         u8 family = info->nfmsg->nfgen_family;
1192         struct net *net = info->net;
1193         const struct nlattr *attr;
1194         struct nft_table *table;
1195         struct nft_ctx ctx;
1196         u32 flags = 0;
1197         int err;
1198
1199         if (!nft_supported_family(family))
1200                 return -EOPNOTSUPP;
1201
1202         lockdep_assert_held(&nft_net->commit_mutex);
1203         attr = nla[NFTA_TABLE_NAME];
1204         table = nft_table_lookup(net, attr, family, genmask,
1205                                  NETLINK_CB(skb).portid);
1206         if (IS_ERR(table)) {
1207                 if (PTR_ERR(table) != -ENOENT)
1208                         return PTR_ERR(table);
1209         } else {
1210                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
1211                         NL_SET_BAD_ATTR(extack, attr);
1212                         return -EEXIST;
1213                 }
1214                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
1215                         return -EOPNOTSUPP;
1216
1217                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1218
1219                 return nf_tables_updtable(&ctx);
1220         }
1221
1222         if (nla[NFTA_TABLE_FLAGS]) {
1223                 flags = ntohl(nla_get_be32(nla[NFTA_TABLE_FLAGS]));
1224                 if (flags & ~NFT_TABLE_F_MASK)
1225                         return -EOPNOTSUPP;
1226         }
1227
1228         err = -ENOMEM;
1229         table = kzalloc(sizeof(*table), GFP_KERNEL_ACCOUNT);
1230         if (table == NULL)
1231                 goto err_kzalloc;
1232
1233         table->validate_state = NFT_VALIDATE_SKIP;
1234         table->name = nla_strdup(attr, GFP_KERNEL_ACCOUNT);
1235         if (table->name == NULL)
1236                 goto err_strdup;
1237
1238         if (nla[NFTA_TABLE_USERDATA]) {
1239                 table->udata = nla_memdup(nla[NFTA_TABLE_USERDATA], GFP_KERNEL_ACCOUNT);
1240                 if (table->udata == NULL)
1241                         goto err_table_udata;
1242
1243                 table->udlen = nla_len(nla[NFTA_TABLE_USERDATA]);
1244         }
1245
1246         err = rhltable_init(&table->chains_ht, &nft_chain_ht_params);
1247         if (err)
1248                 goto err_chain_ht;
1249
1250         INIT_LIST_HEAD(&table->chains);
1251         INIT_LIST_HEAD(&table->sets);
1252         INIT_LIST_HEAD(&table->objects);
1253         INIT_LIST_HEAD(&table->flowtables);
1254         table->family = family;
1255         table->flags = flags;
1256         table->handle = ++nft_net->table_handle;
1257         if (table->flags & NFT_TABLE_F_OWNER)
1258                 table->nlpid = NETLINK_CB(skb).portid;
1259
1260         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1261         err = nft_trans_table_add(&ctx, NFT_MSG_NEWTABLE);
1262         if (err < 0)
1263                 goto err_trans;
1264
1265         list_add_tail_rcu(&table->list, &nft_net->tables);
1266         return 0;
1267 err_trans:
1268         rhltable_destroy(&table->chains_ht);
1269 err_chain_ht:
1270         kfree(table->udata);
1271 err_table_udata:
1272         kfree(table->name);
1273 err_strdup:
1274         kfree(table);
1275 err_kzalloc:
1276         return err;
1277 }
1278
1279 static int nft_flush_table(struct nft_ctx *ctx)
1280 {
1281         struct nft_flowtable *flowtable, *nft;
1282         struct nft_chain *chain, *nc;
1283         struct nft_object *obj, *ne;
1284         struct nft_set *set, *ns;
1285         int err;
1286
1287         list_for_each_entry(chain, &ctx->table->chains, list) {
1288                 if (!nft_is_active_next(ctx->net, chain))
1289                         continue;
1290
1291                 if (nft_chain_is_bound(chain))
1292                         continue;
1293
1294                 ctx->chain = chain;
1295
1296                 err = nft_delrule_by_chain(ctx);
1297                 if (err < 0)
1298                         goto out;
1299         }
1300
1301         list_for_each_entry_safe(set, ns, &ctx->table->sets, list) {
1302                 if (!nft_is_active_next(ctx->net, set))
1303                         continue;
1304
1305                 if (nft_set_is_anonymous(set) &&
1306                     !list_empty(&set->bindings))
1307                         continue;
1308
1309                 err = nft_delset(ctx, set);
1310                 if (err < 0)
1311                         goto out;
1312         }
1313
1314         list_for_each_entry_safe(flowtable, nft, &ctx->table->flowtables, list) {
1315                 if (!nft_is_active_next(ctx->net, flowtable))
1316                         continue;
1317
1318                 err = nft_delflowtable(ctx, flowtable);
1319                 if (err < 0)
1320                         goto out;
1321         }
1322
1323         list_for_each_entry_safe(obj, ne, &ctx->table->objects, list) {
1324                 if (!nft_is_active_next(ctx->net, obj))
1325                         continue;
1326
1327                 err = nft_delobj(ctx, obj);
1328                 if (err < 0)
1329                         goto out;
1330         }
1331
1332         list_for_each_entry_safe(chain, nc, &ctx->table->chains, list) {
1333                 if (!nft_is_active_next(ctx->net, chain))
1334                         continue;
1335
1336                 if (nft_chain_is_bound(chain))
1337                         continue;
1338
1339                 ctx->chain = chain;
1340
1341                 err = nft_delchain(ctx);
1342                 if (err < 0)
1343                         goto out;
1344         }
1345
1346         err = nft_deltable(ctx);
1347 out:
1348         return err;
1349 }
1350
1351 static int nft_flush(struct nft_ctx *ctx, int family)
1352 {
1353         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
1354         const struct nlattr * const *nla = ctx->nla;
1355         struct nft_table *table, *nt;
1356         int err = 0;
1357
1358         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
1359                 if (family != AF_UNSPEC && table->family != family)
1360                         continue;
1361
1362                 ctx->family = table->family;
1363
1364                 if (!nft_is_active_next(ctx->net, table))
1365                         continue;
1366
1367                 if (nft_table_has_owner(table) && table->nlpid != ctx->portid)
1368                         continue;
1369
1370                 if (nla[NFTA_TABLE_NAME] &&
1371                     nla_strcmp(nla[NFTA_TABLE_NAME], table->name) != 0)
1372                         continue;
1373
1374                 ctx->table = table;
1375
1376                 err = nft_flush_table(ctx);
1377                 if (err < 0)
1378                         goto out;
1379         }
1380 out:
1381         return err;
1382 }
1383
1384 static int nf_tables_deltable(struct sk_buff *skb, const struct nfnl_info *info,
1385                               const struct nlattr * const nla[])
1386 {
1387         struct netlink_ext_ack *extack = info->extack;
1388         u8 genmask = nft_genmask_next(info->net);
1389         u8 family = info->nfmsg->nfgen_family;
1390         struct net *net = info->net;
1391         const struct nlattr *attr;
1392         struct nft_table *table;
1393         struct nft_ctx ctx;
1394
1395         nft_ctx_init(&ctx, net, skb, info->nlh, 0, NULL, NULL, nla);
1396         if (family == AF_UNSPEC ||
1397             (!nla[NFTA_TABLE_NAME] && !nla[NFTA_TABLE_HANDLE]))
1398                 return nft_flush(&ctx, family);
1399
1400         if (nla[NFTA_TABLE_HANDLE]) {
1401                 attr = nla[NFTA_TABLE_HANDLE];
1402                 table = nft_table_lookup_byhandle(net, attr, genmask,
1403                                                   NETLINK_CB(skb).portid);
1404         } else {
1405                 attr = nla[NFTA_TABLE_NAME];
1406                 table = nft_table_lookup(net, attr, family, genmask,
1407                                          NETLINK_CB(skb).portid);
1408         }
1409
1410         if (IS_ERR(table)) {
1411                 if (PTR_ERR(table) == -ENOENT &&
1412                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYTABLE)
1413                         return 0;
1414
1415                 NL_SET_BAD_ATTR(extack, attr);
1416                 return PTR_ERR(table);
1417         }
1418
1419         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
1420             table->use > 0)
1421                 return -EBUSY;
1422
1423         ctx.family = family;
1424         ctx.table = table;
1425
1426         return nft_flush_table(&ctx);
1427 }
1428
1429 static void nf_tables_table_destroy(struct nft_ctx *ctx)
1430 {
1431         if (WARN_ON(ctx->table->use > 0))
1432                 return;
1433
1434         rhltable_destroy(&ctx->table->chains_ht);
1435         kfree(ctx->table->name);
1436         kfree(ctx->table->udata);
1437         kfree(ctx->table);
1438 }
1439
1440 void nft_register_chain_type(const struct nft_chain_type *ctype)
1441 {
1442         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1443         if (WARN_ON(__nft_chain_type_get(ctype->family, ctype->type))) {
1444                 nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1445                 return;
1446         }
1447         chain_type[ctype->family][ctype->type] = ctype;
1448         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1449 }
1450 EXPORT_SYMBOL_GPL(nft_register_chain_type);
1451
1452 void nft_unregister_chain_type(const struct nft_chain_type *ctype)
1453 {
1454         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1455         chain_type[ctype->family][ctype->type] = NULL;
1456         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1457 }
1458 EXPORT_SYMBOL_GPL(nft_unregister_chain_type);
1459
1460 /*
1461  * Chains
1462  */
1463
1464 static struct nft_chain *
1465 nft_chain_lookup_byhandle(const struct nft_table *table, u64 handle, u8 genmask)
1466 {
1467         struct nft_chain *chain;
1468
1469         list_for_each_entry(chain, &table->chains, list) {
1470                 if (chain->handle == handle &&
1471                     nft_active_genmask(chain, genmask))
1472                         return chain;
1473         }
1474
1475         return ERR_PTR(-ENOENT);
1476 }
1477
1478 static bool lockdep_commit_lock_is_held(const struct net *net)
1479 {
1480 #ifdef CONFIG_PROVE_LOCKING
1481         struct nftables_pernet *nft_net = nft_pernet(net);
1482
1483         return lockdep_is_held(&nft_net->commit_mutex);
1484 #else
1485         return true;
1486 #endif
1487 }
1488
1489 static struct nft_chain *nft_chain_lookup(struct net *net,
1490                                           struct nft_table *table,
1491                                           const struct nlattr *nla, u8 genmask)
1492 {
1493         char search[NFT_CHAIN_MAXNAMELEN + 1];
1494         struct rhlist_head *tmp, *list;
1495         struct nft_chain *chain;
1496
1497         if (nla == NULL)
1498                 return ERR_PTR(-EINVAL);
1499
1500         nla_strscpy(search, nla, sizeof(search));
1501
1502         WARN_ON(!rcu_read_lock_held() &&
1503                 !lockdep_commit_lock_is_held(net));
1504
1505         chain = ERR_PTR(-ENOENT);
1506         rcu_read_lock();
1507         list = rhltable_lookup(&table->chains_ht, search, nft_chain_ht_params);
1508         if (!list)
1509                 goto out_unlock;
1510
1511         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
1512                 if (nft_active_genmask(chain, genmask))
1513                         goto out_unlock;
1514         }
1515         chain = ERR_PTR(-ENOENT);
1516 out_unlock:
1517         rcu_read_unlock();
1518         return chain;
1519 }
1520
1521 static const struct nla_policy nft_chain_policy[NFTA_CHAIN_MAX + 1] = {
1522         [NFTA_CHAIN_TABLE]      = { .type = NLA_STRING,
1523                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
1524         [NFTA_CHAIN_HANDLE]     = { .type = NLA_U64 },
1525         [NFTA_CHAIN_NAME]       = { .type = NLA_STRING,
1526                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
1527         [NFTA_CHAIN_HOOK]       = { .type = NLA_NESTED },
1528         [NFTA_CHAIN_POLICY]     = { .type = NLA_U32 },
1529         [NFTA_CHAIN_TYPE]       = { .type = NLA_STRING,
1530                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
1531         [NFTA_CHAIN_COUNTERS]   = { .type = NLA_NESTED },
1532         [NFTA_CHAIN_FLAGS]      = { .type = NLA_U32 },
1533         [NFTA_CHAIN_ID]         = { .type = NLA_U32 },
1534         [NFTA_CHAIN_USERDATA]   = { .type = NLA_BINARY,
1535                                     .len = NFT_USERDATA_MAXLEN },
1536 };
1537
1538 static const struct nla_policy nft_hook_policy[NFTA_HOOK_MAX + 1] = {
1539         [NFTA_HOOK_HOOKNUM]     = { .type = NLA_U32 },
1540         [NFTA_HOOK_PRIORITY]    = { .type = NLA_U32 },
1541         [NFTA_HOOK_DEV]         = { .type = NLA_STRING,
1542                                     .len = IFNAMSIZ - 1 },
1543 };
1544
1545 static int nft_dump_stats(struct sk_buff *skb, struct nft_stats __percpu *stats)
1546 {
1547         struct nft_stats *cpu_stats, total;
1548         struct nlattr *nest;
1549         unsigned int seq;
1550         u64 pkts, bytes;
1551         int cpu;
1552
1553         if (!stats)
1554                 return 0;
1555
1556         memset(&total, 0, sizeof(total));
1557         for_each_possible_cpu(cpu) {
1558                 cpu_stats = per_cpu_ptr(stats, cpu);
1559                 do {
1560                         seq = u64_stats_fetch_begin(&cpu_stats->syncp);
1561                         pkts = cpu_stats->pkts;
1562                         bytes = cpu_stats->bytes;
1563                 } while (u64_stats_fetch_retry(&cpu_stats->syncp, seq));
1564                 total.pkts += pkts;
1565                 total.bytes += bytes;
1566         }
1567         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_COUNTERS);
1568         if (nest == NULL)
1569                 goto nla_put_failure;
1570
1571         if (nla_put_be64(skb, NFTA_COUNTER_PACKETS, cpu_to_be64(total.pkts),
1572                          NFTA_COUNTER_PAD) ||
1573             nla_put_be64(skb, NFTA_COUNTER_BYTES, cpu_to_be64(total.bytes),
1574                          NFTA_COUNTER_PAD))
1575                 goto nla_put_failure;
1576
1577         nla_nest_end(skb, nest);
1578         return 0;
1579
1580 nla_put_failure:
1581         return -ENOSPC;
1582 }
1583
1584 static int nft_dump_basechain_hook(struct sk_buff *skb, int family,
1585                                    const struct nft_base_chain *basechain,
1586                                    const struct list_head *hook_list)
1587 {
1588         const struct nf_hook_ops *ops = &basechain->ops;
1589         struct nft_hook *hook, *first = NULL;
1590         struct nlattr *nest, *nest_devs;
1591         int n = 0;
1592
1593         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_HOOK);
1594         if (nest == NULL)
1595                 goto nla_put_failure;
1596         if (nla_put_be32(skb, NFTA_HOOK_HOOKNUM, htonl(ops->hooknum)))
1597                 goto nla_put_failure;
1598         if (nla_put_be32(skb, NFTA_HOOK_PRIORITY, htonl(ops->priority)))
1599                 goto nla_put_failure;
1600
1601         if (nft_base_chain_netdev(family, ops->hooknum)) {
1602                 nest_devs = nla_nest_start_noflag(skb, NFTA_HOOK_DEVS);
1603                 if (!nest_devs)
1604                         goto nla_put_failure;
1605
1606                 if (!hook_list)
1607                         hook_list = &basechain->hook_list;
1608
1609                 list_for_each_entry(hook, hook_list, list) {
1610                         if (!first)
1611                                 first = hook;
1612
1613                         if (nla_put_string(skb, NFTA_DEVICE_NAME,
1614                                            hook->ops.dev->name))
1615                                 goto nla_put_failure;
1616                         n++;
1617                 }
1618                 nla_nest_end(skb, nest_devs);
1619
1620                 if (n == 1 &&
1621                     nla_put_string(skb, NFTA_HOOK_DEV, first->ops.dev->name))
1622                         goto nla_put_failure;
1623         }
1624         nla_nest_end(skb, nest);
1625
1626         return 0;
1627 nla_put_failure:
1628         return -1;
1629 }
1630
1631 static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net,
1632                                      u32 portid, u32 seq, int event, u32 flags,
1633                                      int family, const struct nft_table *table,
1634                                      const struct nft_chain *chain,
1635                                      const struct list_head *hook_list)
1636 {
1637         struct nlmsghdr *nlh;
1638
1639         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
1640         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
1641                            NFNETLINK_V0, nft_base_seq(net));
1642         if (!nlh)
1643                 goto nla_put_failure;
1644
1645         if (nla_put_string(skb, NFTA_CHAIN_TABLE, table->name) ||
1646             nla_put_string(skb, NFTA_CHAIN_NAME, chain->name) ||
1647             nla_put_be64(skb, NFTA_CHAIN_HANDLE, cpu_to_be64(chain->handle),
1648                          NFTA_CHAIN_PAD))
1649                 goto nla_put_failure;
1650
1651         if (event == NFT_MSG_DELCHAIN && !hook_list) {
1652                 nlmsg_end(skb, nlh);
1653                 return 0;
1654         }
1655
1656         if (nft_is_base_chain(chain)) {
1657                 const struct nft_base_chain *basechain = nft_base_chain(chain);
1658                 struct nft_stats __percpu *stats;
1659
1660                 if (nft_dump_basechain_hook(skb, family, basechain, hook_list))
1661                         goto nla_put_failure;
1662
1663                 if (nla_put_be32(skb, NFTA_CHAIN_POLICY,
1664                                  htonl(basechain->policy)))
1665                         goto nla_put_failure;
1666
1667                 if (nla_put_string(skb, NFTA_CHAIN_TYPE, basechain->type->name))
1668                         goto nla_put_failure;
1669
1670                 stats = rcu_dereference_check(basechain->stats,
1671                                               lockdep_commit_lock_is_held(net));
1672                 if (nft_dump_stats(skb, stats))
1673                         goto nla_put_failure;
1674         }
1675
1676         if (chain->flags &&
1677             nla_put_be32(skb, NFTA_CHAIN_FLAGS, htonl(chain->flags)))
1678                 goto nla_put_failure;
1679
1680         if (nla_put_be32(skb, NFTA_CHAIN_USE, htonl(chain->use)))
1681                 goto nla_put_failure;
1682
1683         if (chain->udata &&
1684             nla_put(skb, NFTA_CHAIN_USERDATA, chain->udlen, chain->udata))
1685                 goto nla_put_failure;
1686
1687         nlmsg_end(skb, nlh);
1688         return 0;
1689
1690 nla_put_failure:
1691         nlmsg_trim(skb, nlh);
1692         return -1;
1693 }
1694
1695 static void nf_tables_chain_notify(const struct nft_ctx *ctx, int event,
1696                                    const struct list_head *hook_list)
1697 {
1698         struct nftables_pernet *nft_net;
1699         struct sk_buff *skb;
1700         u16 flags = 0;
1701         int err;
1702
1703         if (!ctx->report &&
1704             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
1705                 return;
1706
1707         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
1708         if (skb == NULL)
1709                 goto err;
1710
1711         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
1712                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
1713
1714         err = nf_tables_fill_chain_info(skb, ctx->net, ctx->portid, ctx->seq,
1715                                         event, flags, ctx->family, ctx->table,
1716                                         ctx->chain, hook_list);
1717         if (err < 0) {
1718                 kfree_skb(skb);
1719                 goto err;
1720         }
1721
1722         nft_net = nft_pernet(ctx->net);
1723         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
1724         return;
1725 err:
1726         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
1727 }
1728
1729 static int nf_tables_dump_chains(struct sk_buff *skb,
1730                                  struct netlink_callback *cb)
1731 {
1732         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
1733         unsigned int idx = 0, s_idx = cb->args[0];
1734         struct net *net = sock_net(skb->sk);
1735         int family = nfmsg->nfgen_family;
1736         struct nftables_pernet *nft_net;
1737         const struct nft_table *table;
1738         const struct nft_chain *chain;
1739
1740         rcu_read_lock();
1741         nft_net = nft_pernet(net);
1742         cb->seq = READ_ONCE(nft_net->base_seq);
1743
1744         list_for_each_entry_rcu(table, &nft_net->tables, list) {
1745                 if (family != NFPROTO_UNSPEC && family != table->family)
1746                         continue;
1747
1748                 list_for_each_entry_rcu(chain, &table->chains, list) {
1749                         if (idx < s_idx)
1750                                 goto cont;
1751                         if (idx > s_idx)
1752                                 memset(&cb->args[1], 0,
1753                                        sizeof(cb->args) - sizeof(cb->args[0]));
1754                         if (!nft_is_active(net, chain))
1755                                 continue;
1756                         if (nf_tables_fill_chain_info(skb, net,
1757                                                       NETLINK_CB(cb->skb).portid,
1758                                                       cb->nlh->nlmsg_seq,
1759                                                       NFT_MSG_NEWCHAIN,
1760                                                       NLM_F_MULTI,
1761                                                       table->family, table,
1762                                                       chain, NULL) < 0)
1763                                 goto done;
1764
1765                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
1766 cont:
1767                         idx++;
1768                 }
1769         }
1770 done:
1771         rcu_read_unlock();
1772         cb->args[0] = idx;
1773         return skb->len;
1774 }
1775
1776 /* called with rcu_read_lock held */
1777 static int nf_tables_getchain(struct sk_buff *skb, const struct nfnl_info *info,
1778                               const struct nlattr * const nla[])
1779 {
1780         struct netlink_ext_ack *extack = info->extack;
1781         u8 genmask = nft_genmask_cur(info->net);
1782         u8 family = info->nfmsg->nfgen_family;
1783         const struct nft_chain *chain;
1784         struct net *net = info->net;
1785         struct nft_table *table;
1786         struct sk_buff *skb2;
1787         int err;
1788
1789         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
1790                 struct netlink_dump_control c = {
1791                         .dump = nf_tables_dump_chains,
1792                         .module = THIS_MODULE,
1793                 };
1794
1795                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
1796         }
1797
1798         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask, 0);
1799         if (IS_ERR(table)) {
1800                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
1801                 return PTR_ERR(table);
1802         }
1803
1804         chain = nft_chain_lookup(net, table, nla[NFTA_CHAIN_NAME], genmask);
1805         if (IS_ERR(chain)) {
1806                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
1807                 return PTR_ERR(chain);
1808         }
1809
1810         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
1811         if (!skb2)
1812                 return -ENOMEM;
1813
1814         err = nf_tables_fill_chain_info(skb2, net, NETLINK_CB(skb).portid,
1815                                         info->nlh->nlmsg_seq, NFT_MSG_NEWCHAIN,
1816                                         0, family, table, chain, NULL);
1817         if (err < 0)
1818                 goto err_fill_chain_info;
1819
1820         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
1821
1822 err_fill_chain_info:
1823         kfree_skb(skb2);
1824         return err;
1825 }
1826
1827 static const struct nla_policy nft_counter_policy[NFTA_COUNTER_MAX + 1] = {
1828         [NFTA_COUNTER_PACKETS]  = { .type = NLA_U64 },
1829         [NFTA_COUNTER_BYTES]    = { .type = NLA_U64 },
1830 };
1831
1832 static struct nft_stats __percpu *nft_stats_alloc(const struct nlattr *attr)
1833 {
1834         struct nlattr *tb[NFTA_COUNTER_MAX+1];
1835         struct nft_stats __percpu *newstats;
1836         struct nft_stats *stats;
1837         int err;
1838
1839         err = nla_parse_nested_deprecated(tb, NFTA_COUNTER_MAX, attr,
1840                                           nft_counter_policy, NULL);
1841         if (err < 0)
1842                 return ERR_PTR(err);
1843
1844         if (!tb[NFTA_COUNTER_BYTES] || !tb[NFTA_COUNTER_PACKETS])
1845                 return ERR_PTR(-EINVAL);
1846
1847         newstats = netdev_alloc_pcpu_stats(struct nft_stats);
1848         if (newstats == NULL)
1849                 return ERR_PTR(-ENOMEM);
1850
1851         /* Restore old counters on this cpu, no problem. Per-cpu statistics
1852          * are not exposed to userspace.
1853          */
1854         preempt_disable();
1855         stats = this_cpu_ptr(newstats);
1856         stats->bytes = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_BYTES]));
1857         stats->pkts = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_PACKETS]));
1858         preempt_enable();
1859
1860         return newstats;
1861 }
1862
1863 static void nft_chain_stats_replace(struct nft_trans *trans)
1864 {
1865         struct nft_base_chain *chain = nft_base_chain(trans->ctx.chain);
1866
1867         if (!nft_trans_chain_stats(trans))
1868                 return;
1869
1870         nft_trans_chain_stats(trans) =
1871                 rcu_replace_pointer(chain->stats, nft_trans_chain_stats(trans),
1872                                     lockdep_commit_lock_is_held(trans->ctx.net));
1873
1874         if (!nft_trans_chain_stats(trans))
1875                 static_branch_inc(&nft_counters_enabled);
1876 }
1877
1878 static void nf_tables_chain_free_chain_rules(struct nft_chain *chain)
1879 {
1880         struct nft_rule_blob *g0 = rcu_dereference_raw(chain->blob_gen_0);
1881         struct nft_rule_blob *g1 = rcu_dereference_raw(chain->blob_gen_1);
1882
1883         if (g0 != g1)
1884                 kvfree(g1);
1885         kvfree(g0);
1886
1887         /* should be NULL either via abort or via successful commit */
1888         WARN_ON_ONCE(chain->blob_next);
1889         kvfree(chain->blob_next);
1890 }
1891
1892 void nf_tables_chain_destroy(struct nft_ctx *ctx)
1893 {
1894         struct nft_chain *chain = ctx->chain;
1895         struct nft_hook *hook, *next;
1896
1897         if (WARN_ON(chain->use > 0))
1898                 return;
1899
1900         /* no concurrent access possible anymore */
1901         nf_tables_chain_free_chain_rules(chain);
1902
1903         if (nft_is_base_chain(chain)) {
1904                 struct nft_base_chain *basechain = nft_base_chain(chain);
1905
1906                 if (nft_base_chain_netdev(ctx->family, basechain->ops.hooknum)) {
1907                         list_for_each_entry_safe(hook, next,
1908                                                  &basechain->hook_list, list) {
1909                                 list_del_rcu(&hook->list);
1910                                 kfree_rcu(hook, rcu);
1911                         }
1912                 }
1913                 module_put(basechain->type->owner);
1914                 if (rcu_access_pointer(basechain->stats)) {
1915                         static_branch_dec(&nft_counters_enabled);
1916                         free_percpu(rcu_dereference_raw(basechain->stats));
1917                 }
1918                 kfree(chain->name);
1919                 kfree(chain->udata);
1920                 kfree(basechain);
1921         } else {
1922                 kfree(chain->name);
1923                 kfree(chain->udata);
1924                 kfree(chain);
1925         }
1926 }
1927
1928 static struct nft_hook *nft_netdev_hook_alloc(struct net *net,
1929                                               const struct nlattr *attr)
1930 {
1931         struct net_device *dev;
1932         char ifname[IFNAMSIZ];
1933         struct nft_hook *hook;
1934         int err;
1935
1936         hook = kmalloc(sizeof(struct nft_hook), GFP_KERNEL_ACCOUNT);
1937         if (!hook) {
1938                 err = -ENOMEM;
1939                 goto err_hook_alloc;
1940         }
1941
1942         nla_strscpy(ifname, attr, IFNAMSIZ);
1943         /* nf_tables_netdev_event() is called under rtnl_mutex, this is
1944          * indirectly serializing all the other holders of the commit_mutex with
1945          * the rtnl_mutex.
1946          */
1947         dev = __dev_get_by_name(net, ifname);
1948         if (!dev) {
1949                 err = -ENOENT;
1950                 goto err_hook_dev;
1951         }
1952         hook->ops.dev = dev;
1953
1954         return hook;
1955
1956 err_hook_dev:
1957         kfree(hook);
1958 err_hook_alloc:
1959         return ERR_PTR(err);
1960 }
1961
1962 static struct nft_hook *nft_hook_list_find(struct list_head *hook_list,
1963                                            const struct nft_hook *this)
1964 {
1965         struct nft_hook *hook;
1966
1967         list_for_each_entry(hook, hook_list, list) {
1968                 if (this->ops.dev == hook->ops.dev)
1969                         return hook;
1970         }
1971
1972         return NULL;
1973 }
1974
1975 static int nf_tables_parse_netdev_hooks(struct net *net,
1976                                         const struct nlattr *attr,
1977                                         struct list_head *hook_list,
1978                                         struct netlink_ext_ack *extack)
1979 {
1980         struct nft_hook *hook, *next;
1981         const struct nlattr *tmp;
1982         int rem, n = 0, err;
1983
1984         nla_for_each_nested(tmp, attr, rem) {
1985                 if (nla_type(tmp) != NFTA_DEVICE_NAME) {
1986                         err = -EINVAL;
1987                         goto err_hook;
1988                 }
1989
1990                 hook = nft_netdev_hook_alloc(net, tmp);
1991                 if (IS_ERR(hook)) {
1992                         NL_SET_BAD_ATTR(extack, tmp);
1993                         err = PTR_ERR(hook);
1994                         goto err_hook;
1995                 }
1996                 if (nft_hook_list_find(hook_list, hook)) {
1997                         NL_SET_BAD_ATTR(extack, tmp);
1998                         kfree(hook);
1999                         err = -EEXIST;
2000                         goto err_hook;
2001                 }
2002                 list_add_tail(&hook->list, hook_list);
2003                 n++;
2004
2005                 if (n == NFT_NETDEVICE_MAX) {
2006                         err = -EFBIG;
2007                         goto err_hook;
2008                 }
2009         }
2010
2011         return 0;
2012
2013 err_hook:
2014         list_for_each_entry_safe(hook, next, hook_list, list) {
2015                 list_del(&hook->list);
2016                 kfree(hook);
2017         }
2018         return err;
2019 }
2020
2021 struct nft_chain_hook {
2022         u32                             num;
2023         s32                             priority;
2024         const struct nft_chain_type     *type;
2025         struct list_head                list;
2026 };
2027
2028 static int nft_chain_parse_netdev(struct net *net, struct nlattr *tb[],
2029                                   struct list_head *hook_list,
2030                                   struct netlink_ext_ack *extack, u32 flags)
2031 {
2032         struct nft_hook *hook;
2033         int err;
2034
2035         if (tb[NFTA_HOOK_DEV]) {
2036                 hook = nft_netdev_hook_alloc(net, tb[NFTA_HOOK_DEV]);
2037                 if (IS_ERR(hook)) {
2038                         NL_SET_BAD_ATTR(extack, tb[NFTA_HOOK_DEV]);
2039                         return PTR_ERR(hook);
2040                 }
2041
2042                 list_add_tail(&hook->list, hook_list);
2043         } else if (tb[NFTA_HOOK_DEVS]) {
2044                 err = nf_tables_parse_netdev_hooks(net, tb[NFTA_HOOK_DEVS],
2045                                                    hook_list, extack);
2046                 if (err < 0)
2047                         return err;
2048
2049         }
2050
2051         if (flags & NFT_CHAIN_HW_OFFLOAD &&
2052             list_empty(hook_list))
2053                 return -EINVAL;
2054
2055         return 0;
2056 }
2057
2058 static int nft_chain_parse_hook(struct net *net,
2059                                 struct nft_base_chain *basechain,
2060                                 const struct nlattr * const nla[],
2061                                 struct nft_chain_hook *hook, u8 family,
2062                                 u32 flags, struct netlink_ext_ack *extack)
2063 {
2064         struct nftables_pernet *nft_net = nft_pernet(net);
2065         struct nlattr *ha[NFTA_HOOK_MAX + 1];
2066         const struct nft_chain_type *type;
2067         int err;
2068
2069         lockdep_assert_held(&nft_net->commit_mutex);
2070         lockdep_nfnl_nft_mutex_not_held();
2071
2072         err = nla_parse_nested_deprecated(ha, NFTA_HOOK_MAX,
2073                                           nla[NFTA_CHAIN_HOOK],
2074                                           nft_hook_policy, NULL);
2075         if (err < 0)
2076                 return err;
2077
2078         if (!basechain) {
2079                 if (!ha[NFTA_HOOK_HOOKNUM] ||
2080                     !ha[NFTA_HOOK_PRIORITY]) {
2081                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2082                         return -ENOENT;
2083                 }
2084
2085                 hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
2086                 hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
2087
2088                 type = __nft_chain_type_get(family, NFT_CHAIN_T_DEFAULT);
2089                 if (!type)
2090                         return -EOPNOTSUPP;
2091
2092                 if (nla[NFTA_CHAIN_TYPE]) {
2093                         type = nf_tables_chain_type_lookup(net, nla[NFTA_CHAIN_TYPE],
2094                                                            family, true);
2095                         if (IS_ERR(type)) {
2096                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2097                                 return PTR_ERR(type);
2098                         }
2099                 }
2100                 if (hook->num >= NFT_MAX_HOOKS || !(type->hook_mask & (1 << hook->num)))
2101                         return -EOPNOTSUPP;
2102
2103                 if (type->type == NFT_CHAIN_T_NAT &&
2104                     hook->priority <= NF_IP_PRI_CONNTRACK)
2105                         return -EOPNOTSUPP;
2106         } else {
2107                 if (ha[NFTA_HOOK_HOOKNUM]) {
2108                         hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
2109                         if (hook->num != basechain->ops.hooknum)
2110                                 return -EOPNOTSUPP;
2111                 }
2112                 if (ha[NFTA_HOOK_PRIORITY]) {
2113                         hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
2114                         if (hook->priority != basechain->ops.priority)
2115                                 return -EOPNOTSUPP;
2116                 }
2117
2118                 type = basechain->type;
2119         }
2120
2121         if (!try_module_get(type->owner)) {
2122                 if (nla[NFTA_CHAIN_TYPE])
2123                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2124                 return -ENOENT;
2125         }
2126
2127         hook->type = type;
2128
2129         INIT_LIST_HEAD(&hook->list);
2130         if (nft_base_chain_netdev(family, hook->num)) {
2131                 err = nft_chain_parse_netdev(net, ha, &hook->list, extack, flags);
2132                 if (err < 0) {
2133                         module_put(type->owner);
2134                         return err;
2135                 }
2136         } else if (ha[NFTA_HOOK_DEV] || ha[NFTA_HOOK_DEVS]) {
2137                 module_put(type->owner);
2138                 return -EOPNOTSUPP;
2139         }
2140
2141         return 0;
2142 }
2143
2144 static void nft_chain_release_hook(struct nft_chain_hook *hook)
2145 {
2146         struct nft_hook *h, *next;
2147
2148         list_for_each_entry_safe(h, next, &hook->list, list) {
2149                 list_del(&h->list);
2150                 kfree(h);
2151         }
2152         module_put(hook->type->owner);
2153 }
2154
2155 static void nft_last_rule(const struct nft_chain *chain, const void *ptr)
2156 {
2157         struct nft_rule_dp_last *lrule;
2158
2159         BUILD_BUG_ON(offsetof(struct nft_rule_dp_last, end) != 0);
2160
2161         lrule = (struct nft_rule_dp_last *)ptr;
2162         lrule->end.is_last = 1;
2163         lrule->chain = chain;
2164         /* blob size does not include the trailer rule */
2165 }
2166
2167 static struct nft_rule_blob *nf_tables_chain_alloc_rules(const struct nft_chain *chain,
2168                                                          unsigned int size)
2169 {
2170         struct nft_rule_blob *blob;
2171
2172         if (size > INT_MAX)
2173                 return NULL;
2174
2175         size += sizeof(struct nft_rule_blob) + sizeof(struct nft_rule_dp_last);
2176
2177         blob = kvmalloc(size, GFP_KERNEL_ACCOUNT);
2178         if (!blob)
2179                 return NULL;
2180
2181         blob->size = 0;
2182         nft_last_rule(chain, blob->data);
2183
2184         return blob;
2185 }
2186
2187 static void nft_basechain_hook_init(struct nf_hook_ops *ops, u8 family,
2188                                     const struct nft_chain_hook *hook,
2189                                     struct nft_chain *chain)
2190 {
2191         ops->pf                 = family;
2192         ops->hooknum            = hook->num;
2193         ops->priority           = hook->priority;
2194         ops->priv               = chain;
2195         ops->hook               = hook->type->hooks[ops->hooknum];
2196         ops->hook_ops_type      = NF_HOOK_OP_NF_TABLES;
2197 }
2198
2199 static int nft_basechain_init(struct nft_base_chain *basechain, u8 family,
2200                               struct nft_chain_hook *hook, u32 flags)
2201 {
2202         struct nft_chain *chain;
2203         struct nft_hook *h;
2204
2205         basechain->type = hook->type;
2206         INIT_LIST_HEAD(&basechain->hook_list);
2207         chain = &basechain->chain;
2208
2209         if (nft_base_chain_netdev(family, hook->num)) {
2210                 list_splice_init(&hook->list, &basechain->hook_list);
2211                 list_for_each_entry(h, &basechain->hook_list, list)
2212                         nft_basechain_hook_init(&h->ops, family, hook, chain);
2213         }
2214         nft_basechain_hook_init(&basechain->ops, family, hook, chain);
2215
2216         chain->flags |= NFT_CHAIN_BASE | flags;
2217         basechain->policy = NF_ACCEPT;
2218         if (chain->flags & NFT_CHAIN_HW_OFFLOAD &&
2219             !nft_chain_offload_support(basechain)) {
2220                 list_splice_init(&basechain->hook_list, &hook->list);
2221                 return -EOPNOTSUPP;
2222         }
2223
2224         flow_block_init(&basechain->flow_block);
2225
2226         return 0;
2227 }
2228
2229 static int nft_chain_add(struct nft_table *table, struct nft_chain *chain)
2230 {
2231         int err;
2232
2233         err = rhltable_insert_key(&table->chains_ht, chain->name,
2234                                   &chain->rhlhead, nft_chain_ht_params);
2235         if (err)
2236                 return err;
2237
2238         list_add_tail_rcu(&chain->list, &table->chains);
2239
2240         return 0;
2241 }
2242
2243 static u64 chain_id;
2244
2245 static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 genmask,
2246                               u8 policy, u32 flags,
2247                               struct netlink_ext_ack *extack)
2248 {
2249         const struct nlattr * const *nla = ctx->nla;
2250         struct nft_table *table = ctx->table;
2251         struct nft_base_chain *basechain;
2252         struct net *net = ctx->net;
2253         char name[NFT_NAME_MAXLEN];
2254         struct nft_rule_blob *blob;
2255         struct nft_trans *trans;
2256         struct nft_chain *chain;
2257         int err;
2258
2259         if (table->use == UINT_MAX)
2260                 return -EOVERFLOW;
2261
2262         if (nla[NFTA_CHAIN_HOOK]) {
2263                 struct nft_stats __percpu *stats = NULL;
2264                 struct nft_chain_hook hook = {};
2265
2266                 if (flags & NFT_CHAIN_BINDING)
2267                         return -EOPNOTSUPP;
2268
2269                 err = nft_chain_parse_hook(net, NULL, nla, &hook, family, flags,
2270                                            extack);
2271                 if (err < 0)
2272                         return err;
2273
2274                 basechain = kzalloc(sizeof(*basechain), GFP_KERNEL_ACCOUNT);
2275                 if (basechain == NULL) {
2276                         nft_chain_release_hook(&hook);
2277                         return -ENOMEM;
2278                 }
2279                 chain = &basechain->chain;
2280
2281                 if (nla[NFTA_CHAIN_COUNTERS]) {
2282                         stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2283                         if (IS_ERR(stats)) {
2284                                 nft_chain_release_hook(&hook);
2285                                 kfree(basechain);
2286                                 return PTR_ERR(stats);
2287                         }
2288                         rcu_assign_pointer(basechain->stats, stats);
2289                 }
2290
2291                 err = nft_basechain_init(basechain, family, &hook, flags);
2292                 if (err < 0) {
2293                         nft_chain_release_hook(&hook);
2294                         kfree(basechain);
2295                         free_percpu(stats);
2296                         return err;
2297                 }
2298                 if (stats)
2299                         static_branch_inc(&nft_counters_enabled);
2300         } else {
2301                 if (flags & NFT_CHAIN_BASE)
2302                         return -EINVAL;
2303                 if (flags & NFT_CHAIN_HW_OFFLOAD)
2304                         return -EOPNOTSUPP;
2305
2306                 chain = kzalloc(sizeof(*chain), GFP_KERNEL_ACCOUNT);
2307                 if (chain == NULL)
2308                         return -ENOMEM;
2309
2310                 chain->flags = flags;
2311         }
2312         ctx->chain = chain;
2313
2314         INIT_LIST_HEAD(&chain->rules);
2315         chain->handle = nf_tables_alloc_handle(table);
2316         chain->table = table;
2317
2318         if (nla[NFTA_CHAIN_NAME]) {
2319                 chain->name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
2320         } else {
2321                 if (!(flags & NFT_CHAIN_BINDING)) {
2322                         err = -EINVAL;
2323                         goto err_destroy_chain;
2324                 }
2325
2326                 snprintf(name, sizeof(name), "__chain%llu", ++chain_id);
2327                 chain->name = kstrdup(name, GFP_KERNEL_ACCOUNT);
2328         }
2329
2330         if (!chain->name) {
2331                 err = -ENOMEM;
2332                 goto err_destroy_chain;
2333         }
2334
2335         if (nla[NFTA_CHAIN_USERDATA]) {
2336                 chain->udata = nla_memdup(nla[NFTA_CHAIN_USERDATA], GFP_KERNEL_ACCOUNT);
2337                 if (chain->udata == NULL) {
2338                         err = -ENOMEM;
2339                         goto err_destroy_chain;
2340                 }
2341                 chain->udlen = nla_len(nla[NFTA_CHAIN_USERDATA]);
2342         }
2343
2344         blob = nf_tables_chain_alloc_rules(chain, 0);
2345         if (!blob) {
2346                 err = -ENOMEM;
2347                 goto err_destroy_chain;
2348         }
2349
2350         RCU_INIT_POINTER(chain->blob_gen_0, blob);
2351         RCU_INIT_POINTER(chain->blob_gen_1, blob);
2352
2353         err = nf_tables_register_hook(net, table, chain);
2354         if (err < 0)
2355                 goto err_destroy_chain;
2356
2357         trans = nft_trans_chain_add(ctx, NFT_MSG_NEWCHAIN);
2358         if (IS_ERR(trans)) {
2359                 err = PTR_ERR(trans);
2360                 goto err_unregister_hook;
2361         }
2362
2363         nft_trans_chain_policy(trans) = NFT_CHAIN_POLICY_UNSET;
2364         if (nft_is_base_chain(chain))
2365                 nft_trans_chain_policy(trans) = policy;
2366
2367         err = nft_chain_add(table, chain);
2368         if (err < 0) {
2369                 nft_trans_destroy(trans);
2370                 goto err_unregister_hook;
2371         }
2372
2373         table->use++;
2374
2375         return 0;
2376 err_unregister_hook:
2377         nf_tables_unregister_hook(net, table, chain);
2378 err_destroy_chain:
2379         nf_tables_chain_destroy(ctx);
2380
2381         return err;
2382 }
2383
2384 static int nf_tables_updchain(struct nft_ctx *ctx, u8 genmask, u8 policy,
2385                               u32 flags, const struct nlattr *attr,
2386                               struct netlink_ext_ack *extack)
2387 {
2388         const struct nlattr * const *nla = ctx->nla;
2389         struct nft_base_chain *basechain = NULL;
2390         struct nft_table *table = ctx->table;
2391         struct nft_chain *chain = ctx->chain;
2392         struct nft_chain_hook hook = {};
2393         struct nft_stats *stats = NULL;
2394         struct nft_hook *h, *next;
2395         struct nf_hook_ops *ops;
2396         struct nft_trans *trans;
2397         bool unregister = false;
2398         int err;
2399
2400         if (chain->flags ^ flags)
2401                 return -EOPNOTSUPP;
2402
2403         INIT_LIST_HEAD(&hook.list);
2404
2405         if (nla[NFTA_CHAIN_HOOK]) {
2406                 if (!nft_is_base_chain(chain)) {
2407                         NL_SET_BAD_ATTR(extack, attr);
2408                         return -EEXIST;
2409                 }
2410
2411                 basechain = nft_base_chain(chain);
2412                 err = nft_chain_parse_hook(ctx->net, basechain, nla, &hook,
2413                                            ctx->family, flags, extack);
2414                 if (err < 0)
2415                         return err;
2416
2417                 if (basechain->type != hook.type) {
2418                         nft_chain_release_hook(&hook);
2419                         NL_SET_BAD_ATTR(extack, attr);
2420                         return -EEXIST;
2421                 }
2422
2423                 if (nft_base_chain_netdev(ctx->family, basechain->ops.hooknum)) {
2424                         list_for_each_entry_safe(h, next, &hook.list, list) {
2425                                 h->ops.pf       = basechain->ops.pf;
2426                                 h->ops.hooknum  = basechain->ops.hooknum;
2427                                 h->ops.priority = basechain->ops.priority;
2428                                 h->ops.priv     = basechain->ops.priv;
2429                                 h->ops.hook     = basechain->ops.hook;
2430
2431                                 if (nft_hook_list_find(&basechain->hook_list, h)) {
2432                                         list_del(&h->list);
2433                                         kfree(h);
2434                                 }
2435                         }
2436                 } else {
2437                         ops = &basechain->ops;
2438                         if (ops->hooknum != hook.num ||
2439                             ops->priority != hook.priority) {
2440                                 nft_chain_release_hook(&hook);
2441                                 NL_SET_BAD_ATTR(extack, attr);
2442                                 return -EEXIST;
2443                         }
2444                 }
2445         }
2446
2447         if (nla[NFTA_CHAIN_HANDLE] &&
2448             nla[NFTA_CHAIN_NAME]) {
2449                 struct nft_chain *chain2;
2450
2451                 chain2 = nft_chain_lookup(ctx->net, table,
2452                                           nla[NFTA_CHAIN_NAME], genmask);
2453                 if (!IS_ERR(chain2)) {
2454                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2455                         err = -EEXIST;
2456                         goto err_hooks;
2457                 }
2458         }
2459
2460         if (nla[NFTA_CHAIN_COUNTERS]) {
2461                 if (!nft_is_base_chain(chain)) {
2462                         err = -EOPNOTSUPP;
2463                         goto err_hooks;
2464                 }
2465
2466                 stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2467                 if (IS_ERR(stats)) {
2468                         err = PTR_ERR(stats);
2469                         goto err_hooks;
2470                 }
2471         }
2472
2473         if (!(table->flags & NFT_TABLE_F_DORMANT) &&
2474             nft_is_base_chain(chain) &&
2475             !list_empty(&hook.list)) {
2476                 basechain = nft_base_chain(chain);
2477                 ops = &basechain->ops;
2478
2479                 if (nft_base_chain_netdev(table->family, basechain->ops.hooknum)) {
2480                         err = nft_netdev_register_hooks(ctx->net, &hook.list);
2481                         if (err < 0)
2482                                 goto err_hooks;
2483                 }
2484         }
2485
2486         unregister = true;
2487         err = -ENOMEM;
2488         trans = nft_trans_alloc(ctx, NFT_MSG_NEWCHAIN,
2489                                 sizeof(struct nft_trans_chain));
2490         if (trans == NULL)
2491                 goto err_trans;
2492
2493         nft_trans_chain_stats(trans) = stats;
2494         nft_trans_chain_update(trans) = true;
2495
2496         if (nla[NFTA_CHAIN_POLICY])
2497                 nft_trans_chain_policy(trans) = policy;
2498         else
2499                 nft_trans_chain_policy(trans) = -1;
2500
2501         if (nla[NFTA_CHAIN_HANDLE] &&
2502             nla[NFTA_CHAIN_NAME]) {
2503                 struct nftables_pernet *nft_net = nft_pernet(ctx->net);
2504                 struct nft_trans *tmp;
2505                 char *name;
2506
2507                 err = -ENOMEM;
2508                 name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
2509                 if (!name)
2510                         goto err_trans;
2511
2512                 err = -EEXIST;
2513                 list_for_each_entry(tmp, &nft_net->commit_list, list) {
2514                         if (tmp->msg_type == NFT_MSG_NEWCHAIN &&
2515                             tmp->ctx.table == table &&
2516                             nft_trans_chain_update(tmp) &&
2517                             nft_trans_chain_name(tmp) &&
2518                             strcmp(name, nft_trans_chain_name(tmp)) == 0) {
2519                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2520                                 kfree(name);
2521                                 goto err_trans;
2522                         }
2523                 }
2524
2525                 nft_trans_chain_name(trans) = name;
2526         }
2527
2528         nft_trans_basechain(trans) = basechain;
2529         INIT_LIST_HEAD(&nft_trans_chain_hooks(trans));
2530         list_splice(&hook.list, &nft_trans_chain_hooks(trans));
2531
2532         nft_trans_commit_list_add_tail(ctx->net, trans);
2533
2534         return 0;
2535
2536 err_trans:
2537         free_percpu(stats);
2538         kfree(trans);
2539 err_hooks:
2540         if (nla[NFTA_CHAIN_HOOK]) {
2541                 list_for_each_entry_safe(h, next, &hook.list, list) {
2542                         if (unregister)
2543                                 nf_unregister_net_hook(ctx->net, &h->ops);
2544                         list_del(&h->list);
2545                         kfree_rcu(h, rcu);
2546                 }
2547                 module_put(hook.type->owner);
2548         }
2549
2550         return err;
2551 }
2552
2553 static struct nft_chain *nft_chain_lookup_byid(const struct net *net,
2554                                                const struct nft_table *table,
2555                                                const struct nlattr *nla)
2556 {
2557         struct nftables_pernet *nft_net = nft_pernet(net);
2558         u32 id = ntohl(nla_get_be32(nla));
2559         struct nft_trans *trans;
2560
2561         list_for_each_entry(trans, &nft_net->commit_list, list) {
2562                 struct nft_chain *chain = trans->ctx.chain;
2563
2564                 if (trans->msg_type == NFT_MSG_NEWCHAIN &&
2565                     chain->table == table &&
2566                     id == nft_trans_chain_id(trans))
2567                         return chain;
2568         }
2569         return ERR_PTR(-ENOENT);
2570 }
2571
2572 static int nf_tables_newchain(struct sk_buff *skb, const struct nfnl_info *info,
2573                               const struct nlattr * const nla[])
2574 {
2575         struct nftables_pernet *nft_net = nft_pernet(info->net);
2576         struct netlink_ext_ack *extack = info->extack;
2577         u8 genmask = nft_genmask_next(info->net);
2578         u8 family = info->nfmsg->nfgen_family;
2579         struct nft_chain *chain = NULL;
2580         struct net *net = info->net;
2581         const struct nlattr *attr;
2582         struct nft_table *table;
2583         u8 policy = NF_ACCEPT;
2584         struct nft_ctx ctx;
2585         u64 handle = 0;
2586         u32 flags = 0;
2587
2588         lockdep_assert_held(&nft_net->commit_mutex);
2589
2590         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
2591                                  NETLINK_CB(skb).portid);
2592         if (IS_ERR(table)) {
2593                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2594                 return PTR_ERR(table);
2595         }
2596
2597         chain = NULL;
2598         attr = nla[NFTA_CHAIN_NAME];
2599
2600         if (nla[NFTA_CHAIN_HANDLE]) {
2601                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_CHAIN_HANDLE]));
2602                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2603                 if (IS_ERR(chain)) {
2604                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_HANDLE]);
2605                         return PTR_ERR(chain);
2606                 }
2607                 attr = nla[NFTA_CHAIN_HANDLE];
2608         } else if (nla[NFTA_CHAIN_NAME]) {
2609                 chain = nft_chain_lookup(net, table, attr, genmask);
2610                 if (IS_ERR(chain)) {
2611                         if (PTR_ERR(chain) != -ENOENT) {
2612                                 NL_SET_BAD_ATTR(extack, attr);
2613                                 return PTR_ERR(chain);
2614                         }
2615                         chain = NULL;
2616                 }
2617         } else if (!nla[NFTA_CHAIN_ID]) {
2618                 return -EINVAL;
2619         }
2620
2621         if (nla[NFTA_CHAIN_POLICY]) {
2622                 if (chain != NULL &&
2623                     !nft_is_base_chain(chain)) {
2624                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2625                         return -EOPNOTSUPP;
2626                 }
2627
2628                 if (chain == NULL &&
2629                     nla[NFTA_CHAIN_HOOK] == NULL) {
2630                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2631                         return -EOPNOTSUPP;
2632                 }
2633
2634                 policy = ntohl(nla_get_be32(nla[NFTA_CHAIN_POLICY]));
2635                 switch (policy) {
2636                 case NF_DROP:
2637                 case NF_ACCEPT:
2638                         break;
2639                 default:
2640                         return -EINVAL;
2641                 }
2642         }
2643
2644         if (nla[NFTA_CHAIN_FLAGS])
2645                 flags = ntohl(nla_get_be32(nla[NFTA_CHAIN_FLAGS]));
2646         else if (chain)
2647                 flags = chain->flags;
2648
2649         if (flags & ~NFT_CHAIN_FLAGS)
2650                 return -EOPNOTSUPP;
2651
2652         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
2653
2654         if (chain != NULL) {
2655                 if (chain->flags & NFT_CHAIN_BINDING)
2656                         return -EINVAL;
2657
2658                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
2659                         NL_SET_BAD_ATTR(extack, attr);
2660                         return -EEXIST;
2661                 }
2662                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
2663                         return -EOPNOTSUPP;
2664
2665                 flags |= chain->flags & NFT_CHAIN_BASE;
2666                 return nf_tables_updchain(&ctx, genmask, policy, flags, attr,
2667                                           extack);
2668         }
2669
2670         return nf_tables_addchain(&ctx, family, genmask, policy, flags, extack);
2671 }
2672
2673 static int nft_delchain_hook(struct nft_ctx *ctx, struct nft_chain *chain,
2674                              struct netlink_ext_ack *extack)
2675 {
2676         const struct nlattr * const *nla = ctx->nla;
2677         struct nft_chain_hook chain_hook = {};
2678         struct nft_base_chain *basechain;
2679         struct nft_hook *this, *hook;
2680         LIST_HEAD(chain_del_list);
2681         struct nft_trans *trans;
2682         int err;
2683
2684         if (!nft_is_base_chain(chain))
2685                 return -EOPNOTSUPP;
2686
2687         basechain = nft_base_chain(chain);
2688         err = nft_chain_parse_hook(ctx->net, basechain, nla, &chain_hook,
2689                                    ctx->family, chain->flags, extack);
2690         if (err < 0)
2691                 return err;
2692
2693         list_for_each_entry(this, &chain_hook.list, list) {
2694                 hook = nft_hook_list_find(&basechain->hook_list, this);
2695                 if (!hook) {
2696                         err = -ENOENT;
2697                         goto err_chain_del_hook;
2698                 }
2699                 list_move(&hook->list, &chain_del_list);
2700         }
2701
2702         trans = nft_trans_alloc(ctx, NFT_MSG_DELCHAIN,
2703                                 sizeof(struct nft_trans_chain));
2704         if (!trans) {
2705                 err = -ENOMEM;
2706                 goto err_chain_del_hook;
2707         }
2708
2709         nft_trans_basechain(trans) = basechain;
2710         nft_trans_chain_update(trans) = true;
2711         INIT_LIST_HEAD(&nft_trans_chain_hooks(trans));
2712         list_splice(&chain_del_list, &nft_trans_chain_hooks(trans));
2713         nft_chain_release_hook(&chain_hook);
2714
2715         nft_trans_commit_list_add_tail(ctx->net, trans);
2716
2717         return 0;
2718
2719 err_chain_del_hook:
2720         list_splice(&chain_del_list, &basechain->hook_list);
2721         nft_chain_release_hook(&chain_hook);
2722
2723         return err;
2724 }
2725
2726 static int nf_tables_delchain(struct sk_buff *skb, const struct nfnl_info *info,
2727                               const struct nlattr * const nla[])
2728 {
2729         struct netlink_ext_ack *extack = info->extack;
2730         u8 genmask = nft_genmask_next(info->net);
2731         u8 family = info->nfmsg->nfgen_family;
2732         struct net *net = info->net;
2733         const struct nlattr *attr;
2734         struct nft_table *table;
2735         struct nft_chain *chain;
2736         struct nft_rule *rule;
2737         struct nft_ctx ctx;
2738         u64 handle;
2739         u32 use;
2740         int err;
2741
2742         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
2743                                  NETLINK_CB(skb).portid);
2744         if (IS_ERR(table)) {
2745                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2746                 return PTR_ERR(table);
2747         }
2748
2749         if (nla[NFTA_CHAIN_HANDLE]) {
2750                 attr = nla[NFTA_CHAIN_HANDLE];
2751                 handle = be64_to_cpu(nla_get_be64(attr));
2752                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2753         } else {
2754                 attr = nla[NFTA_CHAIN_NAME];
2755                 chain = nft_chain_lookup(net, table, attr, genmask);
2756         }
2757         if (IS_ERR(chain)) {
2758                 if (PTR_ERR(chain) == -ENOENT &&
2759                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYCHAIN)
2760                         return 0;
2761
2762                 NL_SET_BAD_ATTR(extack, attr);
2763                 return PTR_ERR(chain);
2764         }
2765
2766         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
2767
2768         if (nla[NFTA_CHAIN_HOOK]) {
2769                 if (chain->flags & NFT_CHAIN_HW_OFFLOAD)
2770                         return -EOPNOTSUPP;
2771
2772                 return nft_delchain_hook(&ctx, chain, extack);
2773         }
2774
2775         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
2776             chain->use > 0)
2777                 return -EBUSY;
2778
2779         use = chain->use;
2780         list_for_each_entry(rule, &chain->rules, list) {
2781                 if (!nft_is_active_next(net, rule))
2782                         continue;
2783                 use--;
2784
2785                 err = nft_delrule(&ctx, rule);
2786                 if (err < 0)
2787                         return err;
2788         }
2789
2790         /* There are rules and elements that are still holding references to us,
2791          * we cannot do a recursive removal in this case.
2792          */
2793         if (use > 0) {
2794                 NL_SET_BAD_ATTR(extack, attr);
2795                 return -EBUSY;
2796         }
2797
2798         return nft_delchain(&ctx);
2799 }
2800
2801 /*
2802  * Expressions
2803  */
2804
2805 /**
2806  *      nft_register_expr - register nf_tables expr type
2807  *      @type: expr type
2808  *
2809  *      Registers the expr type for use with nf_tables. Returns zero on
2810  *      success or a negative errno code otherwise.
2811  */
2812 int nft_register_expr(struct nft_expr_type *type)
2813 {
2814         nfnl_lock(NFNL_SUBSYS_NFTABLES);
2815         if (type->family == NFPROTO_UNSPEC)
2816                 list_add_tail_rcu(&type->list, &nf_tables_expressions);
2817         else
2818                 list_add_rcu(&type->list, &nf_tables_expressions);
2819         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
2820         return 0;
2821 }
2822 EXPORT_SYMBOL_GPL(nft_register_expr);
2823
2824 /**
2825  *      nft_unregister_expr - unregister nf_tables expr type
2826  *      @type: expr type
2827  *
2828  *      Unregisters the expr typefor use with nf_tables.
2829  */
2830 void nft_unregister_expr(struct nft_expr_type *type)
2831 {
2832         nfnl_lock(NFNL_SUBSYS_NFTABLES);
2833         list_del_rcu(&type->list);
2834         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
2835 }
2836 EXPORT_SYMBOL_GPL(nft_unregister_expr);
2837
2838 static const struct nft_expr_type *__nft_expr_type_get(u8 family,
2839                                                        struct nlattr *nla)
2840 {
2841         const struct nft_expr_type *type, *candidate = NULL;
2842
2843         list_for_each_entry(type, &nf_tables_expressions, list) {
2844                 if (!nla_strcmp(nla, type->name)) {
2845                         if (!type->family && !candidate)
2846                                 candidate = type;
2847                         else if (type->family == family)
2848                                 candidate = type;
2849                 }
2850         }
2851         return candidate;
2852 }
2853
2854 #ifdef CONFIG_MODULES
2855 static int nft_expr_type_request_module(struct net *net, u8 family,
2856                                         struct nlattr *nla)
2857 {
2858         if (nft_request_module(net, "nft-expr-%u-%.*s", family,
2859                                nla_len(nla), (char *)nla_data(nla)) == -EAGAIN)
2860                 return -EAGAIN;
2861
2862         return 0;
2863 }
2864 #endif
2865
2866 static const struct nft_expr_type *nft_expr_type_get(struct net *net,
2867                                                      u8 family,
2868                                                      struct nlattr *nla)
2869 {
2870         const struct nft_expr_type *type;
2871
2872         if (nla == NULL)
2873                 return ERR_PTR(-EINVAL);
2874
2875         type = __nft_expr_type_get(family, nla);
2876         if (type != NULL && try_module_get(type->owner))
2877                 return type;
2878
2879         lockdep_nfnl_nft_mutex_not_held();
2880 #ifdef CONFIG_MODULES
2881         if (type == NULL) {
2882                 if (nft_expr_type_request_module(net, family, nla) == -EAGAIN)
2883                         return ERR_PTR(-EAGAIN);
2884
2885                 if (nft_request_module(net, "nft-expr-%.*s",
2886                                        nla_len(nla),
2887                                        (char *)nla_data(nla)) == -EAGAIN)
2888                         return ERR_PTR(-EAGAIN);
2889         }
2890 #endif
2891         return ERR_PTR(-ENOENT);
2892 }
2893
2894 static const struct nla_policy nft_expr_policy[NFTA_EXPR_MAX + 1] = {
2895         [NFTA_EXPR_NAME]        = { .type = NLA_STRING,
2896                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
2897         [NFTA_EXPR_DATA]        = { .type = NLA_NESTED },
2898 };
2899
2900 static int nf_tables_fill_expr_info(struct sk_buff *skb,
2901                                     const struct nft_expr *expr, bool reset)
2902 {
2903         if (nla_put_string(skb, NFTA_EXPR_NAME, expr->ops->type->name))
2904                 goto nla_put_failure;
2905
2906         if (expr->ops->dump) {
2907                 struct nlattr *data = nla_nest_start_noflag(skb,
2908                                                             NFTA_EXPR_DATA);
2909                 if (data == NULL)
2910                         goto nla_put_failure;
2911                 if (expr->ops->dump(skb, expr, reset) < 0)
2912                         goto nla_put_failure;
2913                 nla_nest_end(skb, data);
2914         }
2915
2916         return skb->len;
2917
2918 nla_put_failure:
2919         return -1;
2920 };
2921
2922 int nft_expr_dump(struct sk_buff *skb, unsigned int attr,
2923                   const struct nft_expr *expr, bool reset)
2924 {
2925         struct nlattr *nest;
2926
2927         nest = nla_nest_start_noflag(skb, attr);
2928         if (!nest)
2929                 goto nla_put_failure;
2930         if (nf_tables_fill_expr_info(skb, expr, reset) < 0)
2931                 goto nla_put_failure;
2932         nla_nest_end(skb, nest);
2933         return 0;
2934
2935 nla_put_failure:
2936         return -1;
2937 }
2938
2939 struct nft_expr_info {
2940         const struct nft_expr_ops       *ops;
2941         const struct nlattr             *attr;
2942         struct nlattr                   *tb[NFT_EXPR_MAXATTR + 1];
2943 };
2944
2945 static int nf_tables_expr_parse(const struct nft_ctx *ctx,
2946                                 const struct nlattr *nla,
2947                                 struct nft_expr_info *info)
2948 {
2949         const struct nft_expr_type *type;
2950         const struct nft_expr_ops *ops;
2951         struct nlattr *tb[NFTA_EXPR_MAX + 1];
2952         int err;
2953
2954         err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
2955                                           nft_expr_policy, NULL);
2956         if (err < 0)
2957                 return err;
2958
2959         type = nft_expr_type_get(ctx->net, ctx->family, tb[NFTA_EXPR_NAME]);
2960         if (IS_ERR(type))
2961                 return PTR_ERR(type);
2962
2963         if (tb[NFTA_EXPR_DATA]) {
2964                 err = nla_parse_nested_deprecated(info->tb, type->maxattr,
2965                                                   tb[NFTA_EXPR_DATA],
2966                                                   type->policy, NULL);
2967                 if (err < 0)
2968                         goto err1;
2969         } else
2970                 memset(info->tb, 0, sizeof(info->tb[0]) * (type->maxattr + 1));
2971
2972         if (type->select_ops != NULL) {
2973                 ops = type->select_ops(ctx,
2974                                        (const struct nlattr * const *)info->tb);
2975                 if (IS_ERR(ops)) {
2976                         err = PTR_ERR(ops);
2977 #ifdef CONFIG_MODULES
2978                         if (err == -EAGAIN)
2979                                 if (nft_expr_type_request_module(ctx->net,
2980                                                                  ctx->family,
2981                                                                  tb[NFTA_EXPR_NAME]) != -EAGAIN)
2982                                         err = -ENOENT;
2983 #endif
2984                         goto err1;
2985                 }
2986         } else
2987                 ops = type->ops;
2988
2989         info->attr = nla;
2990         info->ops = ops;
2991
2992         return 0;
2993
2994 err1:
2995         module_put(type->owner);
2996         return err;
2997 }
2998
2999 int nft_expr_inner_parse(const struct nft_ctx *ctx, const struct nlattr *nla,
3000                          struct nft_expr_info *info)
3001 {
3002         struct nlattr *tb[NFTA_EXPR_MAX + 1];
3003         const struct nft_expr_type *type;
3004         int err;
3005
3006         err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
3007                                           nft_expr_policy, NULL);
3008         if (err < 0)
3009                 return err;
3010
3011         if (!tb[NFTA_EXPR_DATA])
3012                 return -EINVAL;
3013
3014         type = __nft_expr_type_get(ctx->family, tb[NFTA_EXPR_NAME]);
3015         if (!type)
3016                 return -ENOENT;
3017
3018         if (!type->inner_ops)
3019                 return -EOPNOTSUPP;
3020
3021         err = nla_parse_nested_deprecated(info->tb, type->maxattr,
3022                                           tb[NFTA_EXPR_DATA],
3023                                           type->policy, NULL);
3024         if (err < 0)
3025                 goto err_nla_parse;
3026
3027         info->attr = nla;
3028         info->ops = type->inner_ops;
3029
3030         return 0;
3031
3032 err_nla_parse:
3033         return err;
3034 }
3035
3036 static int nf_tables_newexpr(const struct nft_ctx *ctx,
3037                              const struct nft_expr_info *expr_info,
3038                              struct nft_expr *expr)
3039 {
3040         const struct nft_expr_ops *ops = expr_info->ops;
3041         int err;
3042
3043         expr->ops = ops;
3044         if (ops->init) {
3045                 err = ops->init(ctx, expr, (const struct nlattr **)expr_info->tb);
3046                 if (err < 0)
3047                         goto err1;
3048         }
3049
3050         return 0;
3051 err1:
3052         expr->ops = NULL;
3053         return err;
3054 }
3055
3056 static void nf_tables_expr_destroy(const struct nft_ctx *ctx,
3057                                    struct nft_expr *expr)
3058 {
3059         const struct nft_expr_type *type = expr->ops->type;
3060
3061         if (expr->ops->destroy)
3062                 expr->ops->destroy(ctx, expr);
3063         module_put(type->owner);
3064 }
3065
3066 static struct nft_expr *nft_expr_init(const struct nft_ctx *ctx,
3067                                       const struct nlattr *nla)
3068 {
3069         struct nft_expr_info expr_info;
3070         struct nft_expr *expr;
3071         struct module *owner;
3072         int err;
3073
3074         err = nf_tables_expr_parse(ctx, nla, &expr_info);
3075         if (err < 0)
3076                 goto err_expr_parse;
3077
3078         err = -EOPNOTSUPP;
3079         if (!(expr_info.ops->type->flags & NFT_EXPR_STATEFUL))
3080                 goto err_expr_stateful;
3081
3082         err = -ENOMEM;
3083         expr = kzalloc(expr_info.ops->size, GFP_KERNEL_ACCOUNT);
3084         if (expr == NULL)
3085                 goto err_expr_stateful;
3086
3087         err = nf_tables_newexpr(ctx, &expr_info, expr);
3088         if (err < 0)
3089                 goto err_expr_new;
3090
3091         return expr;
3092 err_expr_new:
3093         kfree(expr);
3094 err_expr_stateful:
3095         owner = expr_info.ops->type->owner;
3096         if (expr_info.ops->type->release_ops)
3097                 expr_info.ops->type->release_ops(expr_info.ops);
3098
3099         module_put(owner);
3100 err_expr_parse:
3101         return ERR_PTR(err);
3102 }
3103
3104 int nft_expr_clone(struct nft_expr *dst, struct nft_expr *src)
3105 {
3106         int err;
3107
3108         if (src->ops->clone) {
3109                 dst->ops = src->ops;
3110                 err = src->ops->clone(dst, src);
3111                 if (err < 0)
3112                         return err;
3113         } else {
3114                 memcpy(dst, src, src->ops->size);
3115         }
3116
3117         __module_get(src->ops->type->owner);
3118
3119         return 0;
3120 }
3121
3122 void nft_expr_destroy(const struct nft_ctx *ctx, struct nft_expr *expr)
3123 {
3124         nf_tables_expr_destroy(ctx, expr);
3125         kfree(expr);
3126 }
3127
3128 /*
3129  * Rules
3130  */
3131
3132 static struct nft_rule *__nft_rule_lookup(const struct nft_chain *chain,
3133                                           u64 handle)
3134 {
3135         struct nft_rule *rule;
3136
3137         // FIXME: this sucks
3138         list_for_each_entry_rcu(rule, &chain->rules, list) {
3139                 if (handle == rule->handle)
3140                         return rule;
3141         }
3142
3143         return ERR_PTR(-ENOENT);
3144 }
3145
3146 static struct nft_rule *nft_rule_lookup(const struct nft_chain *chain,
3147                                         const struct nlattr *nla)
3148 {
3149         if (nla == NULL)
3150                 return ERR_PTR(-EINVAL);
3151
3152         return __nft_rule_lookup(chain, be64_to_cpu(nla_get_be64(nla)));
3153 }
3154
3155 static const struct nla_policy nft_rule_policy[NFTA_RULE_MAX + 1] = {
3156         [NFTA_RULE_TABLE]       = { .type = NLA_STRING,
3157                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
3158         [NFTA_RULE_CHAIN]       = { .type = NLA_STRING,
3159                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
3160         [NFTA_RULE_HANDLE]      = { .type = NLA_U64 },
3161         [NFTA_RULE_EXPRESSIONS] = { .type = NLA_NESTED },
3162         [NFTA_RULE_COMPAT]      = { .type = NLA_NESTED },
3163         [NFTA_RULE_POSITION]    = { .type = NLA_U64 },
3164         [NFTA_RULE_USERDATA]    = { .type = NLA_BINARY,
3165                                     .len = NFT_USERDATA_MAXLEN },
3166         [NFTA_RULE_ID]          = { .type = NLA_U32 },
3167         [NFTA_RULE_POSITION_ID] = { .type = NLA_U32 },
3168         [NFTA_RULE_CHAIN_ID]    = { .type = NLA_U32 },
3169 };
3170
3171 static int nf_tables_fill_rule_info(struct sk_buff *skb, struct net *net,
3172                                     u32 portid, u32 seq, int event,
3173                                     u32 flags, int family,
3174                                     const struct nft_table *table,
3175                                     const struct nft_chain *chain,
3176                                     const struct nft_rule *rule, u64 handle,
3177                                     bool reset)
3178 {
3179         struct nlmsghdr *nlh;
3180         const struct nft_expr *expr, *next;
3181         struct nlattr *list;
3182         u16 type = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
3183
3184         nlh = nfnl_msg_put(skb, portid, seq, type, flags, family, NFNETLINK_V0,
3185                            nft_base_seq(net));
3186         if (!nlh)
3187                 goto nla_put_failure;
3188
3189         if (nla_put_string(skb, NFTA_RULE_TABLE, table->name))
3190                 goto nla_put_failure;
3191         if (nla_put_string(skb, NFTA_RULE_CHAIN, chain->name))
3192                 goto nla_put_failure;
3193         if (nla_put_be64(skb, NFTA_RULE_HANDLE, cpu_to_be64(rule->handle),
3194                          NFTA_RULE_PAD))
3195                 goto nla_put_failure;
3196
3197         if (event != NFT_MSG_DELRULE && handle) {
3198                 if (nla_put_be64(skb, NFTA_RULE_POSITION, cpu_to_be64(handle),
3199                                  NFTA_RULE_PAD))
3200                         goto nla_put_failure;
3201         }
3202
3203         if (chain->flags & NFT_CHAIN_HW_OFFLOAD)
3204                 nft_flow_rule_stats(chain, rule);
3205
3206         list = nla_nest_start_noflag(skb, NFTA_RULE_EXPRESSIONS);
3207         if (list == NULL)
3208                 goto nla_put_failure;
3209         nft_rule_for_each_expr(expr, next, rule) {
3210                 if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr, reset) < 0)
3211                         goto nla_put_failure;
3212         }
3213         nla_nest_end(skb, list);
3214
3215         if (rule->udata) {
3216                 struct nft_userdata *udata = nft_userdata(rule);
3217                 if (nla_put(skb, NFTA_RULE_USERDATA, udata->len + 1,
3218                             udata->data) < 0)
3219                         goto nla_put_failure;
3220         }
3221
3222         nlmsg_end(skb, nlh);
3223         return 0;
3224
3225 nla_put_failure:
3226         nlmsg_trim(skb, nlh);
3227         return -1;
3228 }
3229
3230 static void nf_tables_rule_notify(const struct nft_ctx *ctx,
3231                                   const struct nft_rule *rule, int event)
3232 {
3233         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
3234         const struct nft_rule *prule;
3235         struct sk_buff *skb;
3236         u64 handle = 0;
3237         u16 flags = 0;
3238         int err;
3239
3240         if (!ctx->report &&
3241             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
3242                 return;
3243
3244         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
3245         if (skb == NULL)
3246                 goto err;
3247
3248         if (event == NFT_MSG_NEWRULE &&
3249             !list_is_first(&rule->list, &ctx->chain->rules) &&
3250             !list_is_last(&rule->list, &ctx->chain->rules)) {
3251                 prule = list_prev_entry(rule, list);
3252                 handle = prule->handle;
3253         }
3254         if (ctx->flags & (NLM_F_APPEND | NLM_F_REPLACE))
3255                 flags |= NLM_F_APPEND;
3256         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
3257                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
3258
3259         err = nf_tables_fill_rule_info(skb, ctx->net, ctx->portid, ctx->seq,
3260                                        event, flags, ctx->family, ctx->table,
3261                                        ctx->chain, rule, handle, false);
3262         if (err < 0) {
3263                 kfree_skb(skb);
3264                 goto err;
3265         }
3266
3267         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
3268         return;
3269 err:
3270         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
3271 }
3272
3273 struct nft_rule_dump_ctx {
3274         char *table;
3275         char *chain;
3276 };
3277
3278 static int __nf_tables_dump_rules(struct sk_buff *skb,
3279                                   unsigned int *idx,
3280                                   struct netlink_callback *cb,
3281                                   const struct nft_table *table,
3282                                   const struct nft_chain *chain,
3283                                   bool reset)
3284 {
3285         struct net *net = sock_net(skb->sk);
3286         const struct nft_rule *rule, *prule;
3287         unsigned int s_idx = cb->args[0];
3288         u64 handle;
3289
3290         prule = NULL;
3291         list_for_each_entry_rcu(rule, &chain->rules, list) {
3292                 if (!nft_is_active(net, rule))
3293                         goto cont_skip;
3294                 if (*idx < s_idx)
3295                         goto cont;
3296                 if (*idx > s_idx) {
3297                         memset(&cb->args[1], 0,
3298                                         sizeof(cb->args) - sizeof(cb->args[0]));
3299                 }
3300                 if (prule)
3301                         handle = prule->handle;
3302                 else
3303                         handle = 0;
3304
3305                 if (nf_tables_fill_rule_info(skb, net, NETLINK_CB(cb->skb).portid,
3306                                         cb->nlh->nlmsg_seq,
3307                                         NFT_MSG_NEWRULE,
3308                                         NLM_F_MULTI | NLM_F_APPEND,
3309                                         table->family,
3310                                         table, chain, rule, handle, reset) < 0)
3311                         return 1;
3312
3313                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
3314 cont:
3315                 prule = rule;
3316 cont_skip:
3317                 (*idx)++;
3318         }
3319         return 0;
3320 }
3321
3322 static int nf_tables_dump_rules(struct sk_buff *skb,
3323                                 struct netlink_callback *cb)
3324 {
3325         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
3326         const struct nft_rule_dump_ctx *ctx = cb->data;
3327         struct nft_table *table;
3328         const struct nft_chain *chain;
3329         unsigned int idx = 0;
3330         struct net *net = sock_net(skb->sk);
3331         int family = nfmsg->nfgen_family;
3332         struct nftables_pernet *nft_net;
3333         bool reset = false;
3334
3335         if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETRULE_RESET)
3336                 reset = true;
3337
3338         rcu_read_lock();
3339         nft_net = nft_pernet(net);
3340         cb->seq = READ_ONCE(nft_net->base_seq);
3341
3342         list_for_each_entry_rcu(table, &nft_net->tables, list) {
3343                 if (family != NFPROTO_UNSPEC && family != table->family)
3344                         continue;
3345
3346                 if (ctx && ctx->table && strcmp(ctx->table, table->name) != 0)
3347                         continue;
3348
3349                 if (ctx && ctx->table && ctx->chain) {
3350                         struct rhlist_head *list, *tmp;
3351
3352                         list = rhltable_lookup(&table->chains_ht, ctx->chain,
3353                                                nft_chain_ht_params);
3354                         if (!list)
3355                                 goto done;
3356
3357                         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
3358                                 if (!nft_is_active(net, chain))
3359                                         continue;
3360                                 __nf_tables_dump_rules(skb, &idx,
3361                                                        cb, table, chain, reset);
3362                                 break;
3363                         }
3364                         goto done;
3365                 }
3366
3367                 list_for_each_entry_rcu(chain, &table->chains, list) {
3368                         if (__nf_tables_dump_rules(skb, &idx,
3369                                                    cb, table, chain, reset))
3370                                 goto done;
3371                 }
3372
3373                 if (ctx && ctx->table)
3374                         break;
3375         }
3376 done:
3377         rcu_read_unlock();
3378
3379         cb->args[0] = idx;
3380         return skb->len;
3381 }
3382
3383 static int nf_tables_dump_rules_start(struct netlink_callback *cb)
3384 {
3385         const struct nlattr * const *nla = cb->data;
3386         struct nft_rule_dump_ctx *ctx = NULL;
3387
3388         if (nla[NFTA_RULE_TABLE] || nla[NFTA_RULE_CHAIN]) {
3389                 ctx = kzalloc(sizeof(*ctx), GFP_ATOMIC);
3390                 if (!ctx)
3391                         return -ENOMEM;
3392
3393                 if (nla[NFTA_RULE_TABLE]) {
3394                         ctx->table = nla_strdup(nla[NFTA_RULE_TABLE],
3395                                                         GFP_ATOMIC);
3396                         if (!ctx->table) {
3397                                 kfree(ctx);
3398                                 return -ENOMEM;
3399                         }
3400                 }
3401                 if (nla[NFTA_RULE_CHAIN]) {
3402                         ctx->chain = nla_strdup(nla[NFTA_RULE_CHAIN],
3403                                                 GFP_ATOMIC);
3404                         if (!ctx->chain) {
3405                                 kfree(ctx->table);
3406                                 kfree(ctx);
3407                                 return -ENOMEM;
3408                         }
3409                 }
3410         }
3411
3412         cb->data = ctx;
3413         return 0;
3414 }
3415
3416 static int nf_tables_dump_rules_done(struct netlink_callback *cb)
3417 {
3418         struct nft_rule_dump_ctx *ctx = cb->data;
3419
3420         if (ctx) {
3421                 kfree(ctx->table);
3422                 kfree(ctx->chain);
3423                 kfree(ctx);
3424         }
3425         return 0;
3426 }
3427
3428 /* called with rcu_read_lock held */
3429 static int nf_tables_getrule(struct sk_buff *skb, const struct nfnl_info *info,
3430                              const struct nlattr * const nla[])
3431 {
3432         struct netlink_ext_ack *extack = info->extack;
3433         u8 genmask = nft_genmask_cur(info->net);
3434         u8 family = info->nfmsg->nfgen_family;
3435         const struct nft_chain *chain;
3436         const struct nft_rule *rule;
3437         struct net *net = info->net;
3438         struct nft_table *table;
3439         struct sk_buff *skb2;
3440         bool reset = false;
3441         int err;
3442
3443         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
3444                 struct netlink_dump_control c = {
3445                         .start= nf_tables_dump_rules_start,
3446                         .dump = nf_tables_dump_rules,
3447                         .done = nf_tables_dump_rules_done,
3448                         .module = THIS_MODULE,
3449                         .data = (void *)nla,
3450                 };
3451
3452                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
3453         }
3454
3455         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask, 0);
3456         if (IS_ERR(table)) {
3457                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3458                 return PTR_ERR(table);
3459         }
3460
3461         chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN], genmask);
3462         if (IS_ERR(chain)) {
3463                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3464                 return PTR_ERR(chain);
3465         }
3466
3467         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
3468         if (IS_ERR(rule)) {
3469                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3470                 return PTR_ERR(rule);
3471         }
3472
3473         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
3474         if (!skb2)
3475                 return -ENOMEM;
3476
3477         if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETRULE_RESET)
3478                 reset = true;
3479
3480         err = nf_tables_fill_rule_info(skb2, net, NETLINK_CB(skb).portid,
3481                                        info->nlh->nlmsg_seq, NFT_MSG_NEWRULE, 0,
3482                                        family, table, chain, rule, 0, reset);
3483         if (err < 0)
3484                 goto err_fill_rule_info;
3485
3486         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
3487
3488 err_fill_rule_info:
3489         kfree_skb(skb2);
3490         return err;
3491 }
3492
3493 static void nf_tables_rule_destroy(const struct nft_ctx *ctx,
3494                                    struct nft_rule *rule)
3495 {
3496         struct nft_expr *expr, *next;
3497
3498         /*
3499          * Careful: some expressions might not be initialized in case this
3500          * is called on error from nf_tables_newrule().
3501          */
3502         expr = nft_expr_first(rule);
3503         while (nft_expr_more(rule, expr)) {
3504                 next = nft_expr_next(expr);
3505                 nf_tables_expr_destroy(ctx, expr);
3506                 expr = next;
3507         }
3508         kfree(rule);
3509 }
3510
3511 void nf_tables_rule_release(const struct nft_ctx *ctx, struct nft_rule *rule)
3512 {
3513         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_RELEASE);
3514         nf_tables_rule_destroy(ctx, rule);
3515 }
3516
3517 int nft_chain_validate(const struct nft_ctx *ctx, const struct nft_chain *chain)
3518 {
3519         struct nft_expr *expr, *last;
3520         const struct nft_data *data;
3521         struct nft_rule *rule;
3522         int err;
3523
3524         if (ctx->level == NFT_JUMP_STACK_SIZE)
3525                 return -EMLINK;
3526
3527         list_for_each_entry(rule, &chain->rules, list) {
3528                 if (!nft_is_active_next(ctx->net, rule))
3529                         continue;
3530
3531                 nft_rule_for_each_expr(expr, last, rule) {
3532                         if (!expr->ops->validate)
3533                                 continue;
3534
3535                         err = expr->ops->validate(ctx, expr, &data);
3536                         if (err < 0)
3537                                 return err;
3538                 }
3539
3540                 cond_resched();
3541         }
3542
3543         return 0;
3544 }
3545 EXPORT_SYMBOL_GPL(nft_chain_validate);
3546
3547 static int nft_table_validate(struct net *net, const struct nft_table *table)
3548 {
3549         struct nft_chain *chain;
3550         struct nft_ctx ctx = {
3551                 .net    = net,
3552                 .family = table->family,
3553         };
3554         int err;
3555
3556         list_for_each_entry(chain, &table->chains, list) {
3557                 if (!nft_is_base_chain(chain))
3558                         continue;
3559
3560                 ctx.chain = chain;
3561                 err = nft_chain_validate(&ctx, chain);
3562                 if (err < 0)
3563                         return err;
3564         }
3565
3566         return 0;
3567 }
3568
3569 int nft_setelem_validate(const struct nft_ctx *ctx, struct nft_set *set,
3570                          const struct nft_set_iter *iter,
3571                          struct nft_set_elem *elem)
3572 {
3573         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
3574         struct nft_ctx *pctx = (struct nft_ctx *)ctx;
3575         const struct nft_data *data;
3576         int err;
3577
3578         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
3579             *nft_set_ext_flags(ext) & NFT_SET_ELEM_INTERVAL_END)
3580                 return 0;
3581
3582         data = nft_set_ext_data(ext);
3583         switch (data->verdict.code) {
3584         case NFT_JUMP:
3585         case NFT_GOTO:
3586                 pctx->level++;
3587                 err = nft_chain_validate(ctx, data->verdict.chain);
3588                 if (err < 0)
3589                         return err;
3590                 pctx->level--;
3591                 break;
3592         default:
3593                 break;
3594         }
3595
3596         return 0;
3597 }
3598
3599 struct nft_set_elem_catchall {
3600         struct list_head        list;
3601         struct rcu_head         rcu;
3602         void                    *elem;
3603 };
3604
3605 int nft_set_catchall_validate(const struct nft_ctx *ctx, struct nft_set *set)
3606 {
3607         u8 genmask = nft_genmask_next(ctx->net);
3608         struct nft_set_elem_catchall *catchall;
3609         struct nft_set_elem elem;
3610         struct nft_set_ext *ext;
3611         int ret = 0;
3612
3613         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
3614                 ext = nft_set_elem_ext(set, catchall->elem);
3615                 if (!nft_set_elem_active(ext, genmask))
3616                         continue;
3617
3618                 elem.priv = catchall->elem;
3619                 ret = nft_setelem_validate(ctx, set, NULL, &elem);
3620                 if (ret < 0)
3621                         return ret;
3622         }
3623
3624         return ret;
3625 }
3626
3627 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
3628                                              const struct nft_chain *chain,
3629                                              const struct nlattr *nla);
3630
3631 #define NFT_RULE_MAXEXPRS       128
3632
3633 static int nf_tables_newrule(struct sk_buff *skb, const struct nfnl_info *info,
3634                              const struct nlattr * const nla[])
3635 {
3636         struct nftables_pernet *nft_net = nft_pernet(info->net);
3637         struct netlink_ext_ack *extack = info->extack;
3638         unsigned int size, i, n, ulen = 0, usize = 0;
3639         u8 genmask = nft_genmask_next(info->net);
3640         struct nft_rule *rule, *old_rule = NULL;
3641         struct nft_expr_info *expr_info = NULL;
3642         u8 family = info->nfmsg->nfgen_family;
3643         struct nft_flow_rule *flow = NULL;
3644         struct net *net = info->net;
3645         struct nft_userdata *udata;
3646         struct nft_table *table;
3647         struct nft_chain *chain;
3648         struct nft_trans *trans;
3649         u64 handle, pos_handle;
3650         struct nft_expr *expr;
3651         struct nft_ctx ctx;
3652         struct nlattr *tmp;
3653         int err, rem;
3654
3655         lockdep_assert_held(&nft_net->commit_mutex);
3656
3657         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
3658                                  NETLINK_CB(skb).portid);
3659         if (IS_ERR(table)) {
3660                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3661                 return PTR_ERR(table);
3662         }
3663
3664         if (nla[NFTA_RULE_CHAIN]) {
3665                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
3666                                          genmask);
3667                 if (IS_ERR(chain)) {
3668                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3669                         return PTR_ERR(chain);
3670                 }
3671                 if (nft_chain_is_bound(chain))
3672                         return -EOPNOTSUPP;
3673
3674         } else if (nla[NFTA_RULE_CHAIN_ID]) {
3675                 chain = nft_chain_lookup_byid(net, table, nla[NFTA_RULE_CHAIN_ID]);
3676                 if (IS_ERR(chain)) {
3677                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN_ID]);
3678                         return PTR_ERR(chain);
3679                 }
3680         } else {
3681                 return -EINVAL;
3682         }
3683
3684         if (nla[NFTA_RULE_HANDLE]) {
3685                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_HANDLE]));
3686                 rule = __nft_rule_lookup(chain, handle);
3687                 if (IS_ERR(rule)) {
3688                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3689                         return PTR_ERR(rule);
3690                 }
3691
3692                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
3693                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3694                         return -EEXIST;
3695                 }
3696                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
3697                         old_rule = rule;
3698                 else
3699                         return -EOPNOTSUPP;
3700         } else {
3701                 if (!(info->nlh->nlmsg_flags & NLM_F_CREATE) ||
3702                     info->nlh->nlmsg_flags & NLM_F_REPLACE)
3703                         return -EINVAL;
3704                 handle = nf_tables_alloc_handle(table);
3705
3706                 if (chain->use == UINT_MAX)
3707                         return -EOVERFLOW;
3708
3709                 if (nla[NFTA_RULE_POSITION]) {
3710                         pos_handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_POSITION]));
3711                         old_rule = __nft_rule_lookup(chain, pos_handle);
3712                         if (IS_ERR(old_rule)) {
3713                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION]);
3714                                 return PTR_ERR(old_rule);
3715                         }
3716                 } else if (nla[NFTA_RULE_POSITION_ID]) {
3717                         old_rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_POSITION_ID]);
3718                         if (IS_ERR(old_rule)) {
3719                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION_ID]);
3720                                 return PTR_ERR(old_rule);
3721                         }
3722                 }
3723         }
3724
3725         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
3726
3727         n = 0;
3728         size = 0;
3729         if (nla[NFTA_RULE_EXPRESSIONS]) {
3730                 expr_info = kvmalloc_array(NFT_RULE_MAXEXPRS,
3731                                            sizeof(struct nft_expr_info),
3732                                            GFP_KERNEL);
3733                 if (!expr_info)
3734                         return -ENOMEM;
3735
3736                 nla_for_each_nested(tmp, nla[NFTA_RULE_EXPRESSIONS], rem) {
3737                         err = -EINVAL;
3738                         if (nla_type(tmp) != NFTA_LIST_ELEM)
3739                                 goto err_release_expr;
3740                         if (n == NFT_RULE_MAXEXPRS)
3741                                 goto err_release_expr;
3742                         err = nf_tables_expr_parse(&ctx, tmp, &expr_info[n]);
3743                         if (err < 0) {
3744                                 NL_SET_BAD_ATTR(extack, tmp);
3745                                 goto err_release_expr;
3746                         }
3747                         size += expr_info[n].ops->size;
3748                         n++;
3749                 }
3750         }
3751         /* Check for overflow of dlen field */
3752         err = -EFBIG;
3753         if (size >= 1 << 12)
3754                 goto err_release_expr;
3755
3756         if (nla[NFTA_RULE_USERDATA]) {
3757                 ulen = nla_len(nla[NFTA_RULE_USERDATA]);
3758                 if (ulen > 0)
3759                         usize = sizeof(struct nft_userdata) + ulen;
3760         }
3761
3762         err = -ENOMEM;
3763         rule = kzalloc(sizeof(*rule) + size + usize, GFP_KERNEL_ACCOUNT);
3764         if (rule == NULL)
3765                 goto err_release_expr;
3766
3767         nft_activate_next(net, rule);
3768
3769         rule->handle = handle;
3770         rule->dlen   = size;
3771         rule->udata  = ulen ? 1 : 0;
3772
3773         if (ulen) {
3774                 udata = nft_userdata(rule);
3775                 udata->len = ulen - 1;
3776                 nla_memcpy(udata->data, nla[NFTA_RULE_USERDATA], ulen);
3777         }
3778
3779         expr = nft_expr_first(rule);
3780         for (i = 0; i < n; i++) {
3781                 err = nf_tables_newexpr(&ctx, &expr_info[i], expr);
3782                 if (err < 0) {
3783                         NL_SET_BAD_ATTR(extack, expr_info[i].attr);
3784                         goto err_release_rule;
3785                 }
3786
3787                 if (expr_info[i].ops->validate)
3788                         nft_validate_state_update(table, NFT_VALIDATE_NEED);
3789
3790                 expr_info[i].ops = NULL;
3791                 expr = nft_expr_next(expr);
3792         }
3793
3794         if (chain->flags & NFT_CHAIN_HW_OFFLOAD) {
3795                 flow = nft_flow_rule_create(net, rule);
3796                 if (IS_ERR(flow)) {
3797                         err = PTR_ERR(flow);
3798                         goto err_release_rule;
3799                 }
3800         }
3801
3802         if (info->nlh->nlmsg_flags & NLM_F_REPLACE) {
3803                 err = nft_delrule(&ctx, old_rule);
3804                 if (err < 0)
3805                         goto err_destroy_flow_rule;
3806
3807                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
3808                 if (trans == NULL) {
3809                         err = -ENOMEM;
3810                         goto err_destroy_flow_rule;
3811                 }
3812                 list_add_tail_rcu(&rule->list, &old_rule->list);
3813         } else {
3814                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
3815                 if (!trans) {
3816                         err = -ENOMEM;
3817                         goto err_destroy_flow_rule;
3818                 }
3819
3820                 if (info->nlh->nlmsg_flags & NLM_F_APPEND) {
3821                         if (old_rule)
3822                                 list_add_rcu(&rule->list, &old_rule->list);
3823                         else
3824                                 list_add_tail_rcu(&rule->list, &chain->rules);
3825                  } else {
3826                         if (old_rule)
3827                                 list_add_tail_rcu(&rule->list, &old_rule->list);
3828                         else
3829                                 list_add_rcu(&rule->list, &chain->rules);
3830                 }
3831         }
3832         kvfree(expr_info);
3833         chain->use++;
3834
3835         if (flow)
3836                 nft_trans_flow_rule(trans) = flow;
3837
3838         if (table->validate_state == NFT_VALIDATE_DO)
3839                 return nft_table_validate(net, table);
3840
3841         return 0;
3842
3843 err_destroy_flow_rule:
3844         if (flow)
3845                 nft_flow_rule_destroy(flow);
3846 err_release_rule:
3847         nft_rule_expr_deactivate(&ctx, rule, NFT_TRANS_PREPARE);
3848         nf_tables_rule_destroy(&ctx, rule);
3849 err_release_expr:
3850         for (i = 0; i < n; i++) {
3851                 if (expr_info[i].ops) {
3852                         module_put(expr_info[i].ops->type->owner);
3853                         if (expr_info[i].ops->type->release_ops)
3854                                 expr_info[i].ops->type->release_ops(expr_info[i].ops);
3855                 }
3856         }
3857         kvfree(expr_info);
3858
3859         return err;
3860 }
3861
3862 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
3863                                              const struct nft_chain *chain,
3864                                              const struct nlattr *nla)
3865 {
3866         struct nftables_pernet *nft_net = nft_pernet(net);
3867         u32 id = ntohl(nla_get_be32(nla));
3868         struct nft_trans *trans;
3869
3870         list_for_each_entry(trans, &nft_net->commit_list, list) {
3871                 if (trans->msg_type == NFT_MSG_NEWRULE &&
3872                     trans->ctx.chain == chain &&
3873                     id == nft_trans_rule_id(trans))
3874                         return nft_trans_rule(trans);
3875         }
3876         return ERR_PTR(-ENOENT);
3877 }
3878
3879 static int nf_tables_delrule(struct sk_buff *skb, const struct nfnl_info *info,
3880                              const struct nlattr * const nla[])
3881 {
3882         struct netlink_ext_ack *extack = info->extack;
3883         u8 genmask = nft_genmask_next(info->net);
3884         u8 family = info->nfmsg->nfgen_family;
3885         struct nft_chain *chain = NULL;
3886         struct net *net = info->net;
3887         struct nft_table *table;
3888         struct nft_rule *rule;
3889         struct nft_ctx ctx;
3890         int err = 0;
3891
3892         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
3893                                  NETLINK_CB(skb).portid);
3894         if (IS_ERR(table)) {
3895                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3896                 return PTR_ERR(table);
3897         }
3898
3899         if (nla[NFTA_RULE_CHAIN]) {
3900                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
3901                                          genmask);
3902                 if (IS_ERR(chain)) {
3903                         if (PTR_ERR(chain) == -ENOENT &&
3904                             NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYRULE)
3905                                 return 0;
3906
3907                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3908                         return PTR_ERR(chain);
3909                 }
3910                 if (nft_chain_is_bound(chain))
3911                         return -EOPNOTSUPP;
3912         }
3913
3914         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
3915
3916         if (chain) {
3917                 if (nla[NFTA_RULE_HANDLE]) {
3918                         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
3919                         if (IS_ERR(rule)) {
3920                                 if (PTR_ERR(rule) == -ENOENT &&
3921                                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYRULE)
3922                                         return 0;
3923
3924                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3925                                 return PTR_ERR(rule);
3926                         }
3927
3928                         err = nft_delrule(&ctx, rule);
3929                 } else if (nla[NFTA_RULE_ID]) {
3930                         rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_ID]);
3931                         if (IS_ERR(rule)) {
3932                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_ID]);
3933                                 return PTR_ERR(rule);
3934                         }
3935
3936                         err = nft_delrule(&ctx, rule);
3937                 } else {
3938                         err = nft_delrule_by_chain(&ctx);
3939                 }
3940         } else {
3941                 list_for_each_entry(chain, &table->chains, list) {
3942                         if (!nft_is_active_next(net, chain))
3943                                 continue;
3944
3945                         ctx.chain = chain;
3946                         err = nft_delrule_by_chain(&ctx);
3947                         if (err < 0)
3948                                 break;
3949                 }
3950         }
3951
3952         return err;
3953 }
3954
3955 /*
3956  * Sets
3957  */
3958 static const struct nft_set_type *nft_set_types[] = {
3959         &nft_set_hash_fast_type,
3960         &nft_set_hash_type,
3961         &nft_set_rhash_type,
3962         &nft_set_bitmap_type,
3963         &nft_set_rbtree_type,
3964 #if defined(CONFIG_X86_64) && !defined(CONFIG_UML)
3965         &nft_set_pipapo_avx2_type,
3966 #endif
3967         &nft_set_pipapo_type,
3968 };
3969
3970 #define NFT_SET_FEATURES        (NFT_SET_INTERVAL | NFT_SET_MAP | \
3971                                  NFT_SET_TIMEOUT | NFT_SET_OBJECT | \
3972                                  NFT_SET_EVAL)
3973
3974 static bool nft_set_ops_candidate(const struct nft_set_type *type, u32 flags)
3975 {
3976         return (flags & type->features) == (flags & NFT_SET_FEATURES);
3977 }
3978
3979 /*
3980  * Select a set implementation based on the data characteristics and the
3981  * given policy. The total memory use might not be known if no size is
3982  * given, in that case the amount of memory per element is used.
3983  */
3984 static const struct nft_set_ops *
3985 nft_select_set_ops(const struct nft_ctx *ctx,
3986                    const struct nlattr * const nla[],
3987                    const struct nft_set_desc *desc)
3988 {
3989         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
3990         const struct nft_set_ops *ops, *bops;
3991         struct nft_set_estimate est, best;
3992         const struct nft_set_type *type;
3993         u32 flags = 0;
3994         int i;
3995
3996         lockdep_assert_held(&nft_net->commit_mutex);
3997         lockdep_nfnl_nft_mutex_not_held();
3998
3999         if (nla[NFTA_SET_FLAGS] != NULL)
4000                 flags = ntohl(nla_get_be32(nla[NFTA_SET_FLAGS]));
4001
4002         bops        = NULL;
4003         best.size   = ~0;
4004         best.lookup = ~0;
4005         best.space  = ~0;
4006
4007         for (i = 0; i < ARRAY_SIZE(nft_set_types); i++) {
4008                 type = nft_set_types[i];
4009                 ops = &type->ops;
4010
4011                 if (!nft_set_ops_candidate(type, flags))
4012                         continue;
4013                 if (!ops->estimate(desc, flags, &est))
4014                         continue;
4015
4016                 switch (desc->policy) {
4017                 case NFT_SET_POL_PERFORMANCE:
4018                         if (est.lookup < best.lookup)
4019                                 break;
4020                         if (est.lookup == best.lookup &&
4021                             est.space < best.space)
4022                                 break;
4023                         continue;
4024                 case NFT_SET_POL_MEMORY:
4025                         if (!desc->size) {
4026                                 if (est.space < best.space)
4027                                         break;
4028                                 if (est.space == best.space &&
4029                                     est.lookup < best.lookup)
4030                                         break;
4031                         } else if (est.size < best.size || !bops) {
4032                                 break;
4033                         }
4034                         continue;
4035                 default:
4036                         break;
4037                 }
4038
4039                 bops = ops;
4040                 best = est;
4041         }
4042
4043         if (bops != NULL)
4044                 return bops;
4045
4046         return ERR_PTR(-EOPNOTSUPP);
4047 }
4048
4049 static const struct nla_policy nft_set_policy[NFTA_SET_MAX + 1] = {
4050         [NFTA_SET_TABLE]                = { .type = NLA_STRING,
4051                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
4052         [NFTA_SET_NAME]                 = { .type = NLA_STRING,
4053                                             .len = NFT_SET_MAXNAMELEN - 1 },
4054         [NFTA_SET_FLAGS]                = { .type = NLA_U32 },
4055         [NFTA_SET_KEY_TYPE]             = { .type = NLA_U32 },
4056         [NFTA_SET_KEY_LEN]              = { .type = NLA_U32 },
4057         [NFTA_SET_DATA_TYPE]            = { .type = NLA_U32 },
4058         [NFTA_SET_DATA_LEN]             = { .type = NLA_U32 },
4059         [NFTA_SET_POLICY]               = { .type = NLA_U32 },
4060         [NFTA_SET_DESC]                 = { .type = NLA_NESTED },
4061         [NFTA_SET_ID]                   = { .type = NLA_U32 },
4062         [NFTA_SET_TIMEOUT]              = { .type = NLA_U64 },
4063         [NFTA_SET_GC_INTERVAL]          = { .type = NLA_U32 },
4064         [NFTA_SET_USERDATA]             = { .type = NLA_BINARY,
4065                                             .len  = NFT_USERDATA_MAXLEN },
4066         [NFTA_SET_OBJ_TYPE]             = { .type = NLA_U32 },
4067         [NFTA_SET_HANDLE]               = { .type = NLA_U64 },
4068         [NFTA_SET_EXPR]                 = { .type = NLA_NESTED },
4069         [NFTA_SET_EXPRESSIONS]          = { .type = NLA_NESTED },
4070 };
4071
4072 static const struct nla_policy nft_set_desc_policy[NFTA_SET_DESC_MAX + 1] = {
4073         [NFTA_SET_DESC_SIZE]            = { .type = NLA_U32 },
4074         [NFTA_SET_DESC_CONCAT]          = { .type = NLA_NESTED },
4075 };
4076
4077 static struct nft_set *nft_set_lookup(const struct nft_table *table,
4078                                       const struct nlattr *nla, u8 genmask)
4079 {
4080         struct nft_set *set;
4081
4082         if (nla == NULL)
4083                 return ERR_PTR(-EINVAL);
4084
4085         list_for_each_entry_rcu(set, &table->sets, list) {
4086                 if (!nla_strcmp(nla, set->name) &&
4087                     nft_active_genmask(set, genmask))
4088                         return set;
4089         }
4090         return ERR_PTR(-ENOENT);
4091 }
4092
4093 static struct nft_set *nft_set_lookup_byhandle(const struct nft_table *table,
4094                                                const struct nlattr *nla,
4095                                                u8 genmask)
4096 {
4097         struct nft_set *set;
4098
4099         list_for_each_entry(set, &table->sets, list) {
4100                 if (be64_to_cpu(nla_get_be64(nla)) == set->handle &&
4101                     nft_active_genmask(set, genmask))
4102                         return set;
4103         }
4104         return ERR_PTR(-ENOENT);
4105 }
4106
4107 static struct nft_set *nft_set_lookup_byid(const struct net *net,
4108                                            const struct nft_table *table,
4109                                            const struct nlattr *nla, u8 genmask)
4110 {
4111         struct nftables_pernet *nft_net = nft_pernet(net);
4112         u32 id = ntohl(nla_get_be32(nla));
4113         struct nft_trans *trans;
4114
4115         list_for_each_entry(trans, &nft_net->commit_list, list) {
4116                 if (trans->msg_type == NFT_MSG_NEWSET) {
4117                         struct nft_set *set = nft_trans_set(trans);
4118
4119                         if (id == nft_trans_set_id(trans) &&
4120                             set->table == table &&
4121                             nft_active_genmask(set, genmask))
4122                                 return set;
4123                 }
4124         }
4125         return ERR_PTR(-ENOENT);
4126 }
4127
4128 struct nft_set *nft_set_lookup_global(const struct net *net,
4129                                       const struct nft_table *table,
4130                                       const struct nlattr *nla_set_name,
4131                                       const struct nlattr *nla_set_id,
4132                                       u8 genmask)
4133 {
4134         struct nft_set *set;
4135
4136         set = nft_set_lookup(table, nla_set_name, genmask);
4137         if (IS_ERR(set)) {
4138                 if (!nla_set_id)
4139                         return set;
4140
4141                 set = nft_set_lookup_byid(net, table, nla_set_id, genmask);
4142         }
4143         return set;
4144 }
4145 EXPORT_SYMBOL_GPL(nft_set_lookup_global);
4146
4147 static int nf_tables_set_alloc_name(struct nft_ctx *ctx, struct nft_set *set,
4148                                     const char *name)
4149 {
4150         const struct nft_set *i;
4151         const char *p;
4152         unsigned long *inuse;
4153         unsigned int n = 0, min = 0;
4154
4155         p = strchr(name, '%');
4156         if (p != NULL) {
4157                 if (p[1] != 'd' || strchr(p + 2, '%'))
4158                         return -EINVAL;
4159
4160                 inuse = (unsigned long *)get_zeroed_page(GFP_KERNEL);
4161                 if (inuse == NULL)
4162                         return -ENOMEM;
4163 cont:
4164                 list_for_each_entry(i, &ctx->table->sets, list) {
4165                         int tmp;
4166
4167                         if (!nft_is_active_next(ctx->net, i))
4168                                 continue;
4169                         if (!sscanf(i->name, name, &tmp))
4170                                 continue;
4171                         if (tmp < min || tmp >= min + BITS_PER_BYTE * PAGE_SIZE)
4172                                 continue;
4173
4174                         set_bit(tmp - min, inuse);
4175                 }
4176
4177                 n = find_first_zero_bit(inuse, BITS_PER_BYTE * PAGE_SIZE);
4178                 if (n >= BITS_PER_BYTE * PAGE_SIZE) {
4179                         min += BITS_PER_BYTE * PAGE_SIZE;
4180                         memset(inuse, 0, PAGE_SIZE);
4181                         goto cont;
4182                 }
4183                 free_page((unsigned long)inuse);
4184         }
4185
4186         set->name = kasprintf(GFP_KERNEL_ACCOUNT, name, min + n);
4187         if (!set->name)
4188                 return -ENOMEM;
4189
4190         list_for_each_entry(i, &ctx->table->sets, list) {
4191                 if (!nft_is_active_next(ctx->net, i))
4192                         continue;
4193                 if (!strcmp(set->name, i->name)) {
4194                         kfree(set->name);
4195                         set->name = NULL;
4196                         return -ENFILE;
4197                 }
4198         }
4199         return 0;
4200 }
4201
4202 int nf_msecs_to_jiffies64(const struct nlattr *nla, u64 *result)
4203 {
4204         u64 ms = be64_to_cpu(nla_get_be64(nla));
4205         u64 max = (u64)(~((u64)0));
4206
4207         max = div_u64(max, NSEC_PER_MSEC);
4208         if (ms >= max)
4209                 return -ERANGE;
4210
4211         ms *= NSEC_PER_MSEC;
4212         *result = nsecs_to_jiffies64(ms);
4213         return 0;
4214 }
4215
4216 __be64 nf_jiffies64_to_msecs(u64 input)
4217 {
4218         return cpu_to_be64(jiffies64_to_msecs(input));
4219 }
4220
4221 static int nf_tables_fill_set_concat(struct sk_buff *skb,
4222                                      const struct nft_set *set)
4223 {
4224         struct nlattr *concat, *field;
4225         int i;
4226
4227         concat = nla_nest_start_noflag(skb, NFTA_SET_DESC_CONCAT);
4228         if (!concat)
4229                 return -ENOMEM;
4230
4231         for (i = 0; i < set->field_count; i++) {
4232                 field = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
4233                 if (!field)
4234                         return -ENOMEM;
4235
4236                 if (nla_put_be32(skb, NFTA_SET_FIELD_LEN,
4237                                  htonl(set->field_len[i])))
4238                         return -ENOMEM;
4239
4240                 nla_nest_end(skb, field);
4241         }
4242
4243         nla_nest_end(skb, concat);
4244
4245         return 0;
4246 }
4247
4248 static int nf_tables_fill_set(struct sk_buff *skb, const struct nft_ctx *ctx,
4249                               const struct nft_set *set, u16 event, u16 flags)
4250 {
4251         u64 timeout = READ_ONCE(set->timeout);
4252         u32 gc_int = READ_ONCE(set->gc_int);
4253         u32 portid = ctx->portid;
4254         struct nlmsghdr *nlh;
4255         struct nlattr *nest;
4256         u32 seq = ctx->seq;
4257         int i;
4258
4259         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
4260         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
4261                            NFNETLINK_V0, nft_base_seq(ctx->net));
4262         if (!nlh)
4263                 goto nla_put_failure;
4264
4265         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
4266                 goto nla_put_failure;
4267         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
4268                 goto nla_put_failure;
4269         if (nla_put_be64(skb, NFTA_SET_HANDLE, cpu_to_be64(set->handle),
4270                          NFTA_SET_PAD))
4271                 goto nla_put_failure;
4272
4273         if (event == NFT_MSG_DELSET) {
4274                 nlmsg_end(skb, nlh);
4275                 return 0;
4276         }
4277
4278         if (set->flags != 0)
4279                 if (nla_put_be32(skb, NFTA_SET_FLAGS, htonl(set->flags)))
4280                         goto nla_put_failure;
4281
4282         if (nla_put_be32(skb, NFTA_SET_KEY_TYPE, htonl(set->ktype)))
4283                 goto nla_put_failure;
4284         if (nla_put_be32(skb, NFTA_SET_KEY_LEN, htonl(set->klen)))
4285                 goto nla_put_failure;
4286         if (set->flags & NFT_SET_MAP) {
4287                 if (nla_put_be32(skb, NFTA_SET_DATA_TYPE, htonl(set->dtype)))
4288                         goto nla_put_failure;
4289                 if (nla_put_be32(skb, NFTA_SET_DATA_LEN, htonl(set->dlen)))
4290                         goto nla_put_failure;
4291         }
4292         if (set->flags & NFT_SET_OBJECT &&
4293             nla_put_be32(skb, NFTA_SET_OBJ_TYPE, htonl(set->objtype)))
4294                 goto nla_put_failure;
4295
4296         if (timeout &&
4297             nla_put_be64(skb, NFTA_SET_TIMEOUT,
4298                          nf_jiffies64_to_msecs(timeout),
4299                          NFTA_SET_PAD))
4300                 goto nla_put_failure;
4301         if (gc_int &&
4302             nla_put_be32(skb, NFTA_SET_GC_INTERVAL, htonl(gc_int)))
4303                 goto nla_put_failure;
4304
4305         if (set->policy != NFT_SET_POL_PERFORMANCE) {
4306                 if (nla_put_be32(skb, NFTA_SET_POLICY, htonl(set->policy)))
4307                         goto nla_put_failure;
4308         }
4309
4310         if (set->udata &&
4311             nla_put(skb, NFTA_SET_USERDATA, set->udlen, set->udata))
4312                 goto nla_put_failure;
4313
4314         nest = nla_nest_start_noflag(skb, NFTA_SET_DESC);
4315         if (!nest)
4316                 goto nla_put_failure;
4317         if (set->size &&
4318             nla_put_be32(skb, NFTA_SET_DESC_SIZE, htonl(set->size)))
4319                 goto nla_put_failure;
4320
4321         if (set->field_count > 1 &&
4322             nf_tables_fill_set_concat(skb, set))
4323                 goto nla_put_failure;
4324
4325         nla_nest_end(skb, nest);
4326
4327         if (set->num_exprs == 1) {
4328                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPR);
4329                 if (nf_tables_fill_expr_info(skb, set->exprs[0], false) < 0)
4330                         goto nla_put_failure;
4331
4332                 nla_nest_end(skb, nest);
4333         } else if (set->num_exprs > 1) {
4334                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPRESSIONS);
4335                 if (nest == NULL)
4336                         goto nla_put_failure;
4337
4338                 for (i = 0; i < set->num_exprs; i++) {
4339                         if (nft_expr_dump(skb, NFTA_LIST_ELEM,
4340                                           set->exprs[i], false) < 0)
4341                                 goto nla_put_failure;
4342                 }
4343                 nla_nest_end(skb, nest);
4344         }
4345
4346         nlmsg_end(skb, nlh);
4347         return 0;
4348
4349 nla_put_failure:
4350         nlmsg_trim(skb, nlh);
4351         return -1;
4352 }
4353
4354 static void nf_tables_set_notify(const struct nft_ctx *ctx,
4355                                  const struct nft_set *set, int event,
4356                                  gfp_t gfp_flags)
4357 {
4358         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
4359         u32 portid = ctx->portid;
4360         struct sk_buff *skb;
4361         u16 flags = 0;
4362         int err;
4363
4364         if (!ctx->report &&
4365             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
4366                 return;
4367
4368         skb = nlmsg_new(NLMSG_GOODSIZE, gfp_flags);
4369         if (skb == NULL)
4370                 goto err;
4371
4372         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
4373                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
4374
4375         err = nf_tables_fill_set(skb, ctx, set, event, flags);
4376         if (err < 0) {
4377                 kfree_skb(skb);
4378                 goto err;
4379         }
4380
4381         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
4382         return;
4383 err:
4384         nfnetlink_set_err(ctx->net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
4385 }
4386
4387 static int nf_tables_dump_sets(struct sk_buff *skb, struct netlink_callback *cb)
4388 {
4389         const struct nft_set *set;
4390         unsigned int idx, s_idx = cb->args[0];
4391         struct nft_table *table, *cur_table = (struct nft_table *)cb->args[2];
4392         struct net *net = sock_net(skb->sk);
4393         struct nft_ctx *ctx = cb->data, ctx_set;
4394         struct nftables_pernet *nft_net;
4395
4396         if (cb->args[1])
4397                 return skb->len;
4398
4399         rcu_read_lock();
4400         nft_net = nft_pernet(net);
4401         cb->seq = READ_ONCE(nft_net->base_seq);
4402
4403         list_for_each_entry_rcu(table, &nft_net->tables, list) {
4404                 if (ctx->family != NFPROTO_UNSPEC &&
4405                     ctx->family != table->family)
4406                         continue;
4407
4408                 if (ctx->table && ctx->table != table)
4409                         continue;
4410
4411                 if (cur_table) {
4412                         if (cur_table != table)
4413                                 continue;
4414
4415                         cur_table = NULL;
4416                 }
4417                 idx = 0;
4418                 list_for_each_entry_rcu(set, &table->sets, list) {
4419                         if (idx < s_idx)
4420                                 goto cont;
4421                         if (!nft_is_active(net, set))
4422                                 goto cont;
4423
4424                         ctx_set = *ctx;
4425                         ctx_set.table = table;
4426                         ctx_set.family = table->family;
4427
4428                         if (nf_tables_fill_set(skb, &ctx_set, set,
4429                                                NFT_MSG_NEWSET,
4430                                                NLM_F_MULTI) < 0) {
4431                                 cb->args[0] = idx;
4432                                 cb->args[2] = (unsigned long) table;
4433                                 goto done;
4434                         }
4435                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
4436 cont:
4437                         idx++;
4438                 }
4439                 if (s_idx)
4440                         s_idx = 0;
4441         }
4442         cb->args[1] = 1;
4443 done:
4444         rcu_read_unlock();
4445         return skb->len;
4446 }
4447
4448 static int nf_tables_dump_sets_start(struct netlink_callback *cb)
4449 {
4450         struct nft_ctx *ctx_dump = NULL;
4451
4452         ctx_dump = kmemdup(cb->data, sizeof(*ctx_dump), GFP_ATOMIC);
4453         if (ctx_dump == NULL)
4454                 return -ENOMEM;
4455
4456         cb->data = ctx_dump;
4457         return 0;
4458 }
4459
4460 static int nf_tables_dump_sets_done(struct netlink_callback *cb)
4461 {
4462         kfree(cb->data);
4463         return 0;
4464 }
4465
4466 /* called with rcu_read_lock held */
4467 static int nf_tables_getset(struct sk_buff *skb, const struct nfnl_info *info,
4468                             const struct nlattr * const nla[])
4469 {
4470         struct netlink_ext_ack *extack = info->extack;
4471         u8 genmask = nft_genmask_cur(info->net);
4472         u8 family = info->nfmsg->nfgen_family;
4473         struct nft_table *table = NULL;
4474         struct net *net = info->net;
4475         const struct nft_set *set;
4476         struct sk_buff *skb2;
4477         struct nft_ctx ctx;
4478         int err;
4479
4480         if (nla[NFTA_SET_TABLE]) {
4481                 table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
4482                                          genmask, 0);
4483                 if (IS_ERR(table)) {
4484                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
4485                         return PTR_ERR(table);
4486                 }
4487         }
4488
4489         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
4490
4491         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
4492                 struct netlink_dump_control c = {
4493                         .start = nf_tables_dump_sets_start,
4494                         .dump = nf_tables_dump_sets,
4495                         .done = nf_tables_dump_sets_done,
4496                         .data = &ctx,
4497                         .module = THIS_MODULE,
4498                 };
4499
4500                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
4501         }
4502
4503         /* Only accept unspec with dump */
4504         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
4505                 return -EAFNOSUPPORT;
4506         if (!nla[NFTA_SET_TABLE])
4507                 return -EINVAL;
4508
4509         set = nft_set_lookup(table, nla[NFTA_SET_NAME], genmask);
4510         if (IS_ERR(set))
4511                 return PTR_ERR(set);
4512
4513         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
4514         if (skb2 == NULL)
4515                 return -ENOMEM;
4516
4517         err = nf_tables_fill_set(skb2, &ctx, set, NFT_MSG_NEWSET, 0);
4518         if (err < 0)
4519                 goto err_fill_set_info;
4520
4521         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
4522
4523 err_fill_set_info:
4524         kfree_skb(skb2);
4525         return err;
4526 }
4527
4528 static const struct nla_policy nft_concat_policy[NFTA_SET_FIELD_MAX + 1] = {
4529         [NFTA_SET_FIELD_LEN]    = { .type = NLA_U32 },
4530 };
4531
4532 static int nft_set_desc_concat_parse(const struct nlattr *attr,
4533                                      struct nft_set_desc *desc)
4534 {
4535         struct nlattr *tb[NFTA_SET_FIELD_MAX + 1];
4536         u32 len;
4537         int err;
4538
4539         if (desc->field_count >= ARRAY_SIZE(desc->field_len))
4540                 return -E2BIG;
4541
4542         err = nla_parse_nested_deprecated(tb, NFTA_SET_FIELD_MAX, attr,
4543                                           nft_concat_policy, NULL);
4544         if (err < 0)
4545                 return err;
4546
4547         if (!tb[NFTA_SET_FIELD_LEN])
4548                 return -EINVAL;
4549
4550         len = ntohl(nla_get_be32(tb[NFTA_SET_FIELD_LEN]));
4551         if (!len || len > U8_MAX)
4552                 return -EINVAL;
4553
4554         desc->field_len[desc->field_count++] = len;
4555
4556         return 0;
4557 }
4558
4559 static int nft_set_desc_concat(struct nft_set_desc *desc,
4560                                const struct nlattr *nla)
4561 {
4562         struct nlattr *attr;
4563         u32 num_regs = 0;
4564         int rem, err, i;
4565
4566         nla_for_each_nested(attr, nla, rem) {
4567                 if (nla_type(attr) != NFTA_LIST_ELEM)
4568                         return -EINVAL;
4569
4570                 err = nft_set_desc_concat_parse(attr, desc);
4571                 if (err < 0)
4572                         return err;
4573         }
4574
4575         for (i = 0; i < desc->field_count; i++)
4576                 num_regs += DIV_ROUND_UP(desc->field_len[i], sizeof(u32));
4577
4578         if (num_regs > NFT_REG32_COUNT)
4579                 return -E2BIG;
4580
4581         return 0;
4582 }
4583
4584 static int nf_tables_set_desc_parse(struct nft_set_desc *desc,
4585                                     const struct nlattr *nla)
4586 {
4587         struct nlattr *da[NFTA_SET_DESC_MAX + 1];
4588         int err;
4589
4590         err = nla_parse_nested_deprecated(da, NFTA_SET_DESC_MAX, nla,
4591                                           nft_set_desc_policy, NULL);
4592         if (err < 0)
4593                 return err;
4594
4595         if (da[NFTA_SET_DESC_SIZE] != NULL)
4596                 desc->size = ntohl(nla_get_be32(da[NFTA_SET_DESC_SIZE]));
4597         if (da[NFTA_SET_DESC_CONCAT])
4598                 err = nft_set_desc_concat(desc, da[NFTA_SET_DESC_CONCAT]);
4599
4600         return err;
4601 }
4602
4603 static int nft_set_expr_alloc(struct nft_ctx *ctx, struct nft_set *set,
4604                               const struct nlattr * const *nla,
4605                               struct nft_expr **exprs, int *num_exprs,
4606                               u32 flags)
4607 {
4608         struct nft_expr *expr;
4609         int err, i;
4610
4611         if (nla[NFTA_SET_EXPR]) {
4612                 expr = nft_set_elem_expr_alloc(ctx, set, nla[NFTA_SET_EXPR]);
4613                 if (IS_ERR(expr)) {
4614                         err = PTR_ERR(expr);
4615                         goto err_set_expr_alloc;
4616                 }
4617                 exprs[0] = expr;
4618                 (*num_exprs)++;
4619         } else if (nla[NFTA_SET_EXPRESSIONS]) {
4620                 struct nlattr *tmp;
4621                 int left;
4622
4623                 if (!(flags & NFT_SET_EXPR)) {
4624                         err = -EINVAL;
4625                         goto err_set_expr_alloc;
4626                 }
4627                 i = 0;
4628                 nla_for_each_nested(tmp, nla[NFTA_SET_EXPRESSIONS], left) {
4629                         if (i == NFT_SET_EXPR_MAX) {
4630                                 err = -E2BIG;
4631                                 goto err_set_expr_alloc;
4632                         }
4633                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
4634                                 err = -EINVAL;
4635                                 goto err_set_expr_alloc;
4636                         }
4637                         expr = nft_set_elem_expr_alloc(ctx, set, tmp);
4638                         if (IS_ERR(expr)) {
4639                                 err = PTR_ERR(expr);
4640                                 goto err_set_expr_alloc;
4641                         }
4642                         exprs[i++] = expr;
4643                         (*num_exprs)++;
4644                 }
4645         }
4646
4647         return 0;
4648
4649 err_set_expr_alloc:
4650         for (i = 0; i < *num_exprs; i++)
4651                 nft_expr_destroy(ctx, exprs[i]);
4652
4653         return err;
4654 }
4655
4656 static bool nft_set_is_same(const struct nft_set *set,
4657                             const struct nft_set_desc *desc,
4658                             struct nft_expr *exprs[], u32 num_exprs, u32 flags)
4659 {
4660         int i;
4661
4662         if (set->ktype != desc->ktype ||
4663             set->dtype != desc->dtype ||
4664             set->flags != flags ||
4665             set->klen != desc->klen ||
4666             set->dlen != desc->dlen ||
4667             set->field_count != desc->field_count ||
4668             set->num_exprs != num_exprs)
4669                 return false;
4670
4671         for (i = 0; i < desc->field_count; i++) {
4672                 if (set->field_len[i] != desc->field_len[i])
4673                         return false;
4674         }
4675
4676         for (i = 0; i < num_exprs; i++) {
4677                 if (set->exprs[i]->ops != exprs[i]->ops)
4678                         return false;
4679         }
4680
4681         return true;
4682 }
4683
4684 static int nf_tables_newset(struct sk_buff *skb, const struct nfnl_info *info,
4685                             const struct nlattr * const nla[])
4686 {
4687         struct netlink_ext_ack *extack = info->extack;
4688         u8 genmask = nft_genmask_next(info->net);
4689         u8 family = info->nfmsg->nfgen_family;
4690         const struct nft_set_ops *ops;
4691         struct net *net = info->net;
4692         struct nft_set_desc desc;
4693         struct nft_table *table;
4694         unsigned char *udata;
4695         struct nft_set *set;
4696         struct nft_ctx ctx;
4697         size_t alloc_size;
4698         int num_exprs = 0;
4699         char *name;
4700         int err, i;
4701         u16 udlen;
4702         u32 flags;
4703         u64 size;
4704
4705         if (nla[NFTA_SET_TABLE] == NULL ||
4706             nla[NFTA_SET_NAME] == NULL ||
4707             nla[NFTA_SET_KEY_LEN] == NULL ||
4708             nla[NFTA_SET_ID] == NULL)
4709                 return -EINVAL;
4710
4711         memset(&desc, 0, sizeof(desc));
4712
4713         desc.ktype = NFT_DATA_VALUE;
4714         if (nla[NFTA_SET_KEY_TYPE] != NULL) {
4715                 desc.ktype = ntohl(nla_get_be32(nla[NFTA_SET_KEY_TYPE]));
4716                 if ((desc.ktype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK)
4717                         return -EINVAL;
4718         }
4719
4720         desc.klen = ntohl(nla_get_be32(nla[NFTA_SET_KEY_LEN]));
4721         if (desc.klen == 0 || desc.klen > NFT_DATA_VALUE_MAXLEN)
4722                 return -EINVAL;
4723
4724         flags = 0;
4725         if (nla[NFTA_SET_FLAGS] != NULL) {
4726                 flags = ntohl(nla_get_be32(nla[NFTA_SET_FLAGS]));
4727                 if (flags & ~(NFT_SET_ANONYMOUS | NFT_SET_CONSTANT |
4728                               NFT_SET_INTERVAL | NFT_SET_TIMEOUT |
4729                               NFT_SET_MAP | NFT_SET_EVAL |
4730                               NFT_SET_OBJECT | NFT_SET_CONCAT | NFT_SET_EXPR))
4731                         return -EOPNOTSUPP;
4732                 /* Only one of these operations is supported */
4733                 if ((flags & (NFT_SET_MAP | NFT_SET_OBJECT)) ==
4734                              (NFT_SET_MAP | NFT_SET_OBJECT))
4735                         return -EOPNOTSUPP;
4736                 if ((flags & (NFT_SET_EVAL | NFT_SET_OBJECT)) ==
4737                              (NFT_SET_EVAL | NFT_SET_OBJECT))
4738                         return -EOPNOTSUPP;
4739         }
4740
4741         desc.dtype = 0;
4742         if (nla[NFTA_SET_DATA_TYPE] != NULL) {
4743                 if (!(flags & NFT_SET_MAP))
4744                         return -EINVAL;
4745
4746                 desc.dtype = ntohl(nla_get_be32(nla[NFTA_SET_DATA_TYPE]));
4747                 if ((desc.dtype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK &&
4748                     desc.dtype != NFT_DATA_VERDICT)
4749                         return -EINVAL;
4750
4751                 if (desc.dtype != NFT_DATA_VERDICT) {
4752                         if (nla[NFTA_SET_DATA_LEN] == NULL)
4753                                 return -EINVAL;
4754                         desc.dlen = ntohl(nla_get_be32(nla[NFTA_SET_DATA_LEN]));
4755                         if (desc.dlen == 0 || desc.dlen > NFT_DATA_VALUE_MAXLEN)
4756                                 return -EINVAL;
4757                 } else
4758                         desc.dlen = sizeof(struct nft_verdict);
4759         } else if (flags & NFT_SET_MAP)
4760                 return -EINVAL;
4761
4762         if (nla[NFTA_SET_OBJ_TYPE] != NULL) {
4763                 if (!(flags & NFT_SET_OBJECT))
4764                         return -EINVAL;
4765
4766                 desc.objtype = ntohl(nla_get_be32(nla[NFTA_SET_OBJ_TYPE]));
4767                 if (desc.objtype == NFT_OBJECT_UNSPEC ||
4768                     desc.objtype > NFT_OBJECT_MAX)
4769                         return -EOPNOTSUPP;
4770         } else if (flags & NFT_SET_OBJECT)
4771                 return -EINVAL;
4772         else
4773                 desc.objtype = NFT_OBJECT_UNSPEC;
4774
4775         desc.timeout = 0;
4776         if (nla[NFTA_SET_TIMEOUT] != NULL) {
4777                 if (!(flags & NFT_SET_TIMEOUT))
4778                         return -EINVAL;
4779
4780                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_TIMEOUT], &desc.timeout);
4781                 if (err)
4782                         return err;
4783         }
4784         desc.gc_int = 0;
4785         if (nla[NFTA_SET_GC_INTERVAL] != NULL) {
4786                 if (!(flags & NFT_SET_TIMEOUT))
4787                         return -EINVAL;
4788                 desc.gc_int = ntohl(nla_get_be32(nla[NFTA_SET_GC_INTERVAL]));
4789         }
4790
4791         desc.policy = NFT_SET_POL_PERFORMANCE;
4792         if (nla[NFTA_SET_POLICY] != NULL)
4793                 desc.policy = ntohl(nla_get_be32(nla[NFTA_SET_POLICY]));
4794
4795         if (nla[NFTA_SET_DESC] != NULL) {
4796                 err = nf_tables_set_desc_parse(&desc, nla[NFTA_SET_DESC]);
4797                 if (err < 0)
4798                         return err;
4799
4800                 if (desc.field_count > 1 && !(flags & NFT_SET_CONCAT))
4801                         return -EINVAL;
4802         } else if (flags & NFT_SET_CONCAT) {
4803                 return -EINVAL;
4804         }
4805
4806         if (nla[NFTA_SET_EXPR] || nla[NFTA_SET_EXPRESSIONS])
4807                 desc.expr = true;
4808
4809         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family, genmask,
4810                                  NETLINK_CB(skb).portid);
4811         if (IS_ERR(table)) {
4812                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
4813                 return PTR_ERR(table);
4814         }
4815
4816         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
4817
4818         set = nft_set_lookup(table, nla[NFTA_SET_NAME], genmask);
4819         if (IS_ERR(set)) {
4820                 if (PTR_ERR(set) != -ENOENT) {
4821                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
4822                         return PTR_ERR(set);
4823                 }
4824         } else {
4825                 struct nft_expr *exprs[NFT_SET_EXPR_MAX] = {};
4826
4827                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
4828                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
4829                         return -EEXIST;
4830                 }
4831                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
4832                         return -EOPNOTSUPP;
4833
4834                 err = nft_set_expr_alloc(&ctx, set, nla, exprs, &num_exprs, flags);
4835                 if (err < 0)
4836                         return err;
4837
4838                 err = 0;
4839                 if (!nft_set_is_same(set, &desc, exprs, num_exprs, flags)) {
4840                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
4841                         err = -EEXIST;
4842                 }
4843
4844                 for (i = 0; i < num_exprs; i++)
4845                         nft_expr_destroy(&ctx, exprs[i]);
4846
4847                 if (err < 0)
4848                         return err;
4849
4850                 return __nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set, &desc);
4851         }
4852
4853         if (!(info->nlh->nlmsg_flags & NLM_F_CREATE))
4854                 return -ENOENT;
4855
4856         ops = nft_select_set_ops(&ctx, nla, &desc);
4857         if (IS_ERR(ops))
4858                 return PTR_ERR(ops);
4859
4860         udlen = 0;
4861         if (nla[NFTA_SET_USERDATA])
4862                 udlen = nla_len(nla[NFTA_SET_USERDATA]);
4863
4864         size = 0;
4865         if (ops->privsize != NULL)
4866                 size = ops->privsize(nla, &desc);
4867         alloc_size = sizeof(*set) + size + udlen;
4868         if (alloc_size < size || alloc_size > INT_MAX)
4869                 return -ENOMEM;
4870         set = kvzalloc(alloc_size, GFP_KERNEL_ACCOUNT);
4871         if (!set)
4872                 return -ENOMEM;
4873
4874         name = nla_strdup(nla[NFTA_SET_NAME], GFP_KERNEL_ACCOUNT);
4875         if (!name) {
4876                 err = -ENOMEM;
4877                 goto err_set_name;
4878         }
4879
4880         err = nf_tables_set_alloc_name(&ctx, set, name);
4881         kfree(name);
4882         if (err < 0)
4883                 goto err_set_name;
4884
4885         udata = NULL;
4886         if (udlen) {
4887                 udata = set->data + size;
4888                 nla_memcpy(udata, nla[NFTA_SET_USERDATA], udlen);
4889         }
4890
4891         INIT_LIST_HEAD(&set->bindings);
4892         INIT_LIST_HEAD(&set->catchall_list);
4893         set->table = table;
4894         write_pnet(&set->net, net);
4895         set->ops = ops;
4896         set->ktype = desc.ktype;
4897         set->klen = desc.klen;
4898         set->dtype = desc.dtype;
4899         set->objtype = desc.objtype;
4900         set->dlen = desc.dlen;
4901         set->flags = flags;
4902         set->size = desc.size;
4903         set->policy = desc.policy;
4904         set->udlen = udlen;
4905         set->udata = udata;
4906         set->timeout = desc.timeout;
4907         set->gc_int = desc.gc_int;
4908
4909         set->field_count = desc.field_count;
4910         for (i = 0; i < desc.field_count; i++)
4911                 set->field_len[i] = desc.field_len[i];
4912
4913         err = ops->init(set, &desc, nla);
4914         if (err < 0)
4915                 goto err_set_init;
4916
4917         err = nft_set_expr_alloc(&ctx, set, nla, set->exprs, &num_exprs, flags);
4918         if (err < 0)
4919                 goto err_set_destroy;
4920
4921         set->num_exprs = num_exprs;
4922         set->handle = nf_tables_alloc_handle(table);
4923         INIT_LIST_HEAD(&set->pending_update);
4924
4925         err = nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set);
4926         if (err < 0)
4927                 goto err_set_expr_alloc;
4928
4929         list_add_tail_rcu(&set->list, &table->sets);
4930         table->use++;
4931         return 0;
4932
4933 err_set_expr_alloc:
4934         for (i = 0; i < set->num_exprs; i++)
4935                 nft_expr_destroy(&ctx, set->exprs[i]);
4936 err_set_destroy:
4937         ops->destroy(set);
4938 err_set_init:
4939         kfree(set->name);
4940 err_set_name:
4941         kvfree(set);
4942         return err;
4943 }
4944
4945 static void nft_set_catchall_destroy(const struct nft_ctx *ctx,
4946                                      struct nft_set *set)
4947 {
4948         struct nft_set_elem_catchall *next, *catchall;
4949
4950         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
4951                 list_del_rcu(&catchall->list);
4952                 nft_set_elem_destroy(set, catchall->elem, true);
4953                 kfree_rcu(catchall, rcu);
4954         }
4955 }
4956
4957 static void nft_set_destroy(const struct nft_ctx *ctx, struct nft_set *set)
4958 {
4959         int i;
4960
4961         if (WARN_ON(set->use > 0))
4962                 return;
4963
4964         for (i = 0; i < set->num_exprs; i++)
4965                 nft_expr_destroy(ctx, set->exprs[i]);
4966
4967         set->ops->destroy(set);
4968         nft_set_catchall_destroy(ctx, set);
4969         kfree(set->name);
4970         kvfree(set);
4971 }
4972
4973 static int nf_tables_delset(struct sk_buff *skb, const struct nfnl_info *info,
4974                             const struct nlattr * const nla[])
4975 {
4976         struct netlink_ext_ack *extack = info->extack;
4977         u8 genmask = nft_genmask_next(info->net);
4978         u8 family = info->nfmsg->nfgen_family;
4979         struct net *net = info->net;
4980         const struct nlattr *attr;
4981         struct nft_table *table;
4982         struct nft_set *set;
4983         struct nft_ctx ctx;
4984
4985         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
4986                 return -EAFNOSUPPORT;
4987
4988         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
4989                                  genmask, NETLINK_CB(skb).portid);
4990         if (IS_ERR(table)) {
4991                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
4992                 return PTR_ERR(table);
4993         }
4994
4995         if (nla[NFTA_SET_HANDLE]) {
4996                 attr = nla[NFTA_SET_HANDLE];
4997                 set = nft_set_lookup_byhandle(table, attr, genmask);
4998         } else {
4999                 attr = nla[NFTA_SET_NAME];
5000                 set = nft_set_lookup(table, attr, genmask);
5001         }
5002
5003         if (IS_ERR(set)) {
5004                 if (PTR_ERR(set) == -ENOENT &&
5005                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYSET)
5006                         return 0;
5007
5008                 NL_SET_BAD_ATTR(extack, attr);
5009                 return PTR_ERR(set);
5010         }
5011         if (set->use ||
5012             (info->nlh->nlmsg_flags & NLM_F_NONREC &&
5013              atomic_read(&set->nelems) > 0)) {
5014                 NL_SET_BAD_ATTR(extack, attr);
5015                 return -EBUSY;
5016         }
5017
5018         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
5019
5020         return nft_delset(&ctx, set);
5021 }
5022
5023 static int nft_validate_register_store(const struct nft_ctx *ctx,
5024                                        enum nft_registers reg,
5025                                        const struct nft_data *data,
5026                                        enum nft_data_types type,
5027                                        unsigned int len);
5028
5029 static int nft_setelem_data_validate(const struct nft_ctx *ctx,
5030                                      struct nft_set *set,
5031                                      struct nft_set_elem *elem)
5032 {
5033         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
5034         enum nft_registers dreg;
5035
5036         dreg = nft_type_to_reg(set->dtype);
5037         return nft_validate_register_store(ctx, dreg, nft_set_ext_data(ext),
5038                                            set->dtype == NFT_DATA_VERDICT ?
5039                                            NFT_DATA_VERDICT : NFT_DATA_VALUE,
5040                                            set->dlen);
5041 }
5042
5043 static int nf_tables_bind_check_setelem(const struct nft_ctx *ctx,
5044                                         struct nft_set *set,
5045                                         const struct nft_set_iter *iter,
5046                                         struct nft_set_elem *elem)
5047 {
5048         return nft_setelem_data_validate(ctx, set, elem);
5049 }
5050
5051 static int nft_set_catchall_bind_check(const struct nft_ctx *ctx,
5052                                        struct nft_set *set)
5053 {
5054         u8 genmask = nft_genmask_next(ctx->net);
5055         struct nft_set_elem_catchall *catchall;
5056         struct nft_set_elem elem;
5057         struct nft_set_ext *ext;
5058         int ret = 0;
5059
5060         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5061                 ext = nft_set_elem_ext(set, catchall->elem);
5062                 if (!nft_set_elem_active(ext, genmask))
5063                         continue;
5064
5065                 elem.priv = catchall->elem;
5066                 ret = nft_setelem_data_validate(ctx, set, &elem);
5067                 if (ret < 0)
5068                         break;
5069         }
5070
5071         return ret;
5072 }
5073
5074 int nf_tables_bind_set(const struct nft_ctx *ctx, struct nft_set *set,
5075                        struct nft_set_binding *binding)
5076 {
5077         struct nft_set_binding *i;
5078         struct nft_set_iter iter;
5079
5080         if (set->use == UINT_MAX)
5081                 return -EOVERFLOW;
5082
5083         if (!list_empty(&set->bindings) && nft_set_is_anonymous(set))
5084                 return -EBUSY;
5085
5086         if (binding->flags & NFT_SET_MAP) {
5087                 /* If the set is already bound to the same chain all
5088                  * jumps are already validated for that chain.
5089                  */
5090                 list_for_each_entry(i, &set->bindings, list) {
5091                         if (i->flags & NFT_SET_MAP &&
5092                             i->chain == binding->chain)
5093                                 goto bind;
5094                 }
5095
5096                 iter.genmask    = nft_genmask_next(ctx->net);
5097                 iter.skip       = 0;
5098                 iter.count      = 0;
5099                 iter.err        = 0;
5100                 iter.fn         = nf_tables_bind_check_setelem;
5101
5102                 set->ops->walk(ctx, set, &iter);
5103                 if (!iter.err)
5104                         iter.err = nft_set_catchall_bind_check(ctx, set);
5105
5106                 if (iter.err < 0)
5107                         return iter.err;
5108         }
5109 bind:
5110         binding->chain = ctx->chain;
5111         list_add_tail_rcu(&binding->list, &set->bindings);
5112         nft_set_trans_bind(ctx, set);
5113         set->use++;
5114
5115         return 0;
5116 }
5117 EXPORT_SYMBOL_GPL(nf_tables_bind_set);
5118
5119 static void nf_tables_unbind_set(const struct nft_ctx *ctx, struct nft_set *set,
5120                                  struct nft_set_binding *binding, bool event)
5121 {
5122         list_del_rcu(&binding->list);
5123
5124         if (list_empty(&set->bindings) && nft_set_is_anonymous(set)) {
5125                 list_del_rcu(&set->list);
5126                 if (event)
5127                         nf_tables_set_notify(ctx, set, NFT_MSG_DELSET,
5128                                              GFP_KERNEL);
5129         }
5130 }
5131
5132 void nf_tables_activate_set(const struct nft_ctx *ctx, struct nft_set *set)
5133 {
5134         if (nft_set_is_anonymous(set))
5135                 nft_clear(ctx->net, set);
5136
5137         set->use++;
5138 }
5139 EXPORT_SYMBOL_GPL(nf_tables_activate_set);
5140
5141 void nf_tables_deactivate_set(const struct nft_ctx *ctx, struct nft_set *set,
5142                               struct nft_set_binding *binding,
5143                               enum nft_trans_phase phase)
5144 {
5145         switch (phase) {
5146         case NFT_TRANS_PREPARE:
5147                 if (nft_set_is_anonymous(set))
5148                         nft_deactivate_next(ctx->net, set);
5149
5150                 set->use--;
5151                 return;
5152         case NFT_TRANS_ABORT:
5153         case NFT_TRANS_RELEASE:
5154                 set->use--;
5155                 fallthrough;
5156         default:
5157                 nf_tables_unbind_set(ctx, set, binding,
5158                                      phase == NFT_TRANS_COMMIT);
5159         }
5160 }
5161 EXPORT_SYMBOL_GPL(nf_tables_deactivate_set);
5162
5163 void nf_tables_destroy_set(const struct nft_ctx *ctx, struct nft_set *set)
5164 {
5165         if (list_empty(&set->bindings) && nft_set_is_anonymous(set))
5166                 nft_set_destroy(ctx, set);
5167 }
5168 EXPORT_SYMBOL_GPL(nf_tables_destroy_set);
5169
5170 const struct nft_set_ext_type nft_set_ext_types[] = {
5171         [NFT_SET_EXT_KEY]               = {
5172                 .align  = __alignof__(u32),
5173         },
5174         [NFT_SET_EXT_DATA]              = {
5175                 .align  = __alignof__(u32),
5176         },
5177         [NFT_SET_EXT_EXPRESSIONS]       = {
5178                 .align  = __alignof__(struct nft_set_elem_expr),
5179         },
5180         [NFT_SET_EXT_OBJREF]            = {
5181                 .len    = sizeof(struct nft_object *),
5182                 .align  = __alignof__(struct nft_object *),
5183         },
5184         [NFT_SET_EXT_FLAGS]             = {
5185                 .len    = sizeof(u8),
5186                 .align  = __alignof__(u8),
5187         },
5188         [NFT_SET_EXT_TIMEOUT]           = {
5189                 .len    = sizeof(u64),
5190                 .align  = __alignof__(u64),
5191         },
5192         [NFT_SET_EXT_EXPIRATION]        = {
5193                 .len    = sizeof(u64),
5194                 .align  = __alignof__(u64),
5195         },
5196         [NFT_SET_EXT_USERDATA]          = {
5197                 .len    = sizeof(struct nft_userdata),
5198                 .align  = __alignof__(struct nft_userdata),
5199         },
5200         [NFT_SET_EXT_KEY_END]           = {
5201                 .align  = __alignof__(u32),
5202         },
5203 };
5204
5205 /*
5206  * Set elements
5207  */
5208
5209 static const struct nla_policy nft_set_elem_policy[NFTA_SET_ELEM_MAX + 1] = {
5210         [NFTA_SET_ELEM_KEY]             = { .type = NLA_NESTED },
5211         [NFTA_SET_ELEM_DATA]            = { .type = NLA_NESTED },
5212         [NFTA_SET_ELEM_FLAGS]           = { .type = NLA_U32 },
5213         [NFTA_SET_ELEM_TIMEOUT]         = { .type = NLA_U64 },
5214         [NFTA_SET_ELEM_EXPIRATION]      = { .type = NLA_U64 },
5215         [NFTA_SET_ELEM_USERDATA]        = { .type = NLA_BINARY,
5216                                             .len = NFT_USERDATA_MAXLEN },
5217         [NFTA_SET_ELEM_EXPR]            = { .type = NLA_NESTED },
5218         [NFTA_SET_ELEM_OBJREF]          = { .type = NLA_STRING,
5219                                             .len = NFT_OBJ_MAXNAMELEN - 1 },
5220         [NFTA_SET_ELEM_KEY_END]         = { .type = NLA_NESTED },
5221         [NFTA_SET_ELEM_EXPRESSIONS]     = { .type = NLA_NESTED },
5222 };
5223
5224 static const struct nla_policy nft_set_elem_list_policy[NFTA_SET_ELEM_LIST_MAX + 1] = {
5225         [NFTA_SET_ELEM_LIST_TABLE]      = { .type = NLA_STRING,
5226                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
5227         [NFTA_SET_ELEM_LIST_SET]        = { .type = NLA_STRING,
5228                                             .len = NFT_SET_MAXNAMELEN - 1 },
5229         [NFTA_SET_ELEM_LIST_ELEMENTS]   = { .type = NLA_NESTED },
5230         [NFTA_SET_ELEM_LIST_SET_ID]     = { .type = NLA_U32 },
5231 };
5232
5233 static int nft_set_elem_expr_dump(struct sk_buff *skb,
5234                                   const struct nft_set *set,
5235                                   const struct nft_set_ext *ext)
5236 {
5237         struct nft_set_elem_expr *elem_expr;
5238         u32 size, num_exprs = 0;
5239         struct nft_expr *expr;
5240         struct nlattr *nest;
5241
5242         elem_expr = nft_set_ext_expr(ext);
5243         nft_setelem_expr_foreach(expr, elem_expr, size)
5244                 num_exprs++;
5245
5246         if (num_exprs == 1) {
5247                 expr = nft_setelem_expr_at(elem_expr, 0);
5248                 if (nft_expr_dump(skb, NFTA_SET_ELEM_EXPR, expr, false) < 0)
5249                         return -1;
5250
5251                 return 0;
5252         } else if (num_exprs > 1) {
5253                 nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_EXPRESSIONS);
5254                 if (nest == NULL)
5255                         goto nla_put_failure;
5256
5257                 nft_setelem_expr_foreach(expr, elem_expr, size) {
5258                         expr = nft_setelem_expr_at(elem_expr, size);
5259                         if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr, false) < 0)
5260                                 goto nla_put_failure;
5261                 }
5262                 nla_nest_end(skb, nest);
5263         }
5264         return 0;
5265
5266 nla_put_failure:
5267         return -1;
5268 }
5269
5270 static int nf_tables_fill_setelem(struct sk_buff *skb,
5271                                   const struct nft_set *set,
5272                                   const struct nft_set_elem *elem)
5273 {
5274         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
5275         unsigned char *b = skb_tail_pointer(skb);
5276         struct nlattr *nest;
5277
5278         nest = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
5279         if (nest == NULL)
5280                 goto nla_put_failure;
5281
5282         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
5283             nft_data_dump(skb, NFTA_SET_ELEM_KEY, nft_set_ext_key(ext),
5284                           NFT_DATA_VALUE, set->klen) < 0)
5285                 goto nla_put_failure;
5286
5287         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
5288             nft_data_dump(skb, NFTA_SET_ELEM_KEY_END, nft_set_ext_key_end(ext),
5289                           NFT_DATA_VALUE, set->klen) < 0)
5290                 goto nla_put_failure;
5291
5292         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
5293             nft_data_dump(skb, NFTA_SET_ELEM_DATA, nft_set_ext_data(ext),
5294                           set->dtype == NFT_DATA_VERDICT ? NFT_DATA_VERDICT : NFT_DATA_VALUE,
5295                           set->dlen) < 0)
5296                 goto nla_put_failure;
5297
5298         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS) &&
5299             nft_set_elem_expr_dump(skb, set, ext))
5300                 goto nla_put_failure;
5301
5302         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
5303             nla_put_string(skb, NFTA_SET_ELEM_OBJREF,
5304                            (*nft_set_ext_obj(ext))->key.name) < 0)
5305                 goto nla_put_failure;
5306
5307         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
5308             nla_put_be32(skb, NFTA_SET_ELEM_FLAGS,
5309                          htonl(*nft_set_ext_flags(ext))))
5310                 goto nla_put_failure;
5311
5312         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT) &&
5313             nla_put_be64(skb, NFTA_SET_ELEM_TIMEOUT,
5314                          nf_jiffies64_to_msecs(*nft_set_ext_timeout(ext)),
5315                          NFTA_SET_ELEM_PAD))
5316                 goto nla_put_failure;
5317
5318         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
5319                 u64 expires, now = get_jiffies_64();
5320
5321                 expires = *nft_set_ext_expiration(ext);
5322                 if (time_before64(now, expires))
5323                         expires -= now;
5324                 else
5325                         expires = 0;
5326
5327                 if (nla_put_be64(skb, NFTA_SET_ELEM_EXPIRATION,
5328                                  nf_jiffies64_to_msecs(expires),
5329                                  NFTA_SET_ELEM_PAD))
5330                         goto nla_put_failure;
5331         }
5332
5333         if (nft_set_ext_exists(ext, NFT_SET_EXT_USERDATA)) {
5334                 struct nft_userdata *udata;
5335
5336                 udata = nft_set_ext_userdata(ext);
5337                 if (nla_put(skb, NFTA_SET_ELEM_USERDATA,
5338                             udata->len + 1, udata->data))
5339                         goto nla_put_failure;
5340         }
5341
5342         nla_nest_end(skb, nest);
5343         return 0;
5344
5345 nla_put_failure:
5346         nlmsg_trim(skb, b);
5347         return -EMSGSIZE;
5348 }
5349
5350 struct nft_set_dump_args {
5351         const struct netlink_callback   *cb;
5352         struct nft_set_iter             iter;
5353         struct sk_buff                  *skb;
5354 };
5355
5356 static int nf_tables_dump_setelem(const struct nft_ctx *ctx,
5357                                   struct nft_set *set,
5358                                   const struct nft_set_iter *iter,
5359                                   struct nft_set_elem *elem)
5360 {
5361         struct nft_set_dump_args *args;
5362
5363         args = container_of(iter, struct nft_set_dump_args, iter);
5364         return nf_tables_fill_setelem(args->skb, set, elem);
5365 }
5366
5367 struct nft_set_dump_ctx {
5368         const struct nft_set    *set;
5369         struct nft_ctx          ctx;
5370 };
5371
5372 static int nft_set_catchall_dump(struct net *net, struct sk_buff *skb,
5373                                  const struct nft_set *set)
5374 {
5375         struct nft_set_elem_catchall *catchall;
5376         u8 genmask = nft_genmask_cur(net);
5377         struct nft_set_elem elem;
5378         struct nft_set_ext *ext;
5379         int ret = 0;
5380
5381         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5382                 ext = nft_set_elem_ext(set, catchall->elem);
5383                 if (!nft_set_elem_active(ext, genmask) ||
5384                     nft_set_elem_expired(ext))
5385                         continue;
5386
5387                 elem.priv = catchall->elem;
5388                 ret = nf_tables_fill_setelem(skb, set, &elem);
5389                 break;
5390         }
5391
5392         return ret;
5393 }
5394
5395 static int nf_tables_dump_set(struct sk_buff *skb, struct netlink_callback *cb)
5396 {
5397         struct nft_set_dump_ctx *dump_ctx = cb->data;
5398         struct net *net = sock_net(skb->sk);
5399         struct nftables_pernet *nft_net;
5400         struct nft_table *table;
5401         struct nft_set *set;
5402         struct nft_set_dump_args args;
5403         bool set_found = false;
5404         struct nlmsghdr *nlh;
5405         struct nlattr *nest;
5406         u32 portid, seq;
5407         int event;
5408
5409         rcu_read_lock();
5410         nft_net = nft_pernet(net);
5411         cb->seq = READ_ONCE(nft_net->base_seq);
5412
5413         list_for_each_entry_rcu(table, &nft_net->tables, list) {
5414                 if (dump_ctx->ctx.family != NFPROTO_UNSPEC &&
5415                     dump_ctx->ctx.family != table->family)
5416                         continue;
5417
5418                 if (table != dump_ctx->ctx.table)
5419                         continue;
5420
5421                 list_for_each_entry_rcu(set, &table->sets, list) {
5422                         if (set == dump_ctx->set) {
5423                                 set_found = true;
5424                                 break;
5425                         }
5426                 }
5427                 break;
5428         }
5429
5430         if (!set_found) {
5431                 rcu_read_unlock();
5432                 return -ENOENT;
5433         }
5434
5435         event  = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWSETELEM);
5436         portid = NETLINK_CB(cb->skb).portid;
5437         seq    = cb->nlh->nlmsg_seq;
5438
5439         nlh = nfnl_msg_put(skb, portid, seq, event, NLM_F_MULTI,
5440                            table->family, NFNETLINK_V0, nft_base_seq(net));
5441         if (!nlh)
5442                 goto nla_put_failure;
5443
5444         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_TABLE, table->name))
5445                 goto nla_put_failure;
5446         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_SET, set->name))
5447                 goto nla_put_failure;
5448
5449         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
5450         if (nest == NULL)
5451                 goto nla_put_failure;
5452
5453         args.cb                 = cb;
5454         args.skb                = skb;
5455         args.iter.genmask       = nft_genmask_cur(net);
5456         args.iter.skip          = cb->args[0];
5457         args.iter.count         = 0;
5458         args.iter.err           = 0;
5459         args.iter.fn            = nf_tables_dump_setelem;
5460         set->ops->walk(&dump_ctx->ctx, set, &args.iter);
5461
5462         if (!args.iter.err && args.iter.count == cb->args[0])
5463                 args.iter.err = nft_set_catchall_dump(net, skb, set);
5464         rcu_read_unlock();
5465
5466         nla_nest_end(skb, nest);
5467         nlmsg_end(skb, nlh);
5468
5469         if (args.iter.err && args.iter.err != -EMSGSIZE)
5470                 return args.iter.err;
5471         if (args.iter.count == cb->args[0])
5472                 return 0;
5473
5474         cb->args[0] = args.iter.count;
5475         return skb->len;
5476
5477 nla_put_failure:
5478         rcu_read_unlock();
5479         return -ENOSPC;
5480 }
5481
5482 static int nf_tables_dump_set_start(struct netlink_callback *cb)
5483 {
5484         struct nft_set_dump_ctx *dump_ctx = cb->data;
5485
5486         cb->data = kmemdup(dump_ctx, sizeof(*dump_ctx), GFP_ATOMIC);
5487
5488         return cb->data ? 0 : -ENOMEM;
5489 }
5490
5491 static int nf_tables_dump_set_done(struct netlink_callback *cb)
5492 {
5493         kfree(cb->data);
5494         return 0;
5495 }
5496
5497 static int nf_tables_fill_setelem_info(struct sk_buff *skb,
5498                                        const struct nft_ctx *ctx, u32 seq,
5499                                        u32 portid, int event, u16 flags,
5500                                        const struct nft_set *set,
5501                                        const struct nft_set_elem *elem)
5502 {
5503         struct nlmsghdr *nlh;
5504         struct nlattr *nest;
5505         int err;
5506
5507         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
5508         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
5509                            NFNETLINK_V0, nft_base_seq(ctx->net));
5510         if (!nlh)
5511                 goto nla_put_failure;
5512
5513         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
5514                 goto nla_put_failure;
5515         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
5516                 goto nla_put_failure;
5517
5518         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
5519         if (nest == NULL)
5520                 goto nla_put_failure;
5521
5522         err = nf_tables_fill_setelem(skb, set, elem);
5523         if (err < 0)
5524                 goto nla_put_failure;
5525
5526         nla_nest_end(skb, nest);
5527
5528         nlmsg_end(skb, nlh);
5529         return 0;
5530
5531 nla_put_failure:
5532         nlmsg_trim(skb, nlh);
5533         return -1;
5534 }
5535
5536 static int nft_setelem_parse_flags(const struct nft_set *set,
5537                                    const struct nlattr *attr, u32 *flags)
5538 {
5539         if (attr == NULL)
5540                 return 0;
5541
5542         *flags = ntohl(nla_get_be32(attr));
5543         if (*flags & ~(NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
5544                 return -EOPNOTSUPP;
5545         if (!(set->flags & NFT_SET_INTERVAL) &&
5546             *flags & NFT_SET_ELEM_INTERVAL_END)
5547                 return -EINVAL;
5548         if ((*flags & (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL)) ==
5549             (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
5550                 return -EINVAL;
5551
5552         return 0;
5553 }
5554
5555 static int nft_setelem_parse_key(struct nft_ctx *ctx, struct nft_set *set,
5556                                  struct nft_data *key, struct nlattr *attr)
5557 {
5558         struct nft_data_desc desc = {
5559                 .type   = NFT_DATA_VALUE,
5560                 .size   = NFT_DATA_VALUE_MAXLEN,
5561                 .len    = set->klen,
5562         };
5563
5564         return nft_data_init(ctx, key, &desc, attr);
5565 }
5566
5567 static int nft_setelem_parse_data(struct nft_ctx *ctx, struct nft_set *set,
5568                                   struct nft_data_desc *desc,
5569                                   struct nft_data *data,
5570                                   struct nlattr *attr)
5571 {
5572         u32 dtype;
5573
5574         if (set->dtype == NFT_DATA_VERDICT)
5575                 dtype = NFT_DATA_VERDICT;
5576         else
5577                 dtype = NFT_DATA_VALUE;
5578
5579         desc->type = dtype;
5580         desc->size = NFT_DATA_VALUE_MAXLEN;
5581         desc->len = set->dlen;
5582         desc->flags = NFT_DATA_DESC_SETELEM;
5583
5584         return nft_data_init(ctx, data, desc, attr);
5585 }
5586
5587 static void *nft_setelem_catchall_get(const struct net *net,
5588                                       const struct nft_set *set)
5589 {
5590         struct nft_set_elem_catchall *catchall;
5591         u8 genmask = nft_genmask_cur(net);
5592         struct nft_set_ext *ext;
5593         void *priv = NULL;
5594
5595         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5596                 ext = nft_set_elem_ext(set, catchall->elem);
5597                 if (!nft_set_elem_active(ext, genmask) ||
5598                     nft_set_elem_expired(ext))
5599                         continue;
5600
5601                 priv = catchall->elem;
5602                 break;
5603         }
5604
5605         return priv;
5606 }
5607
5608 static int nft_setelem_get(struct nft_ctx *ctx, struct nft_set *set,
5609                            struct nft_set_elem *elem, u32 flags)
5610 {
5611         void *priv;
5612
5613         if (!(flags & NFT_SET_ELEM_CATCHALL)) {
5614                 priv = set->ops->get(ctx->net, set, elem, flags);
5615                 if (IS_ERR(priv))
5616                         return PTR_ERR(priv);
5617         } else {
5618                 priv = nft_setelem_catchall_get(ctx->net, set);
5619                 if (!priv)
5620                         return -ENOENT;
5621         }
5622         elem->priv = priv;
5623
5624         return 0;
5625 }
5626
5627 static int nft_get_set_elem(struct nft_ctx *ctx, struct nft_set *set,
5628                             const struct nlattr *attr)
5629 {
5630         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
5631         struct nft_set_elem elem;
5632         struct sk_buff *skb;
5633         uint32_t flags = 0;
5634         int err;
5635
5636         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
5637                                           nft_set_elem_policy, NULL);
5638         if (err < 0)
5639                 return err;
5640
5641         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
5642         if (err < 0)
5643                 return err;
5644
5645         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
5646                 return -EINVAL;
5647
5648         if (nla[NFTA_SET_ELEM_KEY]) {
5649                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
5650                                             nla[NFTA_SET_ELEM_KEY]);
5651                 if (err < 0)
5652                         return err;
5653         }
5654
5655         if (nla[NFTA_SET_ELEM_KEY_END]) {
5656                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
5657                                             nla[NFTA_SET_ELEM_KEY_END]);
5658                 if (err < 0)
5659                         return err;
5660         }
5661
5662         err = nft_setelem_get(ctx, set, &elem, flags);
5663         if (err < 0)
5664                 return err;
5665
5666         err = -ENOMEM;
5667         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_ATOMIC);
5668         if (skb == NULL)
5669                 return err;
5670
5671         err = nf_tables_fill_setelem_info(skb, ctx, ctx->seq, ctx->portid,
5672                                           NFT_MSG_NEWSETELEM, 0, set, &elem);
5673         if (err < 0)
5674                 goto err_fill_setelem;
5675
5676         return nfnetlink_unicast(skb, ctx->net, ctx->portid);
5677
5678 err_fill_setelem:
5679         kfree_skb(skb);
5680         return err;
5681 }
5682
5683 /* called with rcu_read_lock held */
5684 static int nf_tables_getsetelem(struct sk_buff *skb,
5685                                 const struct nfnl_info *info,
5686                                 const struct nlattr * const nla[])
5687 {
5688         struct netlink_ext_ack *extack = info->extack;
5689         u8 genmask = nft_genmask_cur(info->net);
5690         u8 family = info->nfmsg->nfgen_family;
5691         struct net *net = info->net;
5692         struct nft_table *table;
5693         struct nft_set *set;
5694         struct nlattr *attr;
5695         struct nft_ctx ctx;
5696         int rem, err = 0;
5697
5698         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
5699                                  genmask, 0);
5700         if (IS_ERR(table)) {
5701                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
5702                 return PTR_ERR(table);
5703         }
5704
5705         set = nft_set_lookup(table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
5706         if (IS_ERR(set))
5707                 return PTR_ERR(set);
5708
5709         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
5710
5711         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
5712                 struct netlink_dump_control c = {
5713                         .start = nf_tables_dump_set_start,
5714                         .dump = nf_tables_dump_set,
5715                         .done = nf_tables_dump_set_done,
5716                         .module = THIS_MODULE,
5717                 };
5718                 struct nft_set_dump_ctx dump_ctx = {
5719                         .set = set,
5720                         .ctx = ctx,
5721                 };
5722
5723                 c.data = &dump_ctx;
5724                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
5725         }
5726
5727         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
5728                 return -EINVAL;
5729
5730         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
5731                 err = nft_get_set_elem(&ctx, set, attr);
5732                 if (err < 0) {
5733                         NL_SET_BAD_ATTR(extack, attr);
5734                         break;
5735                 }
5736         }
5737
5738         return err;
5739 }
5740
5741 static void nf_tables_setelem_notify(const struct nft_ctx *ctx,
5742                                      const struct nft_set *set,
5743                                      const struct nft_set_elem *elem,
5744                                      int event)
5745 {
5746         struct nftables_pernet *nft_net;
5747         struct net *net = ctx->net;
5748         u32 portid = ctx->portid;
5749         struct sk_buff *skb;
5750         u16 flags = 0;
5751         int err;
5752
5753         if (!ctx->report && !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
5754                 return;
5755
5756         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
5757         if (skb == NULL)
5758                 goto err;
5759
5760         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
5761                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
5762
5763         err = nf_tables_fill_setelem_info(skb, ctx, 0, portid, event, flags,
5764                                           set, elem);
5765         if (err < 0) {
5766                 kfree_skb(skb);
5767                 goto err;
5768         }
5769
5770         nft_net = nft_pernet(net);
5771         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
5772         return;
5773 err:
5774         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
5775 }
5776
5777 static struct nft_trans *nft_trans_elem_alloc(struct nft_ctx *ctx,
5778                                               int msg_type,
5779                                               struct nft_set *set)
5780 {
5781         struct nft_trans *trans;
5782
5783         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_elem));
5784         if (trans == NULL)
5785                 return NULL;
5786
5787         nft_trans_elem_set(trans) = set;
5788         return trans;
5789 }
5790
5791 struct nft_expr *nft_set_elem_expr_alloc(const struct nft_ctx *ctx,
5792                                          const struct nft_set *set,
5793                                          const struct nlattr *attr)
5794 {
5795         struct nft_expr *expr;
5796         int err;
5797
5798         expr = nft_expr_init(ctx, attr);
5799         if (IS_ERR(expr))
5800                 return expr;
5801
5802         err = -EOPNOTSUPP;
5803         if (expr->ops->type->flags & NFT_EXPR_GC) {
5804                 if (set->flags & NFT_SET_TIMEOUT)
5805                         goto err_set_elem_expr;
5806                 if (!set->ops->gc_init)
5807                         goto err_set_elem_expr;
5808                 set->ops->gc_init(set);
5809         }
5810
5811         return expr;
5812
5813 err_set_elem_expr:
5814         nft_expr_destroy(ctx, expr);
5815         return ERR_PTR(err);
5816 }
5817
5818 static int nft_set_ext_check(const struct nft_set_ext_tmpl *tmpl, u8 id, u32 len)
5819 {
5820         len += nft_set_ext_types[id].len;
5821         if (len > tmpl->ext_len[id] ||
5822             len > U8_MAX)
5823                 return -1;
5824
5825         return 0;
5826 }
5827
5828 static int nft_set_ext_memcpy(const struct nft_set_ext_tmpl *tmpl, u8 id,
5829                               void *to, const void *from, u32 len)
5830 {
5831         if (nft_set_ext_check(tmpl, id, len) < 0)
5832                 return -1;
5833
5834         memcpy(to, from, len);
5835
5836         return 0;
5837 }
5838
5839 void *nft_set_elem_init(const struct nft_set *set,
5840                         const struct nft_set_ext_tmpl *tmpl,
5841                         const u32 *key, const u32 *key_end,
5842                         const u32 *data, u64 timeout, u64 expiration, gfp_t gfp)
5843 {
5844         struct nft_set_ext *ext;
5845         void *elem;
5846
5847         elem = kzalloc(set->ops->elemsize + tmpl->len, gfp);
5848         if (elem == NULL)
5849                 return ERR_PTR(-ENOMEM);
5850
5851         ext = nft_set_elem_ext(set, elem);
5852         nft_set_ext_init(ext, tmpl);
5853
5854         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
5855             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY,
5856                                nft_set_ext_key(ext), key, set->klen) < 0)
5857                 goto err_ext_check;
5858
5859         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
5860             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY_END,
5861                                nft_set_ext_key_end(ext), key_end, set->klen) < 0)
5862                 goto err_ext_check;
5863
5864         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
5865             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_DATA,
5866                                nft_set_ext_data(ext), data, set->dlen) < 0)
5867                 goto err_ext_check;
5868
5869         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
5870                 *nft_set_ext_expiration(ext) = get_jiffies_64() + expiration;
5871                 if (expiration == 0)
5872                         *nft_set_ext_expiration(ext) += timeout;
5873         }
5874         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT))
5875                 *nft_set_ext_timeout(ext) = timeout;
5876
5877         return elem;
5878
5879 err_ext_check:
5880         kfree(elem);
5881
5882         return ERR_PTR(-EINVAL);
5883 }
5884
5885 static void __nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
5886                                         struct nft_expr *expr)
5887 {
5888         if (expr->ops->destroy_clone) {
5889                 expr->ops->destroy_clone(ctx, expr);
5890                 module_put(expr->ops->type->owner);
5891         } else {
5892                 nf_tables_expr_destroy(ctx, expr);
5893         }
5894 }
5895
5896 static void nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
5897                                       struct nft_set_elem_expr *elem_expr)
5898 {
5899         struct nft_expr *expr;
5900         u32 size;
5901
5902         nft_setelem_expr_foreach(expr, elem_expr, size)
5903                 __nft_set_elem_expr_destroy(ctx, expr);
5904 }
5905
5906 void nft_set_elem_destroy(const struct nft_set *set, void *elem,
5907                           bool destroy_expr)
5908 {
5909         struct nft_set_ext *ext = nft_set_elem_ext(set, elem);
5910         struct nft_ctx ctx = {
5911                 .net    = read_pnet(&set->net),
5912                 .family = set->table->family,
5913         };
5914
5915         nft_data_release(nft_set_ext_key(ext), NFT_DATA_VALUE);
5916         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
5917                 nft_data_release(nft_set_ext_data(ext), set->dtype);
5918         if (destroy_expr && nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
5919                 nft_set_elem_expr_destroy(&ctx, nft_set_ext_expr(ext));
5920
5921         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
5922                 (*nft_set_ext_obj(ext))->use--;
5923         kfree(elem);
5924 }
5925 EXPORT_SYMBOL_GPL(nft_set_elem_destroy);
5926
5927 /* Only called from commit path, nft_setelem_data_deactivate() already deals
5928  * with the refcounting from the preparation phase.
5929  */
5930 static void nf_tables_set_elem_destroy(const struct nft_ctx *ctx,
5931                                        const struct nft_set *set, void *elem)
5932 {
5933         struct nft_set_ext *ext = nft_set_elem_ext(set, elem);
5934
5935         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
5936                 nft_set_elem_expr_destroy(ctx, nft_set_ext_expr(ext));
5937
5938         kfree(elem);
5939 }
5940
5941 int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set,
5942                             struct nft_expr *expr_array[])
5943 {
5944         struct nft_expr *expr;
5945         int err, i, k;
5946
5947         for (i = 0; i < set->num_exprs; i++) {
5948                 expr = kzalloc(set->exprs[i]->ops->size, GFP_KERNEL_ACCOUNT);
5949                 if (!expr)
5950                         goto err_expr;
5951
5952                 err = nft_expr_clone(expr, set->exprs[i]);
5953                 if (err < 0) {
5954                         kfree(expr);
5955                         goto err_expr;
5956                 }
5957                 expr_array[i] = expr;
5958         }
5959
5960         return 0;
5961
5962 err_expr:
5963         for (k = i - 1; k >= 0; k--)
5964                 nft_expr_destroy(ctx, expr_array[k]);
5965
5966         return -ENOMEM;
5967 }
5968
5969 static int nft_set_elem_expr_setup(struct nft_ctx *ctx,
5970                                    const struct nft_set_ext_tmpl *tmpl,
5971                                    const struct nft_set_ext *ext,
5972                                    struct nft_expr *expr_array[],
5973                                    u32 num_exprs)
5974 {
5975         struct nft_set_elem_expr *elem_expr = nft_set_ext_expr(ext);
5976         u32 len = sizeof(struct nft_set_elem_expr);
5977         struct nft_expr *expr;
5978         int i, err;
5979
5980         if (num_exprs == 0)
5981                 return 0;
5982
5983         for (i = 0; i < num_exprs; i++)
5984                 len += expr_array[i]->ops->size;
5985
5986         if (nft_set_ext_check(tmpl, NFT_SET_EXT_EXPRESSIONS, len) < 0)
5987                 return -EINVAL;
5988
5989         for (i = 0; i < num_exprs; i++) {
5990                 expr = nft_setelem_expr_at(elem_expr, elem_expr->size);
5991                 err = nft_expr_clone(expr, expr_array[i]);
5992                 if (err < 0)
5993                         goto err_elem_expr_setup;
5994
5995                 elem_expr->size += expr_array[i]->ops->size;
5996                 nft_expr_destroy(ctx, expr_array[i]);
5997                 expr_array[i] = NULL;
5998         }
5999
6000         return 0;
6001
6002 err_elem_expr_setup:
6003         for (; i < num_exprs; i++) {
6004                 nft_expr_destroy(ctx, expr_array[i]);
6005                 expr_array[i] = NULL;
6006         }
6007
6008         return -ENOMEM;
6009 }
6010
6011 struct nft_set_ext *nft_set_catchall_lookup(const struct net *net,
6012                                             const struct nft_set *set)
6013 {
6014         struct nft_set_elem_catchall *catchall;
6015         u8 genmask = nft_genmask_cur(net);
6016         struct nft_set_ext *ext;
6017
6018         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
6019                 ext = nft_set_elem_ext(set, catchall->elem);
6020                 if (nft_set_elem_active(ext, genmask) &&
6021                     !nft_set_elem_expired(ext))
6022                         return ext;
6023         }
6024
6025         return NULL;
6026 }
6027 EXPORT_SYMBOL_GPL(nft_set_catchall_lookup);
6028
6029 void *nft_set_catchall_gc(const struct nft_set *set)
6030 {
6031         struct nft_set_elem_catchall *catchall, *next;
6032         struct nft_set_ext *ext;
6033         void *elem = NULL;
6034
6035         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
6036                 ext = nft_set_elem_ext(set, catchall->elem);
6037
6038                 if (!nft_set_elem_expired(ext) ||
6039                     nft_set_elem_mark_busy(ext))
6040                         continue;
6041
6042                 elem = catchall->elem;
6043                 list_del_rcu(&catchall->list);
6044                 kfree_rcu(catchall, rcu);
6045                 break;
6046         }
6047
6048         return elem;
6049 }
6050 EXPORT_SYMBOL_GPL(nft_set_catchall_gc);
6051
6052 static int nft_setelem_catchall_insert(const struct net *net,
6053                                        struct nft_set *set,
6054                                        const struct nft_set_elem *elem,
6055                                        struct nft_set_ext **pext)
6056 {
6057         struct nft_set_elem_catchall *catchall;
6058         u8 genmask = nft_genmask_next(net);
6059         struct nft_set_ext *ext;
6060
6061         list_for_each_entry(catchall, &set->catchall_list, list) {
6062                 ext = nft_set_elem_ext(set, catchall->elem);
6063                 if (nft_set_elem_active(ext, genmask)) {
6064                         *pext = ext;
6065                         return -EEXIST;
6066                 }
6067         }
6068
6069         catchall = kmalloc(sizeof(*catchall), GFP_KERNEL);
6070         if (!catchall)
6071                 return -ENOMEM;
6072
6073         catchall->elem = elem->priv;
6074         list_add_tail_rcu(&catchall->list, &set->catchall_list);
6075
6076         return 0;
6077 }
6078
6079 static int nft_setelem_insert(const struct net *net,
6080                               struct nft_set *set,
6081                               const struct nft_set_elem *elem,
6082                               struct nft_set_ext **ext, unsigned int flags)
6083 {
6084         int ret;
6085
6086         if (flags & NFT_SET_ELEM_CATCHALL)
6087                 ret = nft_setelem_catchall_insert(net, set, elem, ext);
6088         else
6089                 ret = set->ops->insert(net, set, elem, ext);
6090
6091         return ret;
6092 }
6093
6094 static bool nft_setelem_is_catchall(const struct nft_set *set,
6095                                     const struct nft_set_elem *elem)
6096 {
6097         struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6098
6099         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
6100             *nft_set_ext_flags(ext) & NFT_SET_ELEM_CATCHALL)
6101                 return true;
6102
6103         return false;
6104 }
6105
6106 static void nft_setelem_activate(struct net *net, struct nft_set *set,
6107                                  struct nft_set_elem *elem)
6108 {
6109         struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6110
6111         if (nft_setelem_is_catchall(set, elem)) {
6112                 nft_set_elem_change_active(net, set, ext);
6113                 nft_set_elem_clear_busy(ext);
6114         } else {
6115                 set->ops->activate(net, set, elem);
6116         }
6117 }
6118
6119 static int nft_setelem_catchall_deactivate(const struct net *net,
6120                                            struct nft_set *set,
6121                                            struct nft_set_elem *elem)
6122 {
6123         struct nft_set_elem_catchall *catchall;
6124         struct nft_set_ext *ext;
6125
6126         list_for_each_entry(catchall, &set->catchall_list, list) {
6127                 ext = nft_set_elem_ext(set, catchall->elem);
6128                 if (!nft_is_active(net, ext) ||
6129                     nft_set_elem_mark_busy(ext))
6130                         continue;
6131
6132                 kfree(elem->priv);
6133                 elem->priv = catchall->elem;
6134                 nft_set_elem_change_active(net, set, ext);
6135                 return 0;
6136         }
6137
6138         return -ENOENT;
6139 }
6140
6141 static int __nft_setelem_deactivate(const struct net *net,
6142                                     struct nft_set *set,
6143                                     struct nft_set_elem *elem)
6144 {
6145         void *priv;
6146
6147         priv = set->ops->deactivate(net, set, elem);
6148         if (!priv)
6149                 return -ENOENT;
6150
6151         kfree(elem->priv);
6152         elem->priv = priv;
6153         set->ndeact++;
6154
6155         return 0;
6156 }
6157
6158 static int nft_setelem_deactivate(const struct net *net,
6159                                   struct nft_set *set,
6160                                   struct nft_set_elem *elem, u32 flags)
6161 {
6162         int ret;
6163
6164         if (flags & NFT_SET_ELEM_CATCHALL)
6165                 ret = nft_setelem_catchall_deactivate(net, set, elem);
6166         else
6167                 ret = __nft_setelem_deactivate(net, set, elem);
6168
6169         return ret;
6170 }
6171
6172 static void nft_setelem_catchall_remove(const struct net *net,
6173                                         const struct nft_set *set,
6174                                         const struct nft_set_elem *elem)
6175 {
6176         struct nft_set_elem_catchall *catchall, *next;
6177
6178         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
6179                 if (catchall->elem == elem->priv) {
6180                         list_del_rcu(&catchall->list);
6181                         kfree_rcu(catchall, rcu);
6182                         break;
6183                 }
6184         }
6185 }
6186
6187 static void nft_setelem_remove(const struct net *net,
6188                                const struct nft_set *set,
6189                                const struct nft_set_elem *elem)
6190 {
6191         if (nft_setelem_is_catchall(set, elem))
6192                 nft_setelem_catchall_remove(net, set, elem);
6193         else
6194                 set->ops->remove(net, set, elem);
6195 }
6196
6197 static bool nft_setelem_valid_key_end(const struct nft_set *set,
6198                                       struct nlattr **nla, u32 flags)
6199 {
6200         if ((set->flags & (NFT_SET_CONCAT | NFT_SET_INTERVAL)) ==
6201                           (NFT_SET_CONCAT | NFT_SET_INTERVAL)) {
6202                 if (flags & NFT_SET_ELEM_INTERVAL_END)
6203                         return false;
6204
6205                 if (nla[NFTA_SET_ELEM_KEY_END] &&
6206                     flags & NFT_SET_ELEM_CATCHALL)
6207                         return false;
6208         } else {
6209                 if (nla[NFTA_SET_ELEM_KEY_END])
6210                         return false;
6211         }
6212
6213         return true;
6214 }
6215
6216 static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set,
6217                             const struct nlattr *attr, u32 nlmsg_flags)
6218 {
6219         struct nft_expr *expr_array[NFT_SET_EXPR_MAX] = {};
6220         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
6221         u8 genmask = nft_genmask_next(ctx->net);
6222         u32 flags = 0, size = 0, num_exprs = 0;
6223         struct nft_set_ext_tmpl tmpl;
6224         struct nft_set_ext *ext, *ext2;
6225         struct nft_set_elem elem;
6226         struct nft_set_binding *binding;
6227         struct nft_object *obj = NULL;
6228         struct nft_userdata *udata;
6229         struct nft_data_desc desc;
6230         enum nft_registers dreg;
6231         struct nft_trans *trans;
6232         u64 timeout;
6233         u64 expiration;
6234         int err, i;
6235         u8 ulen;
6236
6237         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
6238                                           nft_set_elem_policy, NULL);
6239         if (err < 0)
6240                 return err;
6241
6242         nft_set_ext_prepare(&tmpl);
6243
6244         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
6245         if (err < 0)
6246                 return err;
6247
6248         if (((flags & NFT_SET_ELEM_CATCHALL) && nla[NFTA_SET_ELEM_KEY]) ||
6249             (!(flags & NFT_SET_ELEM_CATCHALL) && !nla[NFTA_SET_ELEM_KEY]))
6250                 return -EINVAL;
6251
6252         if (flags != 0) {
6253                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
6254                 if (err < 0)
6255                         return err;
6256         }
6257
6258         if (set->flags & NFT_SET_MAP) {
6259                 if (nla[NFTA_SET_ELEM_DATA] == NULL &&
6260                     !(flags & NFT_SET_ELEM_INTERVAL_END))
6261                         return -EINVAL;
6262         } else {
6263                 if (nla[NFTA_SET_ELEM_DATA] != NULL)
6264                         return -EINVAL;
6265         }
6266
6267         if (set->flags & NFT_SET_OBJECT) {
6268                 if (!nla[NFTA_SET_ELEM_OBJREF] &&
6269                     !(flags & NFT_SET_ELEM_INTERVAL_END))
6270                         return -EINVAL;
6271         } else {
6272                 if (nla[NFTA_SET_ELEM_OBJREF])
6273                         return -EINVAL;
6274         }
6275
6276         if (!nft_setelem_valid_key_end(set, nla, flags))
6277                 return -EINVAL;
6278
6279         if ((flags & NFT_SET_ELEM_INTERVAL_END) &&
6280              (nla[NFTA_SET_ELEM_DATA] ||
6281               nla[NFTA_SET_ELEM_OBJREF] ||
6282               nla[NFTA_SET_ELEM_TIMEOUT] ||
6283               nla[NFTA_SET_ELEM_EXPIRATION] ||
6284               nla[NFTA_SET_ELEM_USERDATA] ||
6285               nla[NFTA_SET_ELEM_EXPR] ||
6286               nla[NFTA_SET_ELEM_KEY_END] ||
6287               nla[NFTA_SET_ELEM_EXPRESSIONS]))
6288                 return -EINVAL;
6289
6290         timeout = 0;
6291         if (nla[NFTA_SET_ELEM_TIMEOUT] != NULL) {
6292                 if (!(set->flags & NFT_SET_TIMEOUT))
6293                         return -EINVAL;
6294                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_TIMEOUT],
6295                                             &timeout);
6296                 if (err)
6297                         return err;
6298         } else if (set->flags & NFT_SET_TIMEOUT &&
6299                    !(flags & NFT_SET_ELEM_INTERVAL_END)) {
6300                 timeout = READ_ONCE(set->timeout);
6301         }
6302
6303         expiration = 0;
6304         if (nla[NFTA_SET_ELEM_EXPIRATION] != NULL) {
6305                 if (!(set->flags & NFT_SET_TIMEOUT))
6306                         return -EINVAL;
6307                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_EXPIRATION],
6308                                             &expiration);
6309                 if (err)
6310                         return err;
6311         }
6312
6313         if (nla[NFTA_SET_ELEM_EXPR]) {
6314                 struct nft_expr *expr;
6315
6316                 if (set->num_exprs && set->num_exprs != 1)
6317                         return -EOPNOTSUPP;
6318
6319                 expr = nft_set_elem_expr_alloc(ctx, set,
6320                                                nla[NFTA_SET_ELEM_EXPR]);
6321                 if (IS_ERR(expr))
6322                         return PTR_ERR(expr);
6323
6324                 expr_array[0] = expr;
6325                 num_exprs = 1;
6326
6327                 if (set->num_exprs && set->exprs[0]->ops != expr->ops) {
6328                         err = -EOPNOTSUPP;
6329                         goto err_set_elem_expr;
6330                 }
6331         } else if (nla[NFTA_SET_ELEM_EXPRESSIONS]) {
6332                 struct nft_expr *expr;
6333                 struct nlattr *tmp;
6334                 int left;
6335
6336                 i = 0;
6337                 nla_for_each_nested(tmp, nla[NFTA_SET_ELEM_EXPRESSIONS], left) {
6338                         if (i == NFT_SET_EXPR_MAX ||
6339                             (set->num_exprs && set->num_exprs == i)) {
6340                                 err = -E2BIG;
6341                                 goto err_set_elem_expr;
6342                         }
6343                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
6344                                 err = -EINVAL;
6345                                 goto err_set_elem_expr;
6346                         }
6347                         expr = nft_set_elem_expr_alloc(ctx, set, tmp);
6348                         if (IS_ERR(expr)) {
6349                                 err = PTR_ERR(expr);
6350                                 goto err_set_elem_expr;
6351                         }
6352                         expr_array[i] = expr;
6353                         num_exprs++;
6354
6355                         if (set->num_exprs && expr->ops != set->exprs[i]->ops) {
6356                                 err = -EOPNOTSUPP;
6357                                 goto err_set_elem_expr;
6358                         }
6359                         i++;
6360                 }
6361                 if (set->num_exprs && set->num_exprs != i) {
6362                         err = -EOPNOTSUPP;
6363                         goto err_set_elem_expr;
6364                 }
6365         } else if (set->num_exprs > 0 &&
6366                    !(flags & NFT_SET_ELEM_INTERVAL_END)) {
6367                 err = nft_set_elem_expr_clone(ctx, set, expr_array);
6368                 if (err < 0)
6369                         goto err_set_elem_expr_clone;
6370
6371                 num_exprs = set->num_exprs;
6372         }
6373
6374         if (nla[NFTA_SET_ELEM_KEY]) {
6375                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
6376                                             nla[NFTA_SET_ELEM_KEY]);
6377                 if (err < 0)
6378                         goto err_set_elem_expr;
6379
6380                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
6381                 if (err < 0)
6382                         goto err_parse_key;
6383         }
6384
6385         if (nla[NFTA_SET_ELEM_KEY_END]) {
6386                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
6387                                             nla[NFTA_SET_ELEM_KEY_END]);
6388                 if (err < 0)
6389                         goto err_parse_key;
6390
6391                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
6392                 if (err < 0)
6393                         goto err_parse_key_end;
6394         }
6395
6396         if (timeout > 0) {
6397                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_EXPIRATION);
6398                 if (err < 0)
6399                         goto err_parse_key_end;
6400
6401                 if (timeout != READ_ONCE(set->timeout)) {
6402                         err = nft_set_ext_add(&tmpl, NFT_SET_EXT_TIMEOUT);
6403                         if (err < 0)
6404                                 goto err_parse_key_end;
6405                 }
6406         }
6407
6408         if (num_exprs) {
6409                 for (i = 0; i < num_exprs; i++)
6410                         size += expr_array[i]->ops->size;
6411
6412                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_EXPRESSIONS,
6413                                              sizeof(struct nft_set_elem_expr) + size);
6414                 if (err < 0)
6415                         goto err_parse_key_end;
6416         }
6417
6418         if (nla[NFTA_SET_ELEM_OBJREF] != NULL) {
6419                 obj = nft_obj_lookup(ctx->net, ctx->table,
6420                                      nla[NFTA_SET_ELEM_OBJREF],
6421                                      set->objtype, genmask);
6422                 if (IS_ERR(obj)) {
6423                         err = PTR_ERR(obj);
6424                         goto err_parse_key_end;
6425                 }
6426                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_OBJREF);
6427                 if (err < 0)
6428                         goto err_parse_key_end;
6429         }
6430
6431         if (nla[NFTA_SET_ELEM_DATA] != NULL) {
6432                 err = nft_setelem_parse_data(ctx, set, &desc, &elem.data.val,
6433                                              nla[NFTA_SET_ELEM_DATA]);
6434                 if (err < 0)
6435                         goto err_parse_key_end;
6436
6437                 dreg = nft_type_to_reg(set->dtype);
6438                 list_for_each_entry(binding, &set->bindings, list) {
6439                         struct nft_ctx bind_ctx = {
6440                                 .net    = ctx->net,
6441                                 .family = ctx->family,
6442                                 .table  = ctx->table,
6443                                 .chain  = (struct nft_chain *)binding->chain,
6444                         };
6445
6446                         if (!(binding->flags & NFT_SET_MAP))
6447                                 continue;
6448
6449                         err = nft_validate_register_store(&bind_ctx, dreg,
6450                                                           &elem.data.val,
6451                                                           desc.type, desc.len);
6452                         if (err < 0)
6453                                 goto err_parse_data;
6454
6455                         if (desc.type == NFT_DATA_VERDICT &&
6456                             (elem.data.val.verdict.code == NFT_GOTO ||
6457                              elem.data.val.verdict.code == NFT_JUMP))
6458                                 nft_validate_state_update(ctx->table,
6459                                                           NFT_VALIDATE_NEED);
6460                 }
6461
6462                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_DATA, desc.len);
6463                 if (err < 0)
6464                         goto err_parse_data;
6465         }
6466
6467         /* The full maximum length of userdata can exceed the maximum
6468          * offset value (U8_MAX) for following extensions, therefor it
6469          * must be the last extension added.
6470          */
6471         ulen = 0;
6472         if (nla[NFTA_SET_ELEM_USERDATA] != NULL) {
6473                 ulen = nla_len(nla[NFTA_SET_ELEM_USERDATA]);
6474                 if (ulen > 0) {
6475                         err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_USERDATA,
6476                                                      ulen);
6477                         if (err < 0)
6478                                 goto err_parse_data;
6479                 }
6480         }
6481
6482         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
6483                                       elem.key_end.val.data, elem.data.val.data,
6484                                       timeout, expiration, GFP_KERNEL_ACCOUNT);
6485         if (IS_ERR(elem.priv)) {
6486                 err = PTR_ERR(elem.priv);
6487                 goto err_parse_data;
6488         }
6489
6490         ext = nft_set_elem_ext(set, elem.priv);
6491         if (flags)
6492                 *nft_set_ext_flags(ext) = flags;
6493
6494         if (ulen > 0) {
6495                 if (nft_set_ext_check(&tmpl, NFT_SET_EXT_USERDATA, ulen) < 0) {
6496                         err = -EINVAL;
6497                         goto err_elem_userdata;
6498                 }
6499                 udata = nft_set_ext_userdata(ext);
6500                 udata->len = ulen - 1;
6501                 nla_memcpy(&udata->data, nla[NFTA_SET_ELEM_USERDATA], ulen);
6502         }
6503         if (obj) {
6504                 *nft_set_ext_obj(ext) = obj;
6505                 obj->use++;
6506         }
6507         err = nft_set_elem_expr_setup(ctx, &tmpl, ext, expr_array, num_exprs);
6508         if (err < 0)
6509                 goto err_elem_free;
6510
6511         trans = nft_trans_elem_alloc(ctx, NFT_MSG_NEWSETELEM, set);
6512         if (trans == NULL) {
6513                 err = -ENOMEM;
6514                 goto err_elem_free;
6515         }
6516
6517         ext->genmask = nft_genmask_cur(ctx->net) | NFT_SET_ELEM_BUSY_MASK;
6518
6519         err = nft_setelem_insert(ctx->net, set, &elem, &ext2, flags);
6520         if (err) {
6521                 if (err == -EEXIST) {
6522                         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) ^
6523                             nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) ||
6524                             nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) ^
6525                             nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF))
6526                                 goto err_element_clash;
6527                         if ((nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
6528                              nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) &&
6529                              memcmp(nft_set_ext_data(ext),
6530                                     nft_set_ext_data(ext2), set->dlen) != 0) ||
6531                             (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
6532                              nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF) &&
6533                              *nft_set_ext_obj(ext) != *nft_set_ext_obj(ext2)))
6534                                 goto err_element_clash;
6535                         else if (!(nlmsg_flags & NLM_F_EXCL))
6536                                 err = 0;
6537                 } else if (err == -ENOTEMPTY) {
6538                         /* ENOTEMPTY reports overlapping between this element
6539                          * and an existing one.
6540                          */
6541                         err = -EEXIST;
6542                 }
6543                 goto err_element_clash;
6544         }
6545
6546         if (!(flags & NFT_SET_ELEM_CATCHALL) && set->size &&
6547             !atomic_add_unless(&set->nelems, 1, set->size + set->ndeact)) {
6548                 err = -ENFILE;
6549                 goto err_set_full;
6550         }
6551
6552         nft_trans_elem(trans) = elem;
6553         nft_trans_commit_list_add_tail(ctx->net, trans);
6554         return 0;
6555
6556 err_set_full:
6557         nft_setelem_remove(ctx->net, set, &elem);
6558 err_element_clash:
6559         kfree(trans);
6560 err_elem_free:
6561         if (obj)
6562                 obj->use--;
6563 err_elem_userdata:
6564         nf_tables_set_elem_destroy(ctx, set, elem.priv);
6565 err_parse_data:
6566         if (nla[NFTA_SET_ELEM_DATA] != NULL)
6567                 nft_data_release(&elem.data.val, desc.type);
6568 err_parse_key_end:
6569         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
6570 err_parse_key:
6571         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
6572 err_set_elem_expr:
6573         for (i = 0; i < num_exprs && expr_array[i]; i++)
6574                 nft_expr_destroy(ctx, expr_array[i]);
6575 err_set_elem_expr_clone:
6576         return err;
6577 }
6578
6579 static int nf_tables_newsetelem(struct sk_buff *skb,
6580                                 const struct nfnl_info *info,
6581                                 const struct nlattr * const nla[])
6582 {
6583         struct netlink_ext_ack *extack = info->extack;
6584         u8 genmask = nft_genmask_next(info->net);
6585         u8 family = info->nfmsg->nfgen_family;
6586         struct net *net = info->net;
6587         const struct nlattr *attr;
6588         struct nft_table *table;
6589         struct nft_set *set;
6590         struct nft_ctx ctx;
6591         int rem, err;
6592
6593         if (nla[NFTA_SET_ELEM_LIST_ELEMENTS] == NULL)
6594                 return -EINVAL;
6595
6596         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
6597                                  genmask, NETLINK_CB(skb).portid);
6598         if (IS_ERR(table)) {
6599                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
6600                 return PTR_ERR(table);
6601         }
6602
6603         set = nft_set_lookup_global(net, table, nla[NFTA_SET_ELEM_LIST_SET],
6604                                     nla[NFTA_SET_ELEM_LIST_SET_ID], genmask);
6605         if (IS_ERR(set))
6606                 return PTR_ERR(set);
6607
6608         if (!list_empty(&set->bindings) && set->flags & NFT_SET_CONSTANT)
6609                 return -EBUSY;
6610
6611         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
6612
6613         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
6614                 err = nft_add_set_elem(&ctx, set, attr, info->nlh->nlmsg_flags);
6615                 if (err < 0) {
6616                         NL_SET_BAD_ATTR(extack, attr);
6617                         return err;
6618                 }
6619         }
6620
6621         if (table->validate_state == NFT_VALIDATE_DO)
6622                 return nft_table_validate(net, table);
6623
6624         return 0;
6625 }
6626
6627 /**
6628  *      nft_data_hold - hold a nft_data item
6629  *
6630  *      @data: struct nft_data to release
6631  *      @type: type of data
6632  *
6633  *      Hold a nft_data item. NFT_DATA_VALUE types can be silently discarded,
6634  *      NFT_DATA_VERDICT bumps the reference to chains in case of NFT_JUMP and
6635  *      NFT_GOTO verdicts. This function must be called on active data objects
6636  *      from the second phase of the commit protocol.
6637  */
6638 void nft_data_hold(const struct nft_data *data, enum nft_data_types type)
6639 {
6640         struct nft_chain *chain;
6641         struct nft_rule *rule;
6642
6643         if (type == NFT_DATA_VERDICT) {
6644                 switch (data->verdict.code) {
6645                 case NFT_JUMP:
6646                 case NFT_GOTO:
6647                         chain = data->verdict.chain;
6648                         chain->use++;
6649
6650                         if (!nft_chain_is_bound(chain))
6651                                 break;
6652
6653                         chain->table->use++;
6654                         list_for_each_entry(rule, &chain->rules, list)
6655                                 chain->use++;
6656
6657                         nft_chain_add(chain->table, chain);
6658                         break;
6659                 }
6660         }
6661 }
6662
6663 static void nft_setelem_data_activate(const struct net *net,
6664                                       const struct nft_set *set,
6665                                       struct nft_set_elem *elem)
6666 {
6667         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6668
6669         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
6670                 nft_data_hold(nft_set_ext_data(ext), set->dtype);
6671         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
6672                 (*nft_set_ext_obj(ext))->use++;
6673 }
6674
6675 static void nft_setelem_data_deactivate(const struct net *net,
6676                                         const struct nft_set *set,
6677                                         struct nft_set_elem *elem)
6678 {
6679         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6680
6681         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
6682                 nft_data_release(nft_set_ext_data(ext), set->dtype);
6683         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
6684                 (*nft_set_ext_obj(ext))->use--;
6685 }
6686
6687 static int nft_del_setelem(struct nft_ctx *ctx, struct nft_set *set,
6688                            const struct nlattr *attr)
6689 {
6690         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
6691         struct nft_set_ext_tmpl tmpl;
6692         struct nft_set_elem elem;
6693         struct nft_set_ext *ext;
6694         struct nft_trans *trans;
6695         u32 flags = 0;
6696         int err;
6697
6698         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
6699                                           nft_set_elem_policy, NULL);
6700         if (err < 0)
6701                 return err;
6702
6703         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
6704         if (err < 0)
6705                 return err;
6706
6707         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
6708                 return -EINVAL;
6709
6710         if (!nft_setelem_valid_key_end(set, nla, flags))
6711                 return -EINVAL;
6712
6713         nft_set_ext_prepare(&tmpl);
6714
6715         if (flags != 0) {
6716                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
6717                 if (err < 0)
6718                         return err;
6719         }
6720
6721         if (nla[NFTA_SET_ELEM_KEY]) {
6722                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
6723                                             nla[NFTA_SET_ELEM_KEY]);
6724                 if (err < 0)
6725                         return err;
6726
6727                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
6728                 if (err < 0)
6729                         goto fail_elem;
6730         }
6731
6732         if (nla[NFTA_SET_ELEM_KEY_END]) {
6733                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
6734                                             nla[NFTA_SET_ELEM_KEY_END]);
6735                 if (err < 0)
6736                         goto fail_elem;
6737
6738                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
6739                 if (err < 0)
6740                         goto fail_elem_key_end;
6741         }
6742
6743         err = -ENOMEM;
6744         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
6745                                       elem.key_end.val.data, NULL, 0, 0,
6746                                       GFP_KERNEL_ACCOUNT);
6747         if (IS_ERR(elem.priv)) {
6748                 err = PTR_ERR(elem.priv);
6749                 goto fail_elem_key_end;
6750         }
6751
6752         ext = nft_set_elem_ext(set, elem.priv);
6753         if (flags)
6754                 *nft_set_ext_flags(ext) = flags;
6755
6756         trans = nft_trans_elem_alloc(ctx, NFT_MSG_DELSETELEM, set);
6757         if (trans == NULL)
6758                 goto fail_trans;
6759
6760         err = nft_setelem_deactivate(ctx->net, set, &elem, flags);
6761         if (err < 0)
6762                 goto fail_ops;
6763
6764         nft_setelem_data_deactivate(ctx->net, set, &elem);
6765
6766         nft_trans_elem(trans) = elem;
6767         nft_trans_commit_list_add_tail(ctx->net, trans);
6768         return 0;
6769
6770 fail_ops:
6771         kfree(trans);
6772 fail_trans:
6773         kfree(elem.priv);
6774 fail_elem_key_end:
6775         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
6776 fail_elem:
6777         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
6778         return err;
6779 }
6780
6781 static int nft_setelem_flush(const struct nft_ctx *ctx,
6782                              struct nft_set *set,
6783                              const struct nft_set_iter *iter,
6784                              struct nft_set_elem *elem)
6785 {
6786         struct nft_trans *trans;
6787         int err;
6788
6789         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
6790                                     sizeof(struct nft_trans_elem), GFP_ATOMIC);
6791         if (!trans)
6792                 return -ENOMEM;
6793
6794         if (!set->ops->flush(ctx->net, set, elem->priv)) {
6795                 err = -ENOENT;
6796                 goto err1;
6797         }
6798         set->ndeact++;
6799
6800         nft_setelem_data_deactivate(ctx->net, set, elem);
6801         nft_trans_elem_set(trans) = set;
6802         nft_trans_elem(trans) = *elem;
6803         nft_trans_commit_list_add_tail(ctx->net, trans);
6804
6805         return 0;
6806 err1:
6807         kfree(trans);
6808         return err;
6809 }
6810
6811 static int __nft_set_catchall_flush(const struct nft_ctx *ctx,
6812                                     struct nft_set *set,
6813                                     struct nft_set_elem *elem)
6814 {
6815         struct nft_trans *trans;
6816
6817         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
6818                                     sizeof(struct nft_trans_elem), GFP_KERNEL);
6819         if (!trans)
6820                 return -ENOMEM;
6821
6822         nft_setelem_data_deactivate(ctx->net, set, elem);
6823         nft_trans_elem_set(trans) = set;
6824         nft_trans_elem(trans) = *elem;
6825         nft_trans_commit_list_add_tail(ctx->net, trans);
6826
6827         return 0;
6828 }
6829
6830 static int nft_set_catchall_flush(const struct nft_ctx *ctx,
6831                                   struct nft_set *set)
6832 {
6833         u8 genmask = nft_genmask_next(ctx->net);
6834         struct nft_set_elem_catchall *catchall;
6835         struct nft_set_elem elem;
6836         struct nft_set_ext *ext;
6837         int ret = 0;
6838
6839         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
6840                 ext = nft_set_elem_ext(set, catchall->elem);
6841                 if (!nft_set_elem_active(ext, genmask) ||
6842                     nft_set_elem_mark_busy(ext))
6843                         continue;
6844
6845                 elem.priv = catchall->elem;
6846                 ret = __nft_set_catchall_flush(ctx, set, &elem);
6847                 if (ret < 0)
6848                         break;
6849         }
6850
6851         return ret;
6852 }
6853
6854 static int nft_set_flush(struct nft_ctx *ctx, struct nft_set *set, u8 genmask)
6855 {
6856         struct nft_set_iter iter = {
6857                 .genmask        = genmask,
6858                 .fn             = nft_setelem_flush,
6859         };
6860
6861         set->ops->walk(ctx, set, &iter);
6862         if (!iter.err)
6863                 iter.err = nft_set_catchall_flush(ctx, set);
6864
6865         return iter.err;
6866 }
6867
6868 static int nf_tables_delsetelem(struct sk_buff *skb,
6869                                 const struct nfnl_info *info,
6870                                 const struct nlattr * const nla[])
6871 {
6872         struct netlink_ext_ack *extack = info->extack;
6873         u8 genmask = nft_genmask_next(info->net);
6874         u8 family = info->nfmsg->nfgen_family;
6875         struct net *net = info->net;
6876         const struct nlattr *attr;
6877         struct nft_table *table;
6878         struct nft_set *set;
6879         struct nft_ctx ctx;
6880         int rem, err = 0;
6881
6882         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
6883                                  genmask, NETLINK_CB(skb).portid);
6884         if (IS_ERR(table)) {
6885                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
6886                 return PTR_ERR(table);
6887         }
6888
6889         set = nft_set_lookup(table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
6890         if (IS_ERR(set))
6891                 return PTR_ERR(set);
6892         if (!list_empty(&set->bindings) && set->flags & NFT_SET_CONSTANT)
6893                 return -EBUSY;
6894
6895         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
6896
6897         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
6898                 return nft_set_flush(&ctx, set, genmask);
6899
6900         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
6901                 err = nft_del_setelem(&ctx, set, attr);
6902                 if (err == -ENOENT &&
6903                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYSETELEM)
6904                         continue;
6905
6906                 if (err < 0) {
6907                         NL_SET_BAD_ATTR(extack, attr);
6908                         break;
6909                 }
6910         }
6911         return err;
6912 }
6913
6914 void nft_set_gc_batch_release(struct rcu_head *rcu)
6915 {
6916         struct nft_set_gc_batch *gcb;
6917         unsigned int i;
6918
6919         gcb = container_of(rcu, struct nft_set_gc_batch, head.rcu);
6920         for (i = 0; i < gcb->head.cnt; i++)
6921                 nft_set_elem_destroy(gcb->head.set, gcb->elems[i], true);
6922         kfree(gcb);
6923 }
6924
6925 struct nft_set_gc_batch *nft_set_gc_batch_alloc(const struct nft_set *set,
6926                                                 gfp_t gfp)
6927 {
6928         struct nft_set_gc_batch *gcb;
6929
6930         gcb = kzalloc(sizeof(*gcb), gfp);
6931         if (gcb == NULL)
6932                 return gcb;
6933         gcb->head.set = set;
6934         return gcb;
6935 }
6936
6937 /*
6938  * Stateful objects
6939  */
6940
6941 /**
6942  *      nft_register_obj- register nf_tables stateful object type
6943  *      @obj_type: object type
6944  *
6945  *      Registers the object type for use with nf_tables. Returns zero on
6946  *      success or a negative errno code otherwise.
6947  */
6948 int nft_register_obj(struct nft_object_type *obj_type)
6949 {
6950         if (obj_type->type == NFT_OBJECT_UNSPEC)
6951                 return -EINVAL;
6952
6953         nfnl_lock(NFNL_SUBSYS_NFTABLES);
6954         list_add_rcu(&obj_type->list, &nf_tables_objects);
6955         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
6956         return 0;
6957 }
6958 EXPORT_SYMBOL_GPL(nft_register_obj);
6959
6960 /**
6961  *      nft_unregister_obj - unregister nf_tables object type
6962  *      @obj_type: object type
6963  *
6964  *      Unregisters the object type for use with nf_tables.
6965  */
6966 void nft_unregister_obj(struct nft_object_type *obj_type)
6967 {
6968         nfnl_lock(NFNL_SUBSYS_NFTABLES);
6969         list_del_rcu(&obj_type->list);
6970         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
6971 }
6972 EXPORT_SYMBOL_GPL(nft_unregister_obj);
6973
6974 struct nft_object *nft_obj_lookup(const struct net *net,
6975                                   const struct nft_table *table,
6976                                   const struct nlattr *nla, u32 objtype,
6977                                   u8 genmask)
6978 {
6979         struct nft_object_hash_key k = { .table = table };
6980         char search[NFT_OBJ_MAXNAMELEN];
6981         struct rhlist_head *tmp, *list;
6982         struct nft_object *obj;
6983
6984         nla_strscpy(search, nla, sizeof(search));
6985         k.name = search;
6986
6987         WARN_ON_ONCE(!rcu_read_lock_held() &&
6988                      !lockdep_commit_lock_is_held(net));
6989
6990         rcu_read_lock();
6991         list = rhltable_lookup(&nft_objname_ht, &k, nft_objname_ht_params);
6992         if (!list)
6993                 goto out;
6994
6995         rhl_for_each_entry_rcu(obj, tmp, list, rhlhead) {
6996                 if (objtype == obj->ops->type->type &&
6997                     nft_active_genmask(obj, genmask)) {
6998                         rcu_read_unlock();
6999                         return obj;
7000                 }
7001         }
7002 out:
7003         rcu_read_unlock();
7004         return ERR_PTR(-ENOENT);
7005 }
7006 EXPORT_SYMBOL_GPL(nft_obj_lookup);
7007
7008 static struct nft_object *nft_obj_lookup_byhandle(const struct nft_table *table,
7009                                                   const struct nlattr *nla,
7010                                                   u32 objtype, u8 genmask)
7011 {
7012         struct nft_object *obj;
7013
7014         list_for_each_entry(obj, &table->objects, list) {
7015                 if (be64_to_cpu(nla_get_be64(nla)) == obj->handle &&
7016                     objtype == obj->ops->type->type &&
7017                     nft_active_genmask(obj, genmask))
7018                         return obj;
7019         }
7020         return ERR_PTR(-ENOENT);
7021 }
7022
7023 static const struct nla_policy nft_obj_policy[NFTA_OBJ_MAX + 1] = {
7024         [NFTA_OBJ_TABLE]        = { .type = NLA_STRING,
7025                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
7026         [NFTA_OBJ_NAME]         = { .type = NLA_STRING,
7027                                     .len = NFT_OBJ_MAXNAMELEN - 1 },
7028         [NFTA_OBJ_TYPE]         = { .type = NLA_U32 },
7029         [NFTA_OBJ_DATA]         = { .type = NLA_NESTED },
7030         [NFTA_OBJ_HANDLE]       = { .type = NLA_U64},
7031         [NFTA_OBJ_USERDATA]     = { .type = NLA_BINARY,
7032                                     .len = NFT_USERDATA_MAXLEN },
7033 };
7034
7035 static struct nft_object *nft_obj_init(const struct nft_ctx *ctx,
7036                                        const struct nft_object_type *type,
7037                                        const struct nlattr *attr)
7038 {
7039         struct nlattr **tb;
7040         const struct nft_object_ops *ops;
7041         struct nft_object *obj;
7042         int err = -ENOMEM;
7043
7044         tb = kmalloc_array(type->maxattr + 1, sizeof(*tb), GFP_KERNEL);
7045         if (!tb)
7046                 goto err1;
7047
7048         if (attr) {
7049                 err = nla_parse_nested_deprecated(tb, type->maxattr, attr,
7050                                                   type->policy, NULL);
7051                 if (err < 0)
7052                         goto err2;
7053         } else {
7054                 memset(tb, 0, sizeof(tb[0]) * (type->maxattr + 1));
7055         }
7056
7057         if (type->select_ops) {
7058                 ops = type->select_ops(ctx, (const struct nlattr * const *)tb);
7059                 if (IS_ERR(ops)) {
7060                         err = PTR_ERR(ops);
7061                         goto err2;
7062                 }
7063         } else {
7064                 ops = type->ops;
7065         }
7066
7067         err = -ENOMEM;
7068         obj = kzalloc(sizeof(*obj) + ops->size, GFP_KERNEL_ACCOUNT);
7069         if (!obj)
7070                 goto err2;
7071
7072         err = ops->init(ctx, (const struct nlattr * const *)tb, obj);
7073         if (err < 0)
7074                 goto err3;
7075
7076         obj->ops = ops;
7077
7078         kfree(tb);
7079         return obj;
7080 err3:
7081         kfree(obj);
7082 err2:
7083         kfree(tb);
7084 err1:
7085         return ERR_PTR(err);
7086 }
7087
7088 static int nft_object_dump(struct sk_buff *skb, unsigned int attr,
7089                            struct nft_object *obj, bool reset)
7090 {
7091         struct nlattr *nest;
7092
7093         nest = nla_nest_start_noflag(skb, attr);
7094         if (!nest)
7095                 goto nla_put_failure;
7096         if (obj->ops->dump(skb, obj, reset) < 0)
7097                 goto nla_put_failure;
7098         nla_nest_end(skb, nest);
7099         return 0;
7100
7101 nla_put_failure:
7102         return -1;
7103 }
7104
7105 static const struct nft_object_type *__nft_obj_type_get(u32 objtype)
7106 {
7107         const struct nft_object_type *type;
7108
7109         list_for_each_entry(type, &nf_tables_objects, list) {
7110                 if (objtype == type->type)
7111                         return type;
7112         }
7113         return NULL;
7114 }
7115
7116 static const struct nft_object_type *
7117 nft_obj_type_get(struct net *net, u32 objtype)
7118 {
7119         const struct nft_object_type *type;
7120
7121         type = __nft_obj_type_get(objtype);
7122         if (type != NULL && try_module_get(type->owner))
7123                 return type;
7124
7125         lockdep_nfnl_nft_mutex_not_held();
7126 #ifdef CONFIG_MODULES
7127         if (type == NULL) {
7128                 if (nft_request_module(net, "nft-obj-%u", objtype) == -EAGAIN)
7129                         return ERR_PTR(-EAGAIN);
7130         }
7131 #endif
7132         return ERR_PTR(-ENOENT);
7133 }
7134
7135 static int nf_tables_updobj(const struct nft_ctx *ctx,
7136                             const struct nft_object_type *type,
7137                             const struct nlattr *attr,
7138                             struct nft_object *obj)
7139 {
7140         struct nft_object *newobj;
7141         struct nft_trans *trans;
7142         int err = -ENOMEM;
7143
7144         if (!try_module_get(type->owner))
7145                 return -ENOENT;
7146
7147         trans = nft_trans_alloc(ctx, NFT_MSG_NEWOBJ,
7148                                 sizeof(struct nft_trans_obj));
7149         if (!trans)
7150                 goto err_trans;
7151
7152         newobj = nft_obj_init(ctx, type, attr);
7153         if (IS_ERR(newobj)) {
7154                 err = PTR_ERR(newobj);
7155                 goto err_free_trans;
7156         }
7157
7158         nft_trans_obj(trans) = obj;
7159         nft_trans_obj_update(trans) = true;
7160         nft_trans_obj_newobj(trans) = newobj;
7161         nft_trans_commit_list_add_tail(ctx->net, trans);
7162
7163         return 0;
7164
7165 err_free_trans:
7166         kfree(trans);
7167 err_trans:
7168         module_put(type->owner);
7169         return err;
7170 }
7171
7172 static int nf_tables_newobj(struct sk_buff *skb, const struct nfnl_info *info,
7173                             const struct nlattr * const nla[])
7174 {
7175         struct netlink_ext_ack *extack = info->extack;
7176         u8 genmask = nft_genmask_next(info->net);
7177         u8 family = info->nfmsg->nfgen_family;
7178         const struct nft_object_type *type;
7179         struct net *net = info->net;
7180         struct nft_table *table;
7181         struct nft_object *obj;
7182         struct nft_ctx ctx;
7183         u32 objtype;
7184         int err;
7185
7186         if (!nla[NFTA_OBJ_TYPE] ||
7187             !nla[NFTA_OBJ_NAME] ||
7188             !nla[NFTA_OBJ_DATA])
7189                 return -EINVAL;
7190
7191         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
7192                                  NETLINK_CB(skb).portid);
7193         if (IS_ERR(table)) {
7194                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7195                 return PTR_ERR(table);
7196         }
7197
7198         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7199         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
7200         if (IS_ERR(obj)) {
7201                 err = PTR_ERR(obj);
7202                 if (err != -ENOENT) {
7203                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7204                         return err;
7205                 }
7206         } else {
7207                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
7208                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7209                         return -EEXIST;
7210                 }
7211                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
7212                         return -EOPNOTSUPP;
7213
7214                 type = __nft_obj_type_get(objtype);
7215                 if (WARN_ON_ONCE(!type))
7216                         return -ENOENT;
7217
7218                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7219
7220                 return nf_tables_updobj(&ctx, type, nla[NFTA_OBJ_DATA], obj);
7221         }
7222
7223         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7224
7225         type = nft_obj_type_get(net, objtype);
7226         if (IS_ERR(type))
7227                 return PTR_ERR(type);
7228
7229         obj = nft_obj_init(&ctx, type, nla[NFTA_OBJ_DATA]);
7230         if (IS_ERR(obj)) {
7231                 err = PTR_ERR(obj);
7232                 goto err_init;
7233         }
7234         obj->key.table = table;
7235         obj->handle = nf_tables_alloc_handle(table);
7236
7237         obj->key.name = nla_strdup(nla[NFTA_OBJ_NAME], GFP_KERNEL_ACCOUNT);
7238         if (!obj->key.name) {
7239                 err = -ENOMEM;
7240                 goto err_strdup;
7241         }
7242
7243         if (nla[NFTA_OBJ_USERDATA]) {
7244                 obj->udata = nla_memdup(nla[NFTA_OBJ_USERDATA], GFP_KERNEL_ACCOUNT);
7245                 if (obj->udata == NULL)
7246                         goto err_userdata;
7247
7248                 obj->udlen = nla_len(nla[NFTA_OBJ_USERDATA]);
7249         }
7250
7251         err = nft_trans_obj_add(&ctx, NFT_MSG_NEWOBJ, obj);
7252         if (err < 0)
7253                 goto err_trans;
7254
7255         err = rhltable_insert(&nft_objname_ht, &obj->rhlhead,
7256                               nft_objname_ht_params);
7257         if (err < 0)
7258                 goto err_obj_ht;
7259
7260         list_add_tail_rcu(&obj->list, &table->objects);
7261         table->use++;
7262         return 0;
7263 err_obj_ht:
7264         /* queued in transaction log */
7265         INIT_LIST_HEAD(&obj->list);
7266         return err;
7267 err_trans:
7268         kfree(obj->udata);
7269 err_userdata:
7270         kfree(obj->key.name);
7271 err_strdup:
7272         if (obj->ops->destroy)
7273                 obj->ops->destroy(&ctx, obj);
7274         kfree(obj);
7275 err_init:
7276         module_put(type->owner);
7277         return err;
7278 }
7279
7280 static int nf_tables_fill_obj_info(struct sk_buff *skb, struct net *net,
7281                                    u32 portid, u32 seq, int event, u32 flags,
7282                                    int family, const struct nft_table *table,
7283                                    struct nft_object *obj, bool reset)
7284 {
7285         struct nlmsghdr *nlh;
7286
7287         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
7288         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
7289                            NFNETLINK_V0, nft_base_seq(net));
7290         if (!nlh)
7291                 goto nla_put_failure;
7292
7293         if (nla_put_string(skb, NFTA_OBJ_TABLE, table->name) ||
7294             nla_put_string(skb, NFTA_OBJ_NAME, obj->key.name) ||
7295             nla_put_be64(skb, NFTA_OBJ_HANDLE, cpu_to_be64(obj->handle),
7296                          NFTA_OBJ_PAD))
7297                 goto nla_put_failure;
7298
7299         if (event == NFT_MSG_DELOBJ) {
7300                 nlmsg_end(skb, nlh);
7301                 return 0;
7302         }
7303
7304         if (nla_put_be32(skb, NFTA_OBJ_TYPE, htonl(obj->ops->type->type)) ||
7305             nla_put_be32(skb, NFTA_OBJ_USE, htonl(obj->use)) ||
7306             nft_object_dump(skb, NFTA_OBJ_DATA, obj, reset))
7307                 goto nla_put_failure;
7308
7309         if (obj->udata &&
7310             nla_put(skb, NFTA_OBJ_USERDATA, obj->udlen, obj->udata))
7311                 goto nla_put_failure;
7312
7313         nlmsg_end(skb, nlh);
7314         return 0;
7315
7316 nla_put_failure:
7317         nlmsg_trim(skb, nlh);
7318         return -1;
7319 }
7320
7321 struct nft_obj_filter {
7322         char            *table;
7323         u32             type;
7324 };
7325
7326 static int nf_tables_dump_obj(struct sk_buff *skb, struct netlink_callback *cb)
7327 {
7328         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
7329         const struct nft_table *table;
7330         unsigned int idx = 0, s_idx = cb->args[0];
7331         struct nft_obj_filter *filter = cb->data;
7332         struct net *net = sock_net(skb->sk);
7333         int family = nfmsg->nfgen_family;
7334         struct nftables_pernet *nft_net;
7335         struct nft_object *obj;
7336         bool reset = false;
7337
7338         if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
7339                 reset = true;
7340
7341         rcu_read_lock();
7342         nft_net = nft_pernet(net);
7343         cb->seq = READ_ONCE(nft_net->base_seq);
7344
7345         list_for_each_entry_rcu(table, &nft_net->tables, list) {
7346                 if (family != NFPROTO_UNSPEC && family != table->family)
7347                         continue;
7348
7349                 list_for_each_entry_rcu(obj, &table->objects, list) {
7350                         if (!nft_is_active(net, obj))
7351                                 goto cont;
7352                         if (idx < s_idx)
7353                                 goto cont;
7354                         if (idx > s_idx)
7355                                 memset(&cb->args[1], 0,
7356                                        sizeof(cb->args) - sizeof(cb->args[0]));
7357                         if (filter && filter->table &&
7358                             strcmp(filter->table, table->name))
7359                                 goto cont;
7360                         if (filter &&
7361                             filter->type != NFT_OBJECT_UNSPEC &&
7362                             obj->ops->type->type != filter->type)
7363                                 goto cont;
7364                         if (reset) {
7365                                 char *buf = kasprintf(GFP_ATOMIC,
7366                                                       "%s:%u",
7367                                                       table->name,
7368                                                       nft_net->base_seq);
7369
7370                                 audit_log_nfcfg(buf,
7371                                                 family,
7372                                                 obj->handle,
7373                                                 AUDIT_NFT_OP_OBJ_RESET,
7374                                                 GFP_ATOMIC);
7375                                 kfree(buf);
7376                         }
7377
7378                         if (nf_tables_fill_obj_info(skb, net, NETLINK_CB(cb->skb).portid,
7379                                                     cb->nlh->nlmsg_seq,
7380                                                     NFT_MSG_NEWOBJ,
7381                                                     NLM_F_MULTI | NLM_F_APPEND,
7382                                                     table->family, table,
7383                                                     obj, reset) < 0)
7384                                 goto done;
7385
7386                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
7387 cont:
7388                         idx++;
7389                 }
7390         }
7391 done:
7392         rcu_read_unlock();
7393
7394         cb->args[0] = idx;
7395         return skb->len;
7396 }
7397
7398 static int nf_tables_dump_obj_start(struct netlink_callback *cb)
7399 {
7400         const struct nlattr * const *nla = cb->data;
7401         struct nft_obj_filter *filter = NULL;
7402
7403         if (nla[NFTA_OBJ_TABLE] || nla[NFTA_OBJ_TYPE]) {
7404                 filter = kzalloc(sizeof(*filter), GFP_ATOMIC);
7405                 if (!filter)
7406                         return -ENOMEM;
7407
7408                 if (nla[NFTA_OBJ_TABLE]) {
7409                         filter->table = nla_strdup(nla[NFTA_OBJ_TABLE], GFP_ATOMIC);
7410                         if (!filter->table) {
7411                                 kfree(filter);
7412                                 return -ENOMEM;
7413                         }
7414                 }
7415
7416                 if (nla[NFTA_OBJ_TYPE])
7417                         filter->type = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7418         }
7419
7420         cb->data = filter;
7421         return 0;
7422 }
7423
7424 static int nf_tables_dump_obj_done(struct netlink_callback *cb)
7425 {
7426         struct nft_obj_filter *filter = cb->data;
7427
7428         if (filter) {
7429                 kfree(filter->table);
7430                 kfree(filter);
7431         }
7432
7433         return 0;
7434 }
7435
7436 /* called with rcu_read_lock held */
7437 static int nf_tables_getobj(struct sk_buff *skb, const struct nfnl_info *info,
7438                             const struct nlattr * const nla[])
7439 {
7440         struct netlink_ext_ack *extack = info->extack;
7441         u8 genmask = nft_genmask_cur(info->net);
7442         u8 family = info->nfmsg->nfgen_family;
7443         const struct nft_table *table;
7444         struct net *net = info->net;
7445         struct nft_object *obj;
7446         struct sk_buff *skb2;
7447         bool reset = false;
7448         u32 objtype;
7449         int err;
7450
7451         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
7452                 struct netlink_dump_control c = {
7453                         .start = nf_tables_dump_obj_start,
7454                         .dump = nf_tables_dump_obj,
7455                         .done = nf_tables_dump_obj_done,
7456                         .module = THIS_MODULE,
7457                         .data = (void *)nla,
7458                 };
7459
7460                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
7461         }
7462
7463         if (!nla[NFTA_OBJ_NAME] ||
7464             !nla[NFTA_OBJ_TYPE])
7465                 return -EINVAL;
7466
7467         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask, 0);
7468         if (IS_ERR(table)) {
7469                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7470                 return PTR_ERR(table);
7471         }
7472
7473         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7474         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
7475         if (IS_ERR(obj)) {
7476                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7477                 return PTR_ERR(obj);
7478         }
7479
7480         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
7481         if (!skb2)
7482                 return -ENOMEM;
7483
7484         if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
7485                 reset = true;
7486
7487         if (reset) {
7488                 const struct nftables_pernet *nft_net;
7489                 char *buf;
7490
7491                 nft_net = nft_pernet(net);
7492                 buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, nft_net->base_seq);
7493
7494                 audit_log_nfcfg(buf,
7495                                 family,
7496                                 obj->handle,
7497                                 AUDIT_NFT_OP_OBJ_RESET,
7498                                 GFP_ATOMIC);
7499                 kfree(buf);
7500         }
7501
7502         err = nf_tables_fill_obj_info(skb2, net, NETLINK_CB(skb).portid,
7503                                       info->nlh->nlmsg_seq, NFT_MSG_NEWOBJ, 0,
7504                                       family, table, obj, reset);
7505         if (err < 0)
7506                 goto err_fill_obj_info;
7507
7508         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
7509
7510 err_fill_obj_info:
7511         kfree_skb(skb2);
7512         return err;
7513 }
7514
7515 static void nft_obj_destroy(const struct nft_ctx *ctx, struct nft_object *obj)
7516 {
7517         if (obj->ops->destroy)
7518                 obj->ops->destroy(ctx, obj);
7519
7520         module_put(obj->ops->type->owner);
7521         kfree(obj->key.name);
7522         kfree(obj->udata);
7523         kfree(obj);
7524 }
7525
7526 static int nf_tables_delobj(struct sk_buff *skb, const struct nfnl_info *info,
7527                             const struct nlattr * const nla[])
7528 {
7529         struct netlink_ext_ack *extack = info->extack;
7530         u8 genmask = nft_genmask_next(info->net);
7531         u8 family = info->nfmsg->nfgen_family;
7532         struct net *net = info->net;
7533         const struct nlattr *attr;
7534         struct nft_table *table;
7535         struct nft_object *obj;
7536         struct nft_ctx ctx;
7537         u32 objtype;
7538
7539         if (!nla[NFTA_OBJ_TYPE] ||
7540             (!nla[NFTA_OBJ_NAME] && !nla[NFTA_OBJ_HANDLE]))
7541                 return -EINVAL;
7542
7543         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
7544                                  NETLINK_CB(skb).portid);
7545         if (IS_ERR(table)) {
7546                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7547                 return PTR_ERR(table);
7548         }
7549
7550         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7551         if (nla[NFTA_OBJ_HANDLE]) {
7552                 attr = nla[NFTA_OBJ_HANDLE];
7553                 obj = nft_obj_lookup_byhandle(table, attr, objtype, genmask);
7554         } else {
7555                 attr = nla[NFTA_OBJ_NAME];
7556                 obj = nft_obj_lookup(net, table, attr, objtype, genmask);
7557         }
7558
7559         if (IS_ERR(obj)) {
7560                 if (PTR_ERR(obj) == -ENOENT &&
7561                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYOBJ)
7562                         return 0;
7563
7564                 NL_SET_BAD_ATTR(extack, attr);
7565                 return PTR_ERR(obj);
7566         }
7567         if (obj->use > 0) {
7568                 NL_SET_BAD_ATTR(extack, attr);
7569                 return -EBUSY;
7570         }
7571
7572         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7573
7574         return nft_delobj(&ctx, obj);
7575 }
7576
7577 void nft_obj_notify(struct net *net, const struct nft_table *table,
7578                     struct nft_object *obj, u32 portid, u32 seq, int event,
7579                     u16 flags, int family, int report, gfp_t gfp)
7580 {
7581         struct nftables_pernet *nft_net = nft_pernet(net);
7582         struct sk_buff *skb;
7583         int err;
7584         char *buf = kasprintf(gfp, "%s:%u",
7585                               table->name, nft_net->base_seq);
7586
7587         audit_log_nfcfg(buf,
7588                         family,
7589                         obj->handle,
7590                         event == NFT_MSG_NEWOBJ ?
7591                                  AUDIT_NFT_OP_OBJ_REGISTER :
7592                                  AUDIT_NFT_OP_OBJ_UNREGISTER,
7593                         gfp);
7594         kfree(buf);
7595
7596         if (!report &&
7597             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
7598                 return;
7599
7600         skb = nlmsg_new(NLMSG_GOODSIZE, gfp);
7601         if (skb == NULL)
7602                 goto err;
7603
7604         err = nf_tables_fill_obj_info(skb, net, portid, seq, event,
7605                                       flags & (NLM_F_CREATE | NLM_F_EXCL),
7606                                       family, table, obj, false);
7607         if (err < 0) {
7608                 kfree_skb(skb);
7609                 goto err;
7610         }
7611
7612         nft_notify_enqueue(skb, report, &nft_net->notify_list);
7613         return;
7614 err:
7615         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
7616 }
7617 EXPORT_SYMBOL_GPL(nft_obj_notify);
7618
7619 static void nf_tables_obj_notify(const struct nft_ctx *ctx,
7620                                  struct nft_object *obj, int event)
7621 {
7622         nft_obj_notify(ctx->net, ctx->table, obj, ctx->portid, ctx->seq, event,
7623                        ctx->flags, ctx->family, ctx->report, GFP_KERNEL);
7624 }
7625
7626 /*
7627  * Flow tables
7628  */
7629 void nft_register_flowtable_type(struct nf_flowtable_type *type)
7630 {
7631         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7632         list_add_tail_rcu(&type->list, &nf_tables_flowtables);
7633         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7634 }
7635 EXPORT_SYMBOL_GPL(nft_register_flowtable_type);
7636
7637 void nft_unregister_flowtable_type(struct nf_flowtable_type *type)
7638 {
7639         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7640         list_del_rcu(&type->list);
7641         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7642 }
7643 EXPORT_SYMBOL_GPL(nft_unregister_flowtable_type);
7644
7645 static const struct nla_policy nft_flowtable_policy[NFTA_FLOWTABLE_MAX + 1] = {
7646         [NFTA_FLOWTABLE_TABLE]          = { .type = NLA_STRING,
7647                                             .len = NFT_NAME_MAXLEN - 1 },
7648         [NFTA_FLOWTABLE_NAME]           = { .type = NLA_STRING,
7649                                             .len = NFT_NAME_MAXLEN - 1 },
7650         [NFTA_FLOWTABLE_HOOK]           = { .type = NLA_NESTED },
7651         [NFTA_FLOWTABLE_HANDLE]         = { .type = NLA_U64 },
7652         [NFTA_FLOWTABLE_FLAGS]          = { .type = NLA_U32 },
7653 };
7654
7655 struct nft_flowtable *nft_flowtable_lookup(const struct nft_table *table,
7656                                            const struct nlattr *nla, u8 genmask)
7657 {
7658         struct nft_flowtable *flowtable;
7659
7660         list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
7661                 if (!nla_strcmp(nla, flowtable->name) &&
7662                     nft_active_genmask(flowtable, genmask))
7663                         return flowtable;
7664         }
7665         return ERR_PTR(-ENOENT);
7666 }
7667 EXPORT_SYMBOL_GPL(nft_flowtable_lookup);
7668
7669 void nf_tables_deactivate_flowtable(const struct nft_ctx *ctx,
7670                                     struct nft_flowtable *flowtable,
7671                                     enum nft_trans_phase phase)
7672 {
7673         switch (phase) {
7674         case NFT_TRANS_PREPARE:
7675         case NFT_TRANS_ABORT:
7676         case NFT_TRANS_RELEASE:
7677                 flowtable->use--;
7678                 fallthrough;
7679         default:
7680                 return;
7681         }
7682 }
7683 EXPORT_SYMBOL_GPL(nf_tables_deactivate_flowtable);
7684
7685 static struct nft_flowtable *
7686 nft_flowtable_lookup_byhandle(const struct nft_table *table,
7687                               const struct nlattr *nla, u8 genmask)
7688 {
7689        struct nft_flowtable *flowtable;
7690
7691        list_for_each_entry(flowtable, &table->flowtables, list) {
7692                if (be64_to_cpu(nla_get_be64(nla)) == flowtable->handle &&
7693                    nft_active_genmask(flowtable, genmask))
7694                        return flowtable;
7695        }
7696        return ERR_PTR(-ENOENT);
7697 }
7698
7699 struct nft_flowtable_hook {
7700         u32                     num;
7701         int                     priority;
7702         struct list_head        list;
7703 };
7704
7705 static const struct nla_policy nft_flowtable_hook_policy[NFTA_FLOWTABLE_HOOK_MAX + 1] = {
7706         [NFTA_FLOWTABLE_HOOK_NUM]       = { .type = NLA_U32 },
7707         [NFTA_FLOWTABLE_HOOK_PRIORITY]  = { .type = NLA_U32 },
7708         [NFTA_FLOWTABLE_HOOK_DEVS]      = { .type = NLA_NESTED },
7709 };
7710
7711 static int nft_flowtable_parse_hook(const struct nft_ctx *ctx,
7712                                     const struct nlattr * const nla[],
7713                                     struct nft_flowtable_hook *flowtable_hook,
7714                                     struct nft_flowtable *flowtable,
7715                                     struct netlink_ext_ack *extack, bool add)
7716 {
7717         struct nlattr *tb[NFTA_FLOWTABLE_HOOK_MAX + 1];
7718         struct nft_hook *hook;
7719         int hooknum, priority;
7720         int err;
7721
7722         INIT_LIST_HEAD(&flowtable_hook->list);
7723
7724         err = nla_parse_nested_deprecated(tb, NFTA_FLOWTABLE_HOOK_MAX,
7725                                           nla[NFTA_FLOWTABLE_HOOK],
7726                                           nft_flowtable_hook_policy, NULL);
7727         if (err < 0)
7728                 return err;
7729
7730         if (add) {
7731                 if (!tb[NFTA_FLOWTABLE_HOOK_NUM] ||
7732                     !tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
7733                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
7734                         return -ENOENT;
7735                 }
7736
7737                 hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
7738                 if (hooknum != NF_NETDEV_INGRESS)
7739                         return -EOPNOTSUPP;
7740
7741                 priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
7742
7743                 flowtable_hook->priority        = priority;
7744                 flowtable_hook->num             = hooknum;
7745         } else {
7746                 if (tb[NFTA_FLOWTABLE_HOOK_NUM]) {
7747                         hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
7748                         if (hooknum != flowtable->hooknum)
7749                                 return -EOPNOTSUPP;
7750                 }
7751
7752                 if (tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
7753                         priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
7754                         if (priority != flowtable->data.priority)
7755                                 return -EOPNOTSUPP;
7756                 }
7757
7758                 flowtable_hook->priority        = flowtable->data.priority;
7759                 flowtable_hook->num             = flowtable->hooknum;
7760         }
7761
7762         if (tb[NFTA_FLOWTABLE_HOOK_DEVS]) {
7763                 err = nf_tables_parse_netdev_hooks(ctx->net,
7764                                                    tb[NFTA_FLOWTABLE_HOOK_DEVS],
7765                                                    &flowtable_hook->list,
7766                                                    extack);
7767                 if (err < 0)
7768                         return err;
7769         }
7770
7771         list_for_each_entry(hook, &flowtable_hook->list, list) {
7772                 hook->ops.pf            = NFPROTO_NETDEV;
7773                 hook->ops.hooknum       = flowtable_hook->num;
7774                 hook->ops.priority      = flowtable_hook->priority;
7775                 hook->ops.priv          = &flowtable->data;
7776                 hook->ops.hook          = flowtable->data.type->hook;
7777         }
7778
7779         return err;
7780 }
7781
7782 static const struct nf_flowtable_type *__nft_flowtable_type_get(u8 family)
7783 {
7784         const struct nf_flowtable_type *type;
7785
7786         list_for_each_entry(type, &nf_tables_flowtables, list) {
7787                 if (family == type->family)
7788                         return type;
7789         }
7790         return NULL;
7791 }
7792
7793 static const struct nf_flowtable_type *
7794 nft_flowtable_type_get(struct net *net, u8 family)
7795 {
7796         const struct nf_flowtable_type *type;
7797
7798         type = __nft_flowtable_type_get(family);
7799         if (type != NULL && try_module_get(type->owner))
7800                 return type;
7801
7802         lockdep_nfnl_nft_mutex_not_held();
7803 #ifdef CONFIG_MODULES
7804         if (type == NULL) {
7805                 if (nft_request_module(net, "nf-flowtable-%u", family) == -EAGAIN)
7806                         return ERR_PTR(-EAGAIN);
7807         }
7808 #endif
7809         return ERR_PTR(-ENOENT);
7810 }
7811
7812 /* Only called from error and netdev event paths. */
7813 static void nft_unregister_flowtable_hook(struct net *net,
7814                                           struct nft_flowtable *flowtable,
7815                                           struct nft_hook *hook)
7816 {
7817         nf_unregister_net_hook(net, &hook->ops);
7818         flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
7819                                     FLOW_BLOCK_UNBIND);
7820 }
7821
7822 static void __nft_unregister_flowtable_net_hooks(struct net *net,
7823                                                  struct list_head *hook_list,
7824                                                  bool release_netdev)
7825 {
7826         struct nft_hook *hook, *next;
7827
7828         list_for_each_entry_safe(hook, next, hook_list, list) {
7829                 nf_unregister_net_hook(net, &hook->ops);
7830                 if (release_netdev) {
7831                         list_del(&hook->list);
7832                         kfree_rcu(hook, rcu);
7833                 }
7834         }
7835 }
7836
7837 static void nft_unregister_flowtable_net_hooks(struct net *net,
7838                                                struct list_head *hook_list)
7839 {
7840         __nft_unregister_flowtable_net_hooks(net, hook_list, false);
7841 }
7842
7843 static int nft_register_flowtable_net_hooks(struct net *net,
7844                                             struct nft_table *table,
7845                                             struct list_head *hook_list,
7846                                             struct nft_flowtable *flowtable)
7847 {
7848         struct nft_hook *hook, *hook2, *next;
7849         struct nft_flowtable *ft;
7850         int err, i = 0;
7851
7852         list_for_each_entry(hook, hook_list, list) {
7853                 list_for_each_entry(ft, &table->flowtables, list) {
7854                         if (!nft_is_active_next(net, ft))
7855                                 continue;
7856
7857                         list_for_each_entry(hook2, &ft->hook_list, list) {
7858                                 if (hook->ops.dev == hook2->ops.dev &&
7859                                     hook->ops.pf == hook2->ops.pf) {
7860                                         err = -EEXIST;
7861                                         goto err_unregister_net_hooks;
7862                                 }
7863                         }
7864                 }
7865
7866                 err = flowtable->data.type->setup(&flowtable->data,
7867                                                   hook->ops.dev,
7868                                                   FLOW_BLOCK_BIND);
7869                 if (err < 0)
7870                         goto err_unregister_net_hooks;
7871
7872                 err = nf_register_net_hook(net, &hook->ops);
7873                 if (err < 0) {
7874                         flowtable->data.type->setup(&flowtable->data,
7875                                                     hook->ops.dev,
7876                                                     FLOW_BLOCK_UNBIND);
7877                         goto err_unregister_net_hooks;
7878                 }
7879
7880                 i++;
7881         }
7882
7883         return 0;
7884
7885 err_unregister_net_hooks:
7886         list_for_each_entry_safe(hook, next, hook_list, list) {
7887                 if (i-- <= 0)
7888                         break;
7889
7890                 nft_unregister_flowtable_hook(net, flowtable, hook);
7891                 list_del_rcu(&hook->list);
7892                 kfree_rcu(hook, rcu);
7893         }
7894
7895         return err;
7896 }
7897
7898 static void nft_hooks_destroy(struct list_head *hook_list)
7899 {
7900         struct nft_hook *hook, *next;
7901
7902         list_for_each_entry_safe(hook, next, hook_list, list) {
7903                 list_del_rcu(&hook->list);
7904                 kfree_rcu(hook, rcu);
7905         }
7906 }
7907
7908 static int nft_flowtable_update(struct nft_ctx *ctx, const struct nlmsghdr *nlh,
7909                                 struct nft_flowtable *flowtable,
7910                                 struct netlink_ext_ack *extack)
7911 {
7912         const struct nlattr * const *nla = ctx->nla;
7913         struct nft_flowtable_hook flowtable_hook;
7914         struct nft_hook *hook, *next;
7915         struct nft_trans *trans;
7916         bool unregister = false;
7917         u32 flags;
7918         int err;
7919
7920         err = nft_flowtable_parse_hook(ctx, nla, &flowtable_hook, flowtable,
7921                                        extack, false);
7922         if (err < 0)
7923                 return err;
7924
7925         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
7926                 if (nft_hook_list_find(&flowtable->hook_list, hook)) {
7927                         list_del(&hook->list);
7928                         kfree(hook);
7929                 }
7930         }
7931
7932         if (nla[NFTA_FLOWTABLE_FLAGS]) {
7933                 flags = ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
7934                 if (flags & ~NFT_FLOWTABLE_MASK) {
7935                         err = -EOPNOTSUPP;
7936                         goto err_flowtable_update_hook;
7937                 }
7938                 if ((flowtable->data.flags & NFT_FLOWTABLE_HW_OFFLOAD) ^
7939                     (flags & NFT_FLOWTABLE_HW_OFFLOAD)) {
7940                         err = -EOPNOTSUPP;
7941                         goto err_flowtable_update_hook;
7942                 }
7943         } else {
7944                 flags = flowtable->data.flags;
7945         }
7946
7947         err = nft_register_flowtable_net_hooks(ctx->net, ctx->table,
7948                                                &flowtable_hook.list, flowtable);
7949         if (err < 0)
7950                 goto err_flowtable_update_hook;
7951
7952         trans = nft_trans_alloc(ctx, NFT_MSG_NEWFLOWTABLE,
7953                                 sizeof(struct nft_trans_flowtable));
7954         if (!trans) {
7955                 unregister = true;
7956                 err = -ENOMEM;
7957                 goto err_flowtable_update_hook;
7958         }
7959
7960         nft_trans_flowtable_flags(trans) = flags;
7961         nft_trans_flowtable(trans) = flowtable;
7962         nft_trans_flowtable_update(trans) = true;
7963         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
7964         list_splice(&flowtable_hook.list, &nft_trans_flowtable_hooks(trans));
7965
7966         nft_trans_commit_list_add_tail(ctx->net, trans);
7967
7968         return 0;
7969
7970 err_flowtable_update_hook:
7971         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
7972                 if (unregister)
7973                         nft_unregister_flowtable_hook(ctx->net, flowtable, hook);
7974                 list_del_rcu(&hook->list);
7975                 kfree_rcu(hook, rcu);
7976         }
7977
7978         return err;
7979
7980 }
7981
7982 static int nf_tables_newflowtable(struct sk_buff *skb,
7983                                   const struct nfnl_info *info,
7984                                   const struct nlattr * const nla[])
7985 {
7986         struct netlink_ext_ack *extack = info->extack;
7987         struct nft_flowtable_hook flowtable_hook;
7988         u8 genmask = nft_genmask_next(info->net);
7989         u8 family = info->nfmsg->nfgen_family;
7990         const struct nf_flowtable_type *type;
7991         struct nft_flowtable *flowtable;
7992         struct nft_hook *hook, *next;
7993         struct net *net = info->net;
7994         struct nft_table *table;
7995         struct nft_ctx ctx;
7996         int err;
7997
7998         if (!nla[NFTA_FLOWTABLE_TABLE] ||
7999             !nla[NFTA_FLOWTABLE_NAME] ||
8000             !nla[NFTA_FLOWTABLE_HOOK])
8001                 return -EINVAL;
8002
8003         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8004                                  genmask, NETLINK_CB(skb).portid);
8005         if (IS_ERR(table)) {
8006                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
8007                 return PTR_ERR(table);
8008         }
8009
8010         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
8011                                          genmask);
8012         if (IS_ERR(flowtable)) {
8013                 err = PTR_ERR(flowtable);
8014                 if (err != -ENOENT) {
8015                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8016                         return err;
8017                 }
8018         } else {
8019                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
8020                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8021                         return -EEXIST;
8022                 }
8023
8024                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8025
8026                 return nft_flowtable_update(&ctx, info->nlh, flowtable, extack);
8027         }
8028
8029         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8030
8031         flowtable = kzalloc(sizeof(*flowtable), GFP_KERNEL_ACCOUNT);
8032         if (!flowtable)
8033                 return -ENOMEM;
8034
8035         flowtable->table = table;
8036         flowtable->handle = nf_tables_alloc_handle(table);
8037         INIT_LIST_HEAD(&flowtable->hook_list);
8038
8039         flowtable->name = nla_strdup(nla[NFTA_FLOWTABLE_NAME], GFP_KERNEL_ACCOUNT);
8040         if (!flowtable->name) {
8041                 err = -ENOMEM;
8042                 goto err1;
8043         }
8044
8045         type = nft_flowtable_type_get(net, family);
8046         if (IS_ERR(type)) {
8047                 err = PTR_ERR(type);
8048                 goto err2;
8049         }
8050
8051         if (nla[NFTA_FLOWTABLE_FLAGS]) {
8052                 flowtable->data.flags =
8053                         ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
8054                 if (flowtable->data.flags & ~NFT_FLOWTABLE_MASK) {
8055                         err = -EOPNOTSUPP;
8056                         goto err3;
8057                 }
8058         }
8059
8060         write_pnet(&flowtable->data.net, net);
8061         flowtable->data.type = type;
8062         err = type->init(&flowtable->data);
8063         if (err < 0)
8064                 goto err3;
8065
8066         err = nft_flowtable_parse_hook(&ctx, nla, &flowtable_hook, flowtable,
8067                                        extack, true);
8068         if (err < 0)
8069                 goto err4;
8070
8071         list_splice(&flowtable_hook.list, &flowtable->hook_list);
8072         flowtable->data.priority = flowtable_hook.priority;
8073         flowtable->hooknum = flowtable_hook.num;
8074
8075         err = nft_register_flowtable_net_hooks(ctx.net, table,
8076                                                &flowtable->hook_list,
8077                                                flowtable);
8078         if (err < 0) {
8079                 nft_hooks_destroy(&flowtable->hook_list);
8080                 goto err4;
8081         }
8082
8083         err = nft_trans_flowtable_add(&ctx, NFT_MSG_NEWFLOWTABLE, flowtable);
8084         if (err < 0)
8085                 goto err5;
8086
8087         list_add_tail_rcu(&flowtable->list, &table->flowtables);
8088         table->use++;
8089
8090         return 0;
8091 err5:
8092         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
8093                 nft_unregister_flowtable_hook(net, flowtable, hook);
8094                 list_del_rcu(&hook->list);
8095                 kfree_rcu(hook, rcu);
8096         }
8097 err4:
8098         flowtable->data.type->free(&flowtable->data);
8099 err3:
8100         module_put(type->owner);
8101 err2:
8102         kfree(flowtable->name);
8103 err1:
8104         kfree(flowtable);
8105         return err;
8106 }
8107
8108 static void nft_flowtable_hook_release(struct nft_flowtable_hook *flowtable_hook)
8109 {
8110         struct nft_hook *this, *next;
8111
8112         list_for_each_entry_safe(this, next, &flowtable_hook->list, list) {
8113                 list_del(&this->list);
8114                 kfree(this);
8115         }
8116 }
8117
8118 static int nft_delflowtable_hook(struct nft_ctx *ctx,
8119                                  struct nft_flowtable *flowtable,
8120                                  struct netlink_ext_ack *extack)
8121 {
8122         const struct nlattr * const *nla = ctx->nla;
8123         struct nft_flowtable_hook flowtable_hook;
8124         LIST_HEAD(flowtable_del_list);
8125         struct nft_hook *this, *hook;
8126         struct nft_trans *trans;
8127         int err;
8128
8129         err = nft_flowtable_parse_hook(ctx, nla, &flowtable_hook, flowtable,
8130                                        extack, false);
8131         if (err < 0)
8132                 return err;
8133
8134         list_for_each_entry(this, &flowtable_hook.list, list) {
8135                 hook = nft_hook_list_find(&flowtable->hook_list, this);
8136                 if (!hook) {
8137                         err = -ENOENT;
8138                         goto err_flowtable_del_hook;
8139                 }
8140                 list_move(&hook->list, &flowtable_del_list);
8141         }
8142
8143         trans = nft_trans_alloc(ctx, NFT_MSG_DELFLOWTABLE,
8144                                 sizeof(struct nft_trans_flowtable));
8145         if (!trans) {
8146                 err = -ENOMEM;
8147                 goto err_flowtable_del_hook;
8148         }
8149
8150         nft_trans_flowtable(trans) = flowtable;
8151         nft_trans_flowtable_update(trans) = true;
8152         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
8153         list_splice(&flowtable_del_list, &nft_trans_flowtable_hooks(trans));
8154         nft_flowtable_hook_release(&flowtable_hook);
8155
8156         nft_trans_commit_list_add_tail(ctx->net, trans);
8157
8158         return 0;
8159
8160 err_flowtable_del_hook:
8161         list_splice(&flowtable_del_list, &flowtable->hook_list);
8162         nft_flowtable_hook_release(&flowtable_hook);
8163
8164         return err;
8165 }
8166
8167 static int nf_tables_delflowtable(struct sk_buff *skb,
8168                                   const struct nfnl_info *info,
8169                                   const struct nlattr * const nla[])
8170 {
8171         struct netlink_ext_ack *extack = info->extack;
8172         u8 genmask = nft_genmask_next(info->net);
8173         u8 family = info->nfmsg->nfgen_family;
8174         struct nft_flowtable *flowtable;
8175         struct net *net = info->net;
8176         const struct nlattr *attr;
8177         struct nft_table *table;
8178         struct nft_ctx ctx;
8179
8180         if (!nla[NFTA_FLOWTABLE_TABLE] ||
8181             (!nla[NFTA_FLOWTABLE_NAME] &&
8182              !nla[NFTA_FLOWTABLE_HANDLE]))
8183                 return -EINVAL;
8184
8185         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8186                                  genmask, NETLINK_CB(skb).portid);
8187         if (IS_ERR(table)) {
8188                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
8189                 return PTR_ERR(table);
8190         }
8191
8192         if (nla[NFTA_FLOWTABLE_HANDLE]) {
8193                 attr = nla[NFTA_FLOWTABLE_HANDLE];
8194                 flowtable = nft_flowtable_lookup_byhandle(table, attr, genmask);
8195         } else {
8196                 attr = nla[NFTA_FLOWTABLE_NAME];
8197                 flowtable = nft_flowtable_lookup(table, attr, genmask);
8198         }
8199
8200         if (IS_ERR(flowtable)) {
8201                 if (PTR_ERR(flowtable) == -ENOENT &&
8202                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYFLOWTABLE)
8203                         return 0;
8204
8205                 NL_SET_BAD_ATTR(extack, attr);
8206                 return PTR_ERR(flowtable);
8207         }
8208
8209         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8210
8211         if (nla[NFTA_FLOWTABLE_HOOK])
8212                 return nft_delflowtable_hook(&ctx, flowtable, extack);
8213
8214         if (flowtable->use > 0) {
8215                 NL_SET_BAD_ATTR(extack, attr);
8216                 return -EBUSY;
8217         }
8218
8219         return nft_delflowtable(&ctx, flowtable);
8220 }
8221
8222 static int nf_tables_fill_flowtable_info(struct sk_buff *skb, struct net *net,
8223                                          u32 portid, u32 seq, int event,
8224                                          u32 flags, int family,
8225                                          struct nft_flowtable *flowtable,
8226                                          struct list_head *hook_list)
8227 {
8228         struct nlattr *nest, *nest_devs;
8229         struct nft_hook *hook;
8230         struct nlmsghdr *nlh;
8231
8232         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
8233         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
8234                            NFNETLINK_V0, nft_base_seq(net));
8235         if (!nlh)
8236                 goto nla_put_failure;
8237
8238         if (nla_put_string(skb, NFTA_FLOWTABLE_TABLE, flowtable->table->name) ||
8239             nla_put_string(skb, NFTA_FLOWTABLE_NAME, flowtable->name) ||
8240             nla_put_be64(skb, NFTA_FLOWTABLE_HANDLE, cpu_to_be64(flowtable->handle),
8241                          NFTA_FLOWTABLE_PAD))
8242                 goto nla_put_failure;
8243
8244         if (event == NFT_MSG_DELFLOWTABLE && !hook_list) {
8245                 nlmsg_end(skb, nlh);
8246                 return 0;
8247         }
8248
8249         if (nla_put_be32(skb, NFTA_FLOWTABLE_USE, htonl(flowtable->use)) ||
8250             nla_put_be32(skb, NFTA_FLOWTABLE_FLAGS, htonl(flowtable->data.flags)))
8251                 goto nla_put_failure;
8252
8253         nest = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK);
8254         if (!nest)
8255                 goto nla_put_failure;
8256         if (nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_NUM, htonl(flowtable->hooknum)) ||
8257             nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_PRIORITY, htonl(flowtable->data.priority)))
8258                 goto nla_put_failure;
8259
8260         nest_devs = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK_DEVS);
8261         if (!nest_devs)
8262                 goto nla_put_failure;
8263
8264         if (!hook_list)
8265                 hook_list = &flowtable->hook_list;
8266
8267         list_for_each_entry_rcu(hook, hook_list, list) {
8268                 if (nla_put_string(skb, NFTA_DEVICE_NAME, hook->ops.dev->name))
8269                         goto nla_put_failure;
8270         }
8271         nla_nest_end(skb, nest_devs);
8272         nla_nest_end(skb, nest);
8273
8274         nlmsg_end(skb, nlh);
8275         return 0;
8276
8277 nla_put_failure:
8278         nlmsg_trim(skb, nlh);
8279         return -1;
8280 }
8281
8282 struct nft_flowtable_filter {
8283         char            *table;
8284 };
8285
8286 static int nf_tables_dump_flowtable(struct sk_buff *skb,
8287                                     struct netlink_callback *cb)
8288 {
8289         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
8290         struct nft_flowtable_filter *filter = cb->data;
8291         unsigned int idx = 0, s_idx = cb->args[0];
8292         struct net *net = sock_net(skb->sk);
8293         int family = nfmsg->nfgen_family;
8294         struct nft_flowtable *flowtable;
8295         struct nftables_pernet *nft_net;
8296         const struct nft_table *table;
8297
8298         rcu_read_lock();
8299         nft_net = nft_pernet(net);
8300         cb->seq = READ_ONCE(nft_net->base_seq);
8301
8302         list_for_each_entry_rcu(table, &nft_net->tables, list) {
8303                 if (family != NFPROTO_UNSPEC && family != table->family)
8304                         continue;
8305
8306                 list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
8307                         if (!nft_is_active(net, flowtable))
8308                                 goto cont;
8309                         if (idx < s_idx)
8310                                 goto cont;
8311                         if (idx > s_idx)
8312                                 memset(&cb->args[1], 0,
8313                                        sizeof(cb->args) - sizeof(cb->args[0]));
8314                         if (filter && filter->table &&
8315                             strcmp(filter->table, table->name))
8316                                 goto cont;
8317
8318                         if (nf_tables_fill_flowtable_info(skb, net, NETLINK_CB(cb->skb).portid,
8319                                                           cb->nlh->nlmsg_seq,
8320                                                           NFT_MSG_NEWFLOWTABLE,
8321                                                           NLM_F_MULTI | NLM_F_APPEND,
8322                                                           table->family,
8323                                                           flowtable, NULL) < 0)
8324                                 goto done;
8325
8326                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
8327 cont:
8328                         idx++;
8329                 }
8330         }
8331 done:
8332         rcu_read_unlock();
8333
8334         cb->args[0] = idx;
8335         return skb->len;
8336 }
8337
8338 static int nf_tables_dump_flowtable_start(struct netlink_callback *cb)
8339 {
8340         const struct nlattr * const *nla = cb->data;
8341         struct nft_flowtable_filter *filter = NULL;
8342
8343         if (nla[NFTA_FLOWTABLE_TABLE]) {
8344                 filter = kzalloc(sizeof(*filter), GFP_ATOMIC);
8345                 if (!filter)
8346                         return -ENOMEM;
8347
8348                 filter->table = nla_strdup(nla[NFTA_FLOWTABLE_TABLE],
8349                                            GFP_ATOMIC);
8350                 if (!filter->table) {
8351                         kfree(filter);
8352                         return -ENOMEM;
8353                 }
8354         }
8355
8356         cb->data = filter;
8357         return 0;
8358 }
8359
8360 static int nf_tables_dump_flowtable_done(struct netlink_callback *cb)
8361 {
8362         struct nft_flowtable_filter *filter = cb->data;
8363
8364         if (!filter)
8365                 return 0;
8366
8367         kfree(filter->table);
8368         kfree(filter);
8369
8370         return 0;
8371 }
8372
8373 /* called with rcu_read_lock held */
8374 static int nf_tables_getflowtable(struct sk_buff *skb,
8375                                   const struct nfnl_info *info,
8376                                   const struct nlattr * const nla[])
8377 {
8378         u8 genmask = nft_genmask_cur(info->net);
8379         u8 family = info->nfmsg->nfgen_family;
8380         struct nft_flowtable *flowtable;
8381         const struct nft_table *table;
8382         struct net *net = info->net;
8383         struct sk_buff *skb2;
8384         int err;
8385
8386         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
8387                 struct netlink_dump_control c = {
8388                         .start = nf_tables_dump_flowtable_start,
8389                         .dump = nf_tables_dump_flowtable,
8390                         .done = nf_tables_dump_flowtable_done,
8391                         .module = THIS_MODULE,
8392                         .data = (void *)nla,
8393                 };
8394
8395                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
8396         }
8397
8398         if (!nla[NFTA_FLOWTABLE_NAME])
8399                 return -EINVAL;
8400
8401         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8402                                  genmask, 0);
8403         if (IS_ERR(table))
8404                 return PTR_ERR(table);
8405
8406         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
8407                                          genmask);
8408         if (IS_ERR(flowtable))
8409                 return PTR_ERR(flowtable);
8410
8411         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
8412         if (!skb2)
8413                 return -ENOMEM;
8414
8415         err = nf_tables_fill_flowtable_info(skb2, net, NETLINK_CB(skb).portid,
8416                                             info->nlh->nlmsg_seq,
8417                                             NFT_MSG_NEWFLOWTABLE, 0, family,
8418                                             flowtable, NULL);
8419         if (err < 0)
8420                 goto err_fill_flowtable_info;
8421
8422         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
8423
8424 err_fill_flowtable_info:
8425         kfree_skb(skb2);
8426         return err;
8427 }
8428
8429 static void nf_tables_flowtable_notify(struct nft_ctx *ctx,
8430                                        struct nft_flowtable *flowtable,
8431                                        struct list_head *hook_list, int event)
8432 {
8433         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
8434         struct sk_buff *skb;
8435         u16 flags = 0;
8436         int err;
8437
8438         if (!ctx->report &&
8439             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
8440                 return;
8441
8442         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
8443         if (skb == NULL)
8444                 goto err;
8445
8446         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
8447                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
8448
8449         err = nf_tables_fill_flowtable_info(skb, ctx->net, ctx->portid,
8450                                             ctx->seq, event, flags,
8451                                             ctx->family, flowtable, hook_list);
8452         if (err < 0) {
8453                 kfree_skb(skb);
8454                 goto err;
8455         }
8456
8457         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
8458         return;
8459 err:
8460         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
8461 }
8462
8463 static void nf_tables_flowtable_destroy(struct nft_flowtable *flowtable)
8464 {
8465         struct nft_hook *hook, *next;
8466
8467         flowtable->data.type->free(&flowtable->data);
8468         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
8469                 flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
8470                                             FLOW_BLOCK_UNBIND);
8471                 list_del_rcu(&hook->list);
8472                 kfree(hook);
8473         }
8474         kfree(flowtable->name);
8475         module_put(flowtable->data.type->owner);
8476         kfree(flowtable);
8477 }
8478
8479 static int nf_tables_fill_gen_info(struct sk_buff *skb, struct net *net,
8480                                    u32 portid, u32 seq)
8481 {
8482         struct nftables_pernet *nft_net = nft_pernet(net);
8483         struct nlmsghdr *nlh;
8484         char buf[TASK_COMM_LEN];
8485         int event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWGEN);
8486
8487         nlh = nfnl_msg_put(skb, portid, seq, event, 0, AF_UNSPEC,
8488                            NFNETLINK_V0, nft_base_seq(net));
8489         if (!nlh)
8490                 goto nla_put_failure;
8491
8492         if (nla_put_be32(skb, NFTA_GEN_ID, htonl(nft_net->base_seq)) ||
8493             nla_put_be32(skb, NFTA_GEN_PROC_PID, htonl(task_pid_nr(current))) ||
8494             nla_put_string(skb, NFTA_GEN_PROC_NAME, get_task_comm(buf, current)))
8495                 goto nla_put_failure;
8496
8497         nlmsg_end(skb, nlh);
8498         return 0;
8499
8500 nla_put_failure:
8501         nlmsg_trim(skb, nlh);
8502         return -EMSGSIZE;
8503 }
8504
8505 static void nft_flowtable_event(unsigned long event, struct net_device *dev,
8506                                 struct nft_flowtable *flowtable)
8507 {
8508         struct nft_hook *hook;
8509
8510         list_for_each_entry(hook, &flowtable->hook_list, list) {
8511                 if (hook->ops.dev != dev)
8512                         continue;
8513
8514                 /* flow_offload_netdev_event() cleans up entries for us. */
8515                 nft_unregister_flowtable_hook(dev_net(dev), flowtable, hook);
8516                 list_del_rcu(&hook->list);
8517                 kfree_rcu(hook, rcu);
8518                 break;
8519         }
8520 }
8521
8522 static int nf_tables_flowtable_event(struct notifier_block *this,
8523                                      unsigned long event, void *ptr)
8524 {
8525         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
8526         struct nft_flowtable *flowtable;
8527         struct nftables_pernet *nft_net;
8528         struct nft_table *table;
8529         struct net *net;
8530
8531         if (event != NETDEV_UNREGISTER)
8532                 return 0;
8533
8534         net = dev_net(dev);
8535         nft_net = nft_pernet(net);
8536         mutex_lock(&nft_net->commit_mutex);
8537         list_for_each_entry(table, &nft_net->tables, list) {
8538                 list_for_each_entry(flowtable, &table->flowtables, list) {
8539                         nft_flowtable_event(event, dev, flowtable);
8540                 }
8541         }
8542         mutex_unlock(&nft_net->commit_mutex);
8543
8544         return NOTIFY_DONE;
8545 }
8546
8547 static struct notifier_block nf_tables_flowtable_notifier = {
8548         .notifier_call  = nf_tables_flowtable_event,
8549 };
8550
8551 static void nf_tables_gen_notify(struct net *net, struct sk_buff *skb,
8552                                  int event)
8553 {
8554         struct nlmsghdr *nlh = nlmsg_hdr(skb);
8555         struct sk_buff *skb2;
8556         int err;
8557
8558         if (!nlmsg_report(nlh) &&
8559             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
8560                 return;
8561
8562         skb2 = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
8563         if (skb2 == NULL)
8564                 goto err;
8565
8566         err = nf_tables_fill_gen_info(skb2, net, NETLINK_CB(skb).portid,
8567                                       nlh->nlmsg_seq);
8568         if (err < 0) {
8569                 kfree_skb(skb2);
8570                 goto err;
8571         }
8572
8573         nfnetlink_send(skb2, net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
8574                        nlmsg_report(nlh), GFP_KERNEL);
8575         return;
8576 err:
8577         nfnetlink_set_err(net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
8578                           -ENOBUFS);
8579 }
8580
8581 static int nf_tables_getgen(struct sk_buff *skb, const struct nfnl_info *info,
8582                             const struct nlattr * const nla[])
8583 {
8584         struct sk_buff *skb2;
8585         int err;
8586
8587         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
8588         if (skb2 == NULL)
8589                 return -ENOMEM;
8590
8591         err = nf_tables_fill_gen_info(skb2, info->net, NETLINK_CB(skb).portid,
8592                                       info->nlh->nlmsg_seq);
8593         if (err < 0)
8594                 goto err_fill_gen_info;
8595
8596         return nfnetlink_unicast(skb2, info->net, NETLINK_CB(skb).portid);
8597
8598 err_fill_gen_info:
8599         kfree_skb(skb2);
8600         return err;
8601 }
8602
8603 static const struct nfnl_callback nf_tables_cb[NFT_MSG_MAX] = {
8604         [NFT_MSG_NEWTABLE] = {
8605                 .call           = nf_tables_newtable,
8606                 .type           = NFNL_CB_BATCH,
8607                 .attr_count     = NFTA_TABLE_MAX,
8608                 .policy         = nft_table_policy,
8609         },
8610         [NFT_MSG_GETTABLE] = {
8611                 .call           = nf_tables_gettable,
8612                 .type           = NFNL_CB_RCU,
8613                 .attr_count     = NFTA_TABLE_MAX,
8614                 .policy         = nft_table_policy,
8615         },
8616         [NFT_MSG_DELTABLE] = {
8617                 .call           = nf_tables_deltable,
8618                 .type           = NFNL_CB_BATCH,
8619                 .attr_count     = NFTA_TABLE_MAX,
8620                 .policy         = nft_table_policy,
8621         },
8622         [NFT_MSG_DESTROYTABLE] = {
8623                 .call           = nf_tables_deltable,
8624                 .type           = NFNL_CB_BATCH,
8625                 .attr_count     = NFTA_TABLE_MAX,
8626                 .policy         = nft_table_policy,
8627         },
8628         [NFT_MSG_NEWCHAIN] = {
8629                 .call           = nf_tables_newchain,
8630                 .type           = NFNL_CB_BATCH,
8631                 .attr_count     = NFTA_CHAIN_MAX,
8632                 .policy         = nft_chain_policy,
8633         },
8634         [NFT_MSG_GETCHAIN] = {
8635                 .call           = nf_tables_getchain,
8636                 .type           = NFNL_CB_RCU,
8637                 .attr_count     = NFTA_CHAIN_MAX,
8638                 .policy         = nft_chain_policy,
8639         },
8640         [NFT_MSG_DELCHAIN] = {
8641                 .call           = nf_tables_delchain,
8642                 .type           = NFNL_CB_BATCH,
8643                 .attr_count     = NFTA_CHAIN_MAX,
8644                 .policy         = nft_chain_policy,
8645         },
8646         [NFT_MSG_DESTROYCHAIN] = {
8647                 .call           = nf_tables_delchain,
8648                 .type           = NFNL_CB_BATCH,
8649                 .attr_count     = NFTA_CHAIN_MAX,
8650                 .policy         = nft_chain_policy,
8651         },
8652         [NFT_MSG_NEWRULE] = {
8653                 .call           = nf_tables_newrule,
8654                 .type           = NFNL_CB_BATCH,
8655                 .attr_count     = NFTA_RULE_MAX,
8656                 .policy         = nft_rule_policy,
8657         },
8658         [NFT_MSG_GETRULE] = {
8659                 .call           = nf_tables_getrule,
8660                 .type           = NFNL_CB_RCU,
8661                 .attr_count     = NFTA_RULE_MAX,
8662                 .policy         = nft_rule_policy,
8663         },
8664         [NFT_MSG_GETRULE_RESET] = {
8665                 .call           = nf_tables_getrule,
8666                 .type           = NFNL_CB_RCU,
8667                 .attr_count     = NFTA_RULE_MAX,
8668                 .policy         = nft_rule_policy,
8669         },
8670         [NFT_MSG_DELRULE] = {
8671                 .call           = nf_tables_delrule,
8672                 .type           = NFNL_CB_BATCH,
8673                 .attr_count     = NFTA_RULE_MAX,
8674                 .policy         = nft_rule_policy,
8675         },
8676         [NFT_MSG_DESTROYRULE] = {
8677                 .call           = nf_tables_delrule,
8678                 .type           = NFNL_CB_BATCH,
8679                 .attr_count     = NFTA_RULE_MAX,
8680                 .policy         = nft_rule_policy,
8681         },
8682         [NFT_MSG_NEWSET] = {
8683                 .call           = nf_tables_newset,
8684                 .type           = NFNL_CB_BATCH,
8685                 .attr_count     = NFTA_SET_MAX,
8686                 .policy         = nft_set_policy,
8687         },
8688         [NFT_MSG_GETSET] = {
8689                 .call           = nf_tables_getset,
8690                 .type           = NFNL_CB_RCU,
8691                 .attr_count     = NFTA_SET_MAX,
8692                 .policy         = nft_set_policy,
8693         },
8694         [NFT_MSG_DELSET] = {
8695                 .call           = nf_tables_delset,
8696                 .type           = NFNL_CB_BATCH,
8697                 .attr_count     = NFTA_SET_MAX,
8698                 .policy         = nft_set_policy,
8699         },
8700         [NFT_MSG_DESTROYSET] = {
8701                 .call           = nf_tables_delset,
8702                 .type           = NFNL_CB_BATCH,
8703                 .attr_count     = NFTA_SET_MAX,
8704                 .policy         = nft_set_policy,
8705         },
8706         [NFT_MSG_NEWSETELEM] = {
8707                 .call           = nf_tables_newsetelem,
8708                 .type           = NFNL_CB_BATCH,
8709                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
8710                 .policy         = nft_set_elem_list_policy,
8711         },
8712         [NFT_MSG_GETSETELEM] = {
8713                 .call           = nf_tables_getsetelem,
8714                 .type           = NFNL_CB_RCU,
8715                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
8716                 .policy         = nft_set_elem_list_policy,
8717         },
8718         [NFT_MSG_DELSETELEM] = {
8719                 .call           = nf_tables_delsetelem,
8720                 .type           = NFNL_CB_BATCH,
8721                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
8722                 .policy         = nft_set_elem_list_policy,
8723         },
8724         [NFT_MSG_DESTROYSETELEM] = {
8725                 .call           = nf_tables_delsetelem,
8726                 .type           = NFNL_CB_BATCH,
8727                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
8728                 .policy         = nft_set_elem_list_policy,
8729         },
8730         [NFT_MSG_GETGEN] = {
8731                 .call           = nf_tables_getgen,
8732                 .type           = NFNL_CB_RCU,
8733         },
8734         [NFT_MSG_NEWOBJ] = {
8735                 .call           = nf_tables_newobj,
8736                 .type           = NFNL_CB_BATCH,
8737                 .attr_count     = NFTA_OBJ_MAX,
8738                 .policy         = nft_obj_policy,
8739         },
8740         [NFT_MSG_GETOBJ] = {
8741                 .call           = nf_tables_getobj,
8742                 .type           = NFNL_CB_RCU,
8743                 .attr_count     = NFTA_OBJ_MAX,
8744                 .policy         = nft_obj_policy,
8745         },
8746         [NFT_MSG_DELOBJ] = {
8747                 .call           = nf_tables_delobj,
8748                 .type           = NFNL_CB_BATCH,
8749                 .attr_count     = NFTA_OBJ_MAX,
8750                 .policy         = nft_obj_policy,
8751         },
8752         [NFT_MSG_DESTROYOBJ] = {
8753                 .call           = nf_tables_delobj,
8754                 .type           = NFNL_CB_BATCH,
8755                 .attr_count     = NFTA_OBJ_MAX,
8756                 .policy         = nft_obj_policy,
8757         },
8758         [NFT_MSG_GETOBJ_RESET] = {
8759                 .call           = nf_tables_getobj,
8760                 .type           = NFNL_CB_RCU,
8761                 .attr_count     = NFTA_OBJ_MAX,
8762                 .policy         = nft_obj_policy,
8763         },
8764         [NFT_MSG_NEWFLOWTABLE] = {
8765                 .call           = nf_tables_newflowtable,
8766                 .type           = NFNL_CB_BATCH,
8767                 .attr_count     = NFTA_FLOWTABLE_MAX,
8768                 .policy         = nft_flowtable_policy,
8769         },
8770         [NFT_MSG_GETFLOWTABLE] = {
8771                 .call           = nf_tables_getflowtable,
8772                 .type           = NFNL_CB_RCU,
8773                 .attr_count     = NFTA_FLOWTABLE_MAX,
8774                 .policy         = nft_flowtable_policy,
8775         },
8776         [NFT_MSG_DELFLOWTABLE] = {
8777                 .call           = nf_tables_delflowtable,
8778                 .type           = NFNL_CB_BATCH,
8779                 .attr_count     = NFTA_FLOWTABLE_MAX,
8780                 .policy         = nft_flowtable_policy,
8781         },
8782         [NFT_MSG_DESTROYFLOWTABLE] = {
8783                 .call           = nf_tables_delflowtable,
8784                 .type           = NFNL_CB_BATCH,
8785                 .attr_count     = NFTA_FLOWTABLE_MAX,
8786                 .policy         = nft_flowtable_policy,
8787         },
8788 };
8789
8790 static int nf_tables_validate(struct net *net)
8791 {
8792         struct nftables_pernet *nft_net = nft_pernet(net);
8793         struct nft_table *table;
8794
8795         list_for_each_entry(table, &nft_net->tables, list) {
8796                 switch (table->validate_state) {
8797                 case NFT_VALIDATE_SKIP:
8798                         continue;
8799                 case NFT_VALIDATE_NEED:
8800                         nft_validate_state_update(table, NFT_VALIDATE_DO);
8801                         fallthrough;
8802                 case NFT_VALIDATE_DO:
8803                         if (nft_table_validate(net, table) < 0)
8804                                 return -EAGAIN;
8805
8806                         nft_validate_state_update(table, NFT_VALIDATE_SKIP);
8807                 }
8808
8809                 break;
8810         }
8811
8812         return 0;
8813 }
8814
8815 /* a drop policy has to be deferred until all rules have been activated,
8816  * otherwise a large ruleset that contains a drop-policy base chain will
8817  * cause all packets to get dropped until the full transaction has been
8818  * processed.
8819  *
8820  * We defer the drop policy until the transaction has been finalized.
8821  */
8822 static void nft_chain_commit_drop_policy(struct nft_trans *trans)
8823 {
8824         struct nft_base_chain *basechain;
8825
8826         if (nft_trans_chain_policy(trans) != NF_DROP)
8827                 return;
8828
8829         if (!nft_is_base_chain(trans->ctx.chain))
8830                 return;
8831
8832         basechain = nft_base_chain(trans->ctx.chain);
8833         basechain->policy = NF_DROP;
8834 }
8835
8836 static void nft_chain_commit_update(struct nft_trans *trans)
8837 {
8838         struct nft_base_chain *basechain;
8839
8840         if (nft_trans_chain_name(trans)) {
8841                 rhltable_remove(&trans->ctx.table->chains_ht,
8842                                 &trans->ctx.chain->rhlhead,
8843                                 nft_chain_ht_params);
8844                 swap(trans->ctx.chain->name, nft_trans_chain_name(trans));
8845                 rhltable_insert_key(&trans->ctx.table->chains_ht,
8846                                     trans->ctx.chain->name,
8847                                     &trans->ctx.chain->rhlhead,
8848                                     nft_chain_ht_params);
8849         }
8850
8851         if (!nft_is_base_chain(trans->ctx.chain))
8852                 return;
8853
8854         nft_chain_stats_replace(trans);
8855
8856         basechain = nft_base_chain(trans->ctx.chain);
8857
8858         switch (nft_trans_chain_policy(trans)) {
8859         case NF_DROP:
8860         case NF_ACCEPT:
8861                 basechain->policy = nft_trans_chain_policy(trans);
8862                 break;
8863         }
8864 }
8865
8866 static void nft_obj_commit_update(struct nft_trans *trans)
8867 {
8868         struct nft_object *newobj;
8869         struct nft_object *obj;
8870
8871         obj = nft_trans_obj(trans);
8872         newobj = nft_trans_obj_newobj(trans);
8873
8874         if (obj->ops->update)
8875                 obj->ops->update(obj, newobj);
8876
8877         nft_obj_destroy(&trans->ctx, newobj);
8878 }
8879
8880 static void nft_commit_release(struct nft_trans *trans)
8881 {
8882         switch (trans->msg_type) {
8883         case NFT_MSG_DELTABLE:
8884         case NFT_MSG_DESTROYTABLE:
8885                 nf_tables_table_destroy(&trans->ctx);
8886                 break;
8887         case NFT_MSG_NEWCHAIN:
8888                 free_percpu(nft_trans_chain_stats(trans));
8889                 kfree(nft_trans_chain_name(trans));
8890                 break;
8891         case NFT_MSG_DELCHAIN:
8892         case NFT_MSG_DESTROYCHAIN:
8893                 if (nft_trans_chain_update(trans))
8894                         nft_hooks_destroy(&nft_trans_chain_hooks(trans));
8895                 else
8896                         nf_tables_chain_destroy(&trans->ctx);
8897                 break;
8898         case NFT_MSG_DELRULE:
8899         case NFT_MSG_DESTROYRULE:
8900                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
8901                 break;
8902         case NFT_MSG_DELSET:
8903         case NFT_MSG_DESTROYSET:
8904                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
8905                 break;
8906         case NFT_MSG_DELSETELEM:
8907         case NFT_MSG_DESTROYSETELEM:
8908                 nf_tables_set_elem_destroy(&trans->ctx,
8909                                            nft_trans_elem_set(trans),
8910                                            nft_trans_elem(trans).priv);
8911                 break;
8912         case NFT_MSG_DELOBJ:
8913         case NFT_MSG_DESTROYOBJ:
8914                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
8915                 break;
8916         case NFT_MSG_DELFLOWTABLE:
8917         case NFT_MSG_DESTROYFLOWTABLE:
8918                 if (nft_trans_flowtable_update(trans))
8919                         nft_hooks_destroy(&nft_trans_flowtable_hooks(trans));
8920                 else
8921                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
8922                 break;
8923         }
8924
8925         if (trans->put_net)
8926                 put_net(trans->ctx.net);
8927
8928         kfree(trans);
8929 }
8930
8931 static void nf_tables_trans_destroy_work(struct work_struct *w)
8932 {
8933         struct nft_trans *trans, *next;
8934         LIST_HEAD(head);
8935
8936         spin_lock(&nf_tables_destroy_list_lock);
8937         list_splice_init(&nf_tables_destroy_list, &head);
8938         spin_unlock(&nf_tables_destroy_list_lock);
8939
8940         if (list_empty(&head))
8941                 return;
8942
8943         synchronize_rcu();
8944
8945         list_for_each_entry_safe(trans, next, &head, list) {
8946                 list_del(&trans->list);
8947                 nft_commit_release(trans);
8948         }
8949 }
8950
8951 void nf_tables_trans_destroy_flush_work(void)
8952 {
8953         flush_work(&trans_destroy_work);
8954 }
8955 EXPORT_SYMBOL_GPL(nf_tables_trans_destroy_flush_work);
8956
8957 static bool nft_expr_reduce(struct nft_regs_track *track,
8958                             const struct nft_expr *expr)
8959 {
8960         return false;
8961 }
8962
8963 static int nf_tables_commit_chain_prepare(struct net *net, struct nft_chain *chain)
8964 {
8965         const struct nft_expr *expr, *last;
8966         struct nft_regs_track track = {};
8967         unsigned int size, data_size;
8968         void *data, *data_boundary;
8969         struct nft_rule_dp *prule;
8970         struct nft_rule *rule;
8971
8972         /* already handled or inactive chain? */
8973         if (chain->blob_next || !nft_is_active_next(net, chain))
8974                 return 0;
8975
8976         data_size = 0;
8977         list_for_each_entry(rule, &chain->rules, list) {
8978                 if (nft_is_active_next(net, rule)) {
8979                         data_size += sizeof(*prule) + rule->dlen;
8980                         if (data_size > INT_MAX)
8981                                 return -ENOMEM;
8982                 }
8983         }
8984
8985         chain->blob_next = nf_tables_chain_alloc_rules(chain, data_size);
8986         if (!chain->blob_next)
8987                 return -ENOMEM;
8988
8989         data = (void *)chain->blob_next->data;
8990         data_boundary = data + data_size;
8991         size = 0;
8992
8993         list_for_each_entry(rule, &chain->rules, list) {
8994                 if (!nft_is_active_next(net, rule))
8995                         continue;
8996
8997                 prule = (struct nft_rule_dp *)data;
8998                 data += offsetof(struct nft_rule_dp, data);
8999                 if (WARN_ON_ONCE(data > data_boundary))
9000                         return -ENOMEM;
9001
9002                 size = 0;
9003                 track.last = nft_expr_last(rule);
9004                 nft_rule_for_each_expr(expr, last, rule) {
9005                         track.cur = expr;
9006
9007                         if (nft_expr_reduce(&track, expr)) {
9008                                 expr = track.cur;
9009                                 continue;
9010                         }
9011
9012                         if (WARN_ON_ONCE(data + size + expr->ops->size > data_boundary))
9013                                 return -ENOMEM;
9014
9015                         memcpy(data + size, expr, expr->ops->size);
9016                         size += expr->ops->size;
9017                 }
9018                 if (WARN_ON_ONCE(size >= 1 << 12))
9019                         return -ENOMEM;
9020
9021                 prule->handle = rule->handle;
9022                 prule->dlen = size;
9023                 prule->is_last = 0;
9024
9025                 data += size;
9026                 size = 0;
9027                 chain->blob_next->size += (unsigned long)(data - (void *)prule);
9028         }
9029
9030         if (WARN_ON_ONCE(data > data_boundary))
9031                 return -ENOMEM;
9032
9033         prule = (struct nft_rule_dp *)data;
9034         nft_last_rule(chain, prule);
9035
9036         return 0;
9037 }
9038
9039 static void nf_tables_commit_chain_prepare_cancel(struct net *net)
9040 {
9041         struct nftables_pernet *nft_net = nft_pernet(net);
9042         struct nft_trans *trans, *next;
9043
9044         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
9045                 struct nft_chain *chain = trans->ctx.chain;
9046
9047                 if (trans->msg_type == NFT_MSG_NEWRULE ||
9048                     trans->msg_type == NFT_MSG_DELRULE) {
9049                         kvfree(chain->blob_next);
9050                         chain->blob_next = NULL;
9051                 }
9052         }
9053 }
9054
9055 static void __nf_tables_commit_chain_free_rules(struct rcu_head *h)
9056 {
9057         struct nft_rule_dp_last *l = container_of(h, struct nft_rule_dp_last, h);
9058
9059         kvfree(l->blob);
9060 }
9061
9062 static void nf_tables_commit_chain_free_rules_old(struct nft_rule_blob *blob)
9063 {
9064         struct nft_rule_dp_last *last;
9065
9066         /* last rule trailer is after end marker */
9067         last = (void *)blob + sizeof(*blob) + blob->size;
9068         last->blob = blob;
9069
9070         call_rcu(&last->h, __nf_tables_commit_chain_free_rules);
9071 }
9072
9073 static void nf_tables_commit_chain(struct net *net, struct nft_chain *chain)
9074 {
9075         struct nft_rule_blob *g0, *g1;
9076         bool next_genbit;
9077
9078         next_genbit = nft_gencursor_next(net);
9079
9080         g0 = rcu_dereference_protected(chain->blob_gen_0,
9081                                        lockdep_commit_lock_is_held(net));
9082         g1 = rcu_dereference_protected(chain->blob_gen_1,
9083                                        lockdep_commit_lock_is_held(net));
9084
9085         /* No changes to this chain? */
9086         if (chain->blob_next == NULL) {
9087                 /* chain had no change in last or next generation */
9088                 if (g0 == g1)
9089                         return;
9090                 /*
9091                  * chain had no change in this generation; make sure next
9092                  * one uses same rules as current generation.
9093                  */
9094                 if (next_genbit) {
9095                         rcu_assign_pointer(chain->blob_gen_1, g0);
9096                         nf_tables_commit_chain_free_rules_old(g1);
9097                 } else {
9098                         rcu_assign_pointer(chain->blob_gen_0, g1);
9099                         nf_tables_commit_chain_free_rules_old(g0);
9100                 }
9101
9102                 return;
9103         }
9104
9105         if (next_genbit)
9106                 rcu_assign_pointer(chain->blob_gen_1, chain->blob_next);
9107         else
9108                 rcu_assign_pointer(chain->blob_gen_0, chain->blob_next);
9109
9110         chain->blob_next = NULL;
9111
9112         if (g0 == g1)
9113                 return;
9114
9115         if (next_genbit)
9116                 nf_tables_commit_chain_free_rules_old(g1);
9117         else
9118                 nf_tables_commit_chain_free_rules_old(g0);
9119 }
9120
9121 static void nft_obj_del(struct nft_object *obj)
9122 {
9123         rhltable_remove(&nft_objname_ht, &obj->rhlhead, nft_objname_ht_params);
9124         list_del_rcu(&obj->list);
9125 }
9126
9127 void nft_chain_del(struct nft_chain *chain)
9128 {
9129         struct nft_table *table = chain->table;
9130
9131         WARN_ON_ONCE(rhltable_remove(&table->chains_ht, &chain->rhlhead,
9132                                      nft_chain_ht_params));
9133         list_del_rcu(&chain->list);
9134 }
9135
9136 static void nf_tables_module_autoload_cleanup(struct net *net)
9137 {
9138         struct nftables_pernet *nft_net = nft_pernet(net);
9139         struct nft_module_request *req, *next;
9140
9141         WARN_ON_ONCE(!list_empty(&nft_net->commit_list));
9142         list_for_each_entry_safe(req, next, &nft_net->module_list, list) {
9143                 WARN_ON_ONCE(!req->done);
9144                 list_del(&req->list);
9145                 kfree(req);
9146         }
9147 }
9148
9149 static void nf_tables_commit_release(struct net *net)
9150 {
9151         struct nftables_pernet *nft_net = nft_pernet(net);
9152         struct nft_trans *trans;
9153
9154         /* all side effects have to be made visible.
9155          * For example, if a chain named 'foo' has been deleted, a
9156          * new transaction must not find it anymore.
9157          *
9158          * Memory reclaim happens asynchronously from work queue
9159          * to prevent expensive synchronize_rcu() in commit phase.
9160          */
9161         if (list_empty(&nft_net->commit_list)) {
9162                 nf_tables_module_autoload_cleanup(net);
9163                 mutex_unlock(&nft_net->commit_mutex);
9164                 return;
9165         }
9166
9167         trans = list_last_entry(&nft_net->commit_list,
9168                                 struct nft_trans, list);
9169         get_net(trans->ctx.net);
9170         WARN_ON_ONCE(trans->put_net);
9171
9172         trans->put_net = true;
9173         spin_lock(&nf_tables_destroy_list_lock);
9174         list_splice_tail_init(&nft_net->commit_list, &nf_tables_destroy_list);
9175         spin_unlock(&nf_tables_destroy_list_lock);
9176
9177         nf_tables_module_autoload_cleanup(net);
9178         schedule_work(&trans_destroy_work);
9179
9180         mutex_unlock(&nft_net->commit_mutex);
9181 }
9182
9183 static void nft_commit_notify(struct net *net, u32 portid)
9184 {
9185         struct nftables_pernet *nft_net = nft_pernet(net);
9186         struct sk_buff *batch_skb = NULL, *nskb, *skb;
9187         unsigned char *data;
9188         int len;
9189
9190         list_for_each_entry_safe(skb, nskb, &nft_net->notify_list, list) {
9191                 if (!batch_skb) {
9192 new_batch:
9193                         batch_skb = skb;
9194                         len = NLMSG_GOODSIZE - skb->len;
9195                         list_del(&skb->list);
9196                         continue;
9197                 }
9198                 len -= skb->len;
9199                 if (len > 0 && NFT_CB(skb).report == NFT_CB(batch_skb).report) {
9200                         data = skb_put(batch_skb, skb->len);
9201                         memcpy(data, skb->data, skb->len);
9202                         list_del(&skb->list);
9203                         kfree_skb(skb);
9204                         continue;
9205                 }
9206                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
9207                                NFT_CB(batch_skb).report, GFP_KERNEL);
9208                 goto new_batch;
9209         }
9210
9211         if (batch_skb) {
9212                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
9213                                NFT_CB(batch_skb).report, GFP_KERNEL);
9214         }
9215
9216         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
9217 }
9218
9219 static int nf_tables_commit_audit_alloc(struct list_head *adl,
9220                                         struct nft_table *table)
9221 {
9222         struct nft_audit_data *adp;
9223
9224         list_for_each_entry(adp, adl, list) {
9225                 if (adp->table == table)
9226                         return 0;
9227         }
9228         adp = kzalloc(sizeof(*adp), GFP_KERNEL);
9229         if (!adp)
9230                 return -ENOMEM;
9231         adp->table = table;
9232         list_add(&adp->list, adl);
9233         return 0;
9234 }
9235
9236 static void nf_tables_commit_audit_free(struct list_head *adl)
9237 {
9238         struct nft_audit_data *adp, *adn;
9239
9240         list_for_each_entry_safe(adp, adn, adl, list) {
9241                 list_del(&adp->list);
9242                 kfree(adp);
9243         }
9244 }
9245
9246 static void nf_tables_commit_audit_collect(struct list_head *adl,
9247                                            struct nft_table *table, u32 op)
9248 {
9249         struct nft_audit_data *adp;
9250
9251         list_for_each_entry(adp, adl, list) {
9252                 if (adp->table == table)
9253                         goto found;
9254         }
9255         WARN_ONCE(1, "table=%s not expected in commit list", table->name);
9256         return;
9257 found:
9258         adp->entries++;
9259         if (!adp->op || adp->op > op)
9260                 adp->op = op;
9261 }
9262
9263 #define AUNFTABLENAMELEN (NFT_TABLE_MAXNAMELEN + 22)
9264
9265 static void nf_tables_commit_audit_log(struct list_head *adl, u32 generation)
9266 {
9267         struct nft_audit_data *adp, *adn;
9268         char aubuf[AUNFTABLENAMELEN];
9269
9270         list_for_each_entry_safe(adp, adn, adl, list) {
9271                 snprintf(aubuf, AUNFTABLENAMELEN, "%s:%u", adp->table->name,
9272                          generation);
9273                 audit_log_nfcfg(aubuf, adp->table->family, adp->entries,
9274                                 nft2audit_op[adp->op], GFP_KERNEL);
9275                 list_del(&adp->list);
9276                 kfree(adp);
9277         }
9278 }
9279
9280 static void nft_set_commit_update(struct list_head *set_update_list)
9281 {
9282         struct nft_set *set, *next;
9283
9284         list_for_each_entry_safe(set, next, set_update_list, pending_update) {
9285                 list_del_init(&set->pending_update);
9286
9287                 if (!set->ops->commit)
9288                         continue;
9289
9290                 set->ops->commit(set);
9291         }
9292 }
9293
9294 static int nf_tables_commit(struct net *net, struct sk_buff *skb)
9295 {
9296         struct nftables_pernet *nft_net = nft_pernet(net);
9297         struct nft_trans *trans, *next;
9298         LIST_HEAD(set_update_list);
9299         struct nft_trans_elem *te;
9300         struct nft_chain *chain;
9301         struct nft_table *table;
9302         unsigned int base_seq;
9303         LIST_HEAD(adl);
9304         int err;
9305
9306         if (list_empty(&nft_net->commit_list)) {
9307                 mutex_unlock(&nft_net->commit_mutex);
9308                 return 0;
9309         }
9310
9311         /* 0. Validate ruleset, otherwise roll back for error reporting. */
9312         if (nf_tables_validate(net) < 0)
9313                 return -EAGAIN;
9314
9315         err = nft_flow_rule_offload_commit(net);
9316         if (err < 0)
9317                 return err;
9318
9319         /* 1.  Allocate space for next generation rules_gen_X[] */
9320         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
9321                 int ret;
9322
9323                 ret = nf_tables_commit_audit_alloc(&adl, trans->ctx.table);
9324                 if (ret) {
9325                         nf_tables_commit_chain_prepare_cancel(net);
9326                         nf_tables_commit_audit_free(&adl);
9327                         return ret;
9328                 }
9329                 if (trans->msg_type == NFT_MSG_NEWRULE ||
9330                     trans->msg_type == NFT_MSG_DELRULE) {
9331                         chain = trans->ctx.chain;
9332
9333                         ret = nf_tables_commit_chain_prepare(net, chain);
9334                         if (ret < 0) {
9335                                 nf_tables_commit_chain_prepare_cancel(net);
9336                                 nf_tables_commit_audit_free(&adl);
9337                                 return ret;
9338                         }
9339                 }
9340         }
9341
9342         /* step 2.  Make rules_gen_X visible to packet path */
9343         list_for_each_entry(table, &nft_net->tables, list) {
9344                 list_for_each_entry(chain, &table->chains, list)
9345                         nf_tables_commit_chain(net, chain);
9346         }
9347
9348         /*
9349          * Bump generation counter, invalidate any dump in progress.
9350          * Cannot fail after this point.
9351          */
9352         base_seq = READ_ONCE(nft_net->base_seq);
9353         while (++base_seq == 0)
9354                 ;
9355
9356         WRITE_ONCE(nft_net->base_seq, base_seq);
9357
9358         /* step 3. Start new generation, rules_gen_X now in use. */
9359         net->nft.gencursor = nft_gencursor_next(net);
9360
9361         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
9362                 nf_tables_commit_audit_collect(&adl, trans->ctx.table,
9363                                                trans->msg_type);
9364                 switch (trans->msg_type) {
9365                 case NFT_MSG_NEWTABLE:
9366                         if (nft_trans_table_update(trans)) {
9367                                 if (!(trans->ctx.table->flags & __NFT_TABLE_F_UPDATE)) {
9368                                         nft_trans_destroy(trans);
9369                                         break;
9370                                 }
9371                                 if (trans->ctx.table->flags & NFT_TABLE_F_DORMANT)
9372                                         nf_tables_table_disable(net, trans->ctx.table);
9373
9374                                 trans->ctx.table->flags &= ~__NFT_TABLE_F_UPDATE;
9375                         } else {
9376                                 nft_clear(net, trans->ctx.table);
9377                         }
9378                         nf_tables_table_notify(&trans->ctx, NFT_MSG_NEWTABLE);
9379                         nft_trans_destroy(trans);
9380                         break;
9381                 case NFT_MSG_DELTABLE:
9382                 case NFT_MSG_DESTROYTABLE:
9383                         list_del_rcu(&trans->ctx.table->list);
9384                         nf_tables_table_notify(&trans->ctx, trans->msg_type);
9385                         break;
9386                 case NFT_MSG_NEWCHAIN:
9387                         if (nft_trans_chain_update(trans)) {
9388                                 nft_chain_commit_update(trans);
9389                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN,
9390                                                        &nft_trans_chain_hooks(trans));
9391                                 list_splice(&nft_trans_chain_hooks(trans),
9392                                             &nft_trans_basechain(trans)->hook_list);
9393                                 /* trans destroyed after rcu grace period */
9394                         } else {
9395                                 nft_chain_commit_drop_policy(trans);
9396                                 nft_clear(net, trans->ctx.chain);
9397                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN, NULL);
9398                                 nft_trans_destroy(trans);
9399                         }
9400                         break;
9401                 case NFT_MSG_DELCHAIN:
9402                 case NFT_MSG_DESTROYCHAIN:
9403                         if (nft_trans_chain_update(trans)) {
9404                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_DELCHAIN,
9405                                                        &nft_trans_chain_hooks(trans));
9406                                 nft_netdev_unregister_hooks(net,
9407                                                             &nft_trans_chain_hooks(trans),
9408                                                             true);
9409                         } else {
9410                                 nft_chain_del(trans->ctx.chain);
9411                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_DELCHAIN,
9412                                                        NULL);
9413                                 nf_tables_unregister_hook(trans->ctx.net,
9414                                                           trans->ctx.table,
9415                                                           trans->ctx.chain);
9416                         }
9417                         break;
9418                 case NFT_MSG_NEWRULE:
9419                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
9420                         nf_tables_rule_notify(&trans->ctx,
9421                                               nft_trans_rule(trans),
9422                                               NFT_MSG_NEWRULE);
9423                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
9424                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
9425
9426                         nft_trans_destroy(trans);
9427                         break;
9428                 case NFT_MSG_DELRULE:
9429                 case NFT_MSG_DESTROYRULE:
9430                         list_del_rcu(&nft_trans_rule(trans)->list);
9431                         nf_tables_rule_notify(&trans->ctx,
9432                                               nft_trans_rule(trans),
9433                                               trans->msg_type);
9434                         nft_rule_expr_deactivate(&trans->ctx,
9435                                                  nft_trans_rule(trans),
9436                                                  NFT_TRANS_COMMIT);
9437
9438                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
9439                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
9440                         break;
9441                 case NFT_MSG_NEWSET:
9442                         if (nft_trans_set_update(trans)) {
9443                                 struct nft_set *set = nft_trans_set(trans);
9444
9445                                 WRITE_ONCE(set->timeout, nft_trans_set_timeout(trans));
9446                                 WRITE_ONCE(set->gc_int, nft_trans_set_gc_int(trans));
9447                         } else {
9448                                 nft_clear(net, nft_trans_set(trans));
9449                                 /* This avoids hitting -EBUSY when deleting the table
9450                                  * from the transaction.
9451                                  */
9452                                 if (nft_set_is_anonymous(nft_trans_set(trans)) &&
9453                                     !list_empty(&nft_trans_set(trans)->bindings))
9454                                         trans->ctx.table->use--;
9455                         }
9456                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
9457                                              NFT_MSG_NEWSET, GFP_KERNEL);
9458                         nft_trans_destroy(trans);
9459                         break;
9460                 case NFT_MSG_DELSET:
9461                 case NFT_MSG_DESTROYSET:
9462                         list_del_rcu(&nft_trans_set(trans)->list);
9463                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
9464                                              trans->msg_type, GFP_KERNEL);
9465                         break;
9466                 case NFT_MSG_NEWSETELEM:
9467                         te = (struct nft_trans_elem *)trans->data;
9468
9469                         nft_setelem_activate(net, te->set, &te->elem);
9470                         nf_tables_setelem_notify(&trans->ctx, te->set,
9471                                                  &te->elem,
9472                                                  NFT_MSG_NEWSETELEM);
9473                         if (te->set->ops->commit &&
9474                             list_empty(&te->set->pending_update)) {
9475                                 list_add_tail(&te->set->pending_update,
9476                                               &set_update_list);
9477                         }
9478                         nft_trans_destroy(trans);
9479                         break;
9480                 case NFT_MSG_DELSETELEM:
9481                 case NFT_MSG_DESTROYSETELEM:
9482                         te = (struct nft_trans_elem *)trans->data;
9483
9484                         nf_tables_setelem_notify(&trans->ctx, te->set,
9485                                                  &te->elem,
9486                                                  trans->msg_type);
9487                         nft_setelem_remove(net, te->set, &te->elem);
9488                         if (!nft_setelem_is_catchall(te->set, &te->elem)) {
9489                                 atomic_dec(&te->set->nelems);
9490                                 te->set->ndeact--;
9491                         }
9492                         if (te->set->ops->commit &&
9493                             list_empty(&te->set->pending_update)) {
9494                                 list_add_tail(&te->set->pending_update,
9495                                               &set_update_list);
9496                         }
9497                         break;
9498                 case NFT_MSG_NEWOBJ:
9499                         if (nft_trans_obj_update(trans)) {
9500                                 nft_obj_commit_update(trans);
9501                                 nf_tables_obj_notify(&trans->ctx,
9502                                                      nft_trans_obj(trans),
9503                                                      NFT_MSG_NEWOBJ);
9504                         } else {
9505                                 nft_clear(net, nft_trans_obj(trans));
9506                                 nf_tables_obj_notify(&trans->ctx,
9507                                                      nft_trans_obj(trans),
9508                                                      NFT_MSG_NEWOBJ);
9509                                 nft_trans_destroy(trans);
9510                         }
9511                         break;
9512                 case NFT_MSG_DELOBJ:
9513                 case NFT_MSG_DESTROYOBJ:
9514                         nft_obj_del(nft_trans_obj(trans));
9515                         nf_tables_obj_notify(&trans->ctx, nft_trans_obj(trans),
9516                                              trans->msg_type);
9517                         break;
9518                 case NFT_MSG_NEWFLOWTABLE:
9519                         if (nft_trans_flowtable_update(trans)) {
9520                                 nft_trans_flowtable(trans)->data.flags =
9521                                         nft_trans_flowtable_flags(trans);
9522                                 nf_tables_flowtable_notify(&trans->ctx,
9523                                                            nft_trans_flowtable(trans),
9524                                                            &nft_trans_flowtable_hooks(trans),
9525                                                            NFT_MSG_NEWFLOWTABLE);
9526                                 list_splice(&nft_trans_flowtable_hooks(trans),
9527                                             &nft_trans_flowtable(trans)->hook_list);
9528                         } else {
9529                                 nft_clear(net, nft_trans_flowtable(trans));
9530                                 nf_tables_flowtable_notify(&trans->ctx,
9531                                                            nft_trans_flowtable(trans),
9532                                                            NULL,
9533                                                            NFT_MSG_NEWFLOWTABLE);
9534                         }
9535                         nft_trans_destroy(trans);
9536                         break;
9537                 case NFT_MSG_DELFLOWTABLE:
9538                 case NFT_MSG_DESTROYFLOWTABLE:
9539                         if (nft_trans_flowtable_update(trans)) {
9540                                 nf_tables_flowtable_notify(&trans->ctx,
9541                                                            nft_trans_flowtable(trans),
9542                                                            &nft_trans_flowtable_hooks(trans),
9543                                                            trans->msg_type);
9544                                 nft_unregister_flowtable_net_hooks(net,
9545                                                                    &nft_trans_flowtable_hooks(trans));
9546                         } else {
9547                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
9548                                 nf_tables_flowtable_notify(&trans->ctx,
9549                                                            nft_trans_flowtable(trans),
9550                                                            NULL,
9551                                                            trans->msg_type);
9552                                 nft_unregister_flowtable_net_hooks(net,
9553                                                 &nft_trans_flowtable(trans)->hook_list);
9554                         }
9555                         break;
9556                 }
9557         }
9558
9559         nft_set_commit_update(&set_update_list);
9560
9561         nft_commit_notify(net, NETLINK_CB(skb).portid);
9562         nf_tables_gen_notify(net, skb, NFT_MSG_NEWGEN);
9563         nf_tables_commit_audit_log(&adl, nft_net->base_seq);
9564         nf_tables_commit_release(net);
9565
9566         return 0;
9567 }
9568
9569 static void nf_tables_module_autoload(struct net *net)
9570 {
9571         struct nftables_pernet *nft_net = nft_pernet(net);
9572         struct nft_module_request *req, *next;
9573         LIST_HEAD(module_list);
9574
9575         list_splice_init(&nft_net->module_list, &module_list);
9576         mutex_unlock(&nft_net->commit_mutex);
9577         list_for_each_entry_safe(req, next, &module_list, list) {
9578                 request_module("%s", req->module);
9579                 req->done = true;
9580         }
9581         mutex_lock(&nft_net->commit_mutex);
9582         list_splice(&module_list, &nft_net->module_list);
9583 }
9584
9585 static void nf_tables_abort_release(struct nft_trans *trans)
9586 {
9587         switch (trans->msg_type) {
9588         case NFT_MSG_NEWTABLE:
9589                 nf_tables_table_destroy(&trans->ctx);
9590                 break;
9591         case NFT_MSG_NEWCHAIN:
9592                 if (nft_trans_chain_update(trans))
9593                         nft_hooks_destroy(&nft_trans_chain_hooks(trans));
9594                 else
9595                         nf_tables_chain_destroy(&trans->ctx);
9596                 break;
9597         case NFT_MSG_NEWRULE:
9598                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
9599                 break;
9600         case NFT_MSG_NEWSET:
9601                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
9602                 break;
9603         case NFT_MSG_NEWSETELEM:
9604                 nft_set_elem_destroy(nft_trans_elem_set(trans),
9605                                      nft_trans_elem(trans).priv, true);
9606                 break;
9607         case NFT_MSG_NEWOBJ:
9608                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
9609                 break;
9610         case NFT_MSG_NEWFLOWTABLE:
9611                 if (nft_trans_flowtable_update(trans))
9612                         nft_hooks_destroy(&nft_trans_flowtable_hooks(trans));
9613                 else
9614                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
9615                 break;
9616         }
9617         kfree(trans);
9618 }
9619
9620 static void nft_set_abort_update(struct list_head *set_update_list)
9621 {
9622         struct nft_set *set, *next;
9623
9624         list_for_each_entry_safe(set, next, set_update_list, pending_update) {
9625                 list_del_init(&set->pending_update);
9626
9627                 if (!set->ops->abort)
9628                         continue;
9629
9630                 set->ops->abort(set);
9631         }
9632 }
9633
9634 static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action)
9635 {
9636         struct nftables_pernet *nft_net = nft_pernet(net);
9637         struct nft_trans *trans, *next;
9638         LIST_HEAD(set_update_list);
9639         struct nft_trans_elem *te;
9640
9641         if (action == NFNL_ABORT_VALIDATE &&
9642             nf_tables_validate(net) < 0)
9643                 return -EAGAIN;
9644
9645         list_for_each_entry_safe_reverse(trans, next, &nft_net->commit_list,
9646                                          list) {
9647                 switch (trans->msg_type) {
9648                 case NFT_MSG_NEWTABLE:
9649                         if (nft_trans_table_update(trans)) {
9650                                 if (!(trans->ctx.table->flags & __NFT_TABLE_F_UPDATE)) {
9651                                         nft_trans_destroy(trans);
9652                                         break;
9653                                 }
9654                                 if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_DORMANT) {
9655                                         nf_tables_table_disable(net, trans->ctx.table);
9656                                         trans->ctx.table->flags |= NFT_TABLE_F_DORMANT;
9657                                 } else if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_AWAKEN) {
9658                                         trans->ctx.table->flags &= ~NFT_TABLE_F_DORMANT;
9659                                 }
9660                                 trans->ctx.table->flags &= ~__NFT_TABLE_F_UPDATE;
9661                                 nft_trans_destroy(trans);
9662                         } else {
9663                                 list_del_rcu(&trans->ctx.table->list);
9664                         }
9665                         break;
9666                 case NFT_MSG_DELTABLE:
9667                 case NFT_MSG_DESTROYTABLE:
9668                         nft_clear(trans->ctx.net, trans->ctx.table);
9669                         nft_trans_destroy(trans);
9670                         break;
9671                 case NFT_MSG_NEWCHAIN:
9672                         if (nft_trans_chain_update(trans)) {
9673                                 nft_netdev_unregister_hooks(net,
9674                                                             &nft_trans_chain_hooks(trans),
9675                                                             true);
9676                                 free_percpu(nft_trans_chain_stats(trans));
9677                                 kfree(nft_trans_chain_name(trans));
9678                                 nft_trans_destroy(trans);
9679                         } else {
9680                                 if (nft_chain_is_bound(trans->ctx.chain)) {
9681                                         nft_trans_destroy(trans);
9682                                         break;
9683                                 }
9684                                 trans->ctx.table->use--;
9685                                 nft_chain_del(trans->ctx.chain);
9686                                 nf_tables_unregister_hook(trans->ctx.net,
9687                                                           trans->ctx.table,
9688                                                           trans->ctx.chain);
9689                         }
9690                         break;
9691                 case NFT_MSG_DELCHAIN:
9692                 case NFT_MSG_DESTROYCHAIN:
9693                         if (nft_trans_chain_update(trans)) {
9694                                 list_splice(&nft_trans_chain_hooks(trans),
9695                                             &nft_trans_basechain(trans)->hook_list);
9696                         } else {
9697                                 trans->ctx.table->use++;
9698                                 nft_clear(trans->ctx.net, trans->ctx.chain);
9699                         }
9700                         nft_trans_destroy(trans);
9701                         break;
9702                 case NFT_MSG_NEWRULE:
9703                         trans->ctx.chain->use--;
9704                         list_del_rcu(&nft_trans_rule(trans)->list);
9705                         nft_rule_expr_deactivate(&trans->ctx,
9706                                                  nft_trans_rule(trans),
9707                                                  NFT_TRANS_ABORT);
9708                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
9709                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
9710                         break;
9711                 case NFT_MSG_DELRULE:
9712                 case NFT_MSG_DESTROYRULE:
9713                         trans->ctx.chain->use++;
9714                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
9715                         nft_rule_expr_activate(&trans->ctx, nft_trans_rule(trans));
9716                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
9717                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
9718
9719                         nft_trans_destroy(trans);
9720                         break;
9721                 case NFT_MSG_NEWSET:
9722                         if (nft_trans_set_update(trans)) {
9723                                 nft_trans_destroy(trans);
9724                                 break;
9725                         }
9726                         trans->ctx.table->use--;
9727                         if (nft_trans_set_bound(trans)) {
9728                                 nft_trans_destroy(trans);
9729                                 break;
9730                         }
9731                         list_del_rcu(&nft_trans_set(trans)->list);
9732                         break;
9733                 case NFT_MSG_DELSET:
9734                 case NFT_MSG_DESTROYSET:
9735                         trans->ctx.table->use++;
9736                         nft_clear(trans->ctx.net, nft_trans_set(trans));
9737                         nft_trans_destroy(trans);
9738                         break;
9739                 case NFT_MSG_NEWSETELEM:
9740                         if (nft_trans_elem_set_bound(trans)) {
9741                                 nft_trans_destroy(trans);
9742                                 break;
9743                         }
9744                         te = (struct nft_trans_elem *)trans->data;
9745                         nft_setelem_remove(net, te->set, &te->elem);
9746                         if (!nft_setelem_is_catchall(te->set, &te->elem))
9747                                 atomic_dec(&te->set->nelems);
9748
9749                         if (te->set->ops->abort &&
9750                             list_empty(&te->set->pending_update)) {
9751                                 list_add_tail(&te->set->pending_update,
9752                                               &set_update_list);
9753                         }
9754                         break;
9755                 case NFT_MSG_DELSETELEM:
9756                 case NFT_MSG_DESTROYSETELEM:
9757                         te = (struct nft_trans_elem *)trans->data;
9758
9759                         nft_setelem_data_activate(net, te->set, &te->elem);
9760                         nft_setelem_activate(net, te->set, &te->elem);
9761                         if (!nft_setelem_is_catchall(te->set, &te->elem))
9762                                 te->set->ndeact--;
9763
9764                         if (te->set->ops->abort &&
9765                             list_empty(&te->set->pending_update)) {
9766                                 list_add_tail(&te->set->pending_update,
9767                                               &set_update_list);
9768                         }
9769                         nft_trans_destroy(trans);
9770                         break;
9771                 case NFT_MSG_NEWOBJ:
9772                         if (nft_trans_obj_update(trans)) {
9773                                 nft_obj_destroy(&trans->ctx, nft_trans_obj_newobj(trans));
9774                                 nft_trans_destroy(trans);
9775                         } else {
9776                                 trans->ctx.table->use--;
9777                                 nft_obj_del(nft_trans_obj(trans));
9778                         }
9779                         break;
9780                 case NFT_MSG_DELOBJ:
9781                 case NFT_MSG_DESTROYOBJ:
9782                         trans->ctx.table->use++;
9783                         nft_clear(trans->ctx.net, nft_trans_obj(trans));
9784                         nft_trans_destroy(trans);
9785                         break;
9786                 case NFT_MSG_NEWFLOWTABLE:
9787                         if (nft_trans_flowtable_update(trans)) {
9788                                 nft_unregister_flowtable_net_hooks(net,
9789                                                 &nft_trans_flowtable_hooks(trans));
9790                         } else {
9791                                 trans->ctx.table->use--;
9792                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
9793                                 nft_unregister_flowtable_net_hooks(net,
9794                                                 &nft_trans_flowtable(trans)->hook_list);
9795                         }
9796                         break;
9797                 case NFT_MSG_DELFLOWTABLE:
9798                 case NFT_MSG_DESTROYFLOWTABLE:
9799                         if (nft_trans_flowtable_update(trans)) {
9800                                 list_splice(&nft_trans_flowtable_hooks(trans),
9801                                             &nft_trans_flowtable(trans)->hook_list);
9802                         } else {
9803                                 trans->ctx.table->use++;
9804                                 nft_clear(trans->ctx.net, nft_trans_flowtable(trans));
9805                         }
9806                         nft_trans_destroy(trans);
9807                         break;
9808                 }
9809         }
9810
9811         nft_set_abort_update(&set_update_list);
9812
9813         synchronize_rcu();
9814
9815         list_for_each_entry_safe_reverse(trans, next,
9816                                          &nft_net->commit_list, list) {
9817                 list_del(&trans->list);
9818                 nf_tables_abort_release(trans);
9819         }
9820
9821         if (action == NFNL_ABORT_AUTOLOAD)
9822                 nf_tables_module_autoload(net);
9823         else
9824                 nf_tables_module_autoload_cleanup(net);
9825
9826         return 0;
9827 }
9828
9829 static int nf_tables_abort(struct net *net, struct sk_buff *skb,
9830                            enum nfnl_abort_action action)
9831 {
9832         struct nftables_pernet *nft_net = nft_pernet(net);
9833         int ret = __nf_tables_abort(net, action);
9834
9835         mutex_unlock(&nft_net->commit_mutex);
9836
9837         return ret;
9838 }
9839
9840 static bool nf_tables_valid_genid(struct net *net, u32 genid)
9841 {
9842         struct nftables_pernet *nft_net = nft_pernet(net);
9843         bool genid_ok;
9844
9845         mutex_lock(&nft_net->commit_mutex);
9846
9847         genid_ok = genid == 0 || nft_net->base_seq == genid;
9848         if (!genid_ok)
9849                 mutex_unlock(&nft_net->commit_mutex);
9850
9851         /* else, commit mutex has to be released by commit or abort function */
9852         return genid_ok;
9853 }
9854
9855 static const struct nfnetlink_subsystem nf_tables_subsys = {
9856         .name           = "nf_tables",
9857         .subsys_id      = NFNL_SUBSYS_NFTABLES,
9858         .cb_count       = NFT_MSG_MAX,
9859         .cb             = nf_tables_cb,
9860         .commit         = nf_tables_commit,
9861         .abort          = nf_tables_abort,
9862         .valid_genid    = nf_tables_valid_genid,
9863         .owner          = THIS_MODULE,
9864 };
9865
9866 int nft_chain_validate_dependency(const struct nft_chain *chain,
9867                                   enum nft_chain_types type)
9868 {
9869         const struct nft_base_chain *basechain;
9870
9871         if (nft_is_base_chain(chain)) {
9872                 basechain = nft_base_chain(chain);
9873                 if (basechain->type->type != type)
9874                         return -EOPNOTSUPP;
9875         }
9876         return 0;
9877 }
9878 EXPORT_SYMBOL_GPL(nft_chain_validate_dependency);
9879
9880 int nft_chain_validate_hooks(const struct nft_chain *chain,
9881                              unsigned int hook_flags)
9882 {
9883         struct nft_base_chain *basechain;
9884
9885         if (nft_is_base_chain(chain)) {
9886                 basechain = nft_base_chain(chain);
9887
9888                 if ((1 << basechain->ops.hooknum) & hook_flags)
9889                         return 0;
9890
9891                 return -EOPNOTSUPP;
9892         }
9893
9894         return 0;
9895 }
9896 EXPORT_SYMBOL_GPL(nft_chain_validate_hooks);
9897
9898 /*
9899  * Loop detection - walk through the ruleset beginning at the destination chain
9900  * of a new jump until either the source chain is reached (loop) or all
9901  * reachable chains have been traversed.
9902  *
9903  * The loop check is performed whenever a new jump verdict is added to an
9904  * expression or verdict map or a verdict map is bound to a new chain.
9905  */
9906
9907 static int nf_tables_check_loops(const struct nft_ctx *ctx,
9908                                  const struct nft_chain *chain);
9909
9910 static int nft_check_loops(const struct nft_ctx *ctx,
9911                            const struct nft_set_ext *ext)
9912 {
9913         const struct nft_data *data;
9914         int ret;
9915
9916         data = nft_set_ext_data(ext);
9917         switch (data->verdict.code) {
9918         case NFT_JUMP:
9919         case NFT_GOTO:
9920                 ret = nf_tables_check_loops(ctx, data->verdict.chain);
9921                 break;
9922         default:
9923                 ret = 0;
9924                 break;
9925         }
9926
9927         return ret;
9928 }
9929
9930 static int nf_tables_loop_check_setelem(const struct nft_ctx *ctx,
9931                                         struct nft_set *set,
9932                                         const struct nft_set_iter *iter,
9933                                         struct nft_set_elem *elem)
9934 {
9935         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
9936
9937         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
9938             *nft_set_ext_flags(ext) & NFT_SET_ELEM_INTERVAL_END)
9939                 return 0;
9940
9941         return nft_check_loops(ctx, ext);
9942 }
9943
9944 static int nft_set_catchall_loops(const struct nft_ctx *ctx,
9945                                   struct nft_set *set)
9946 {
9947         u8 genmask = nft_genmask_next(ctx->net);
9948         struct nft_set_elem_catchall *catchall;
9949         struct nft_set_ext *ext;
9950         int ret = 0;
9951
9952         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
9953                 ext = nft_set_elem_ext(set, catchall->elem);
9954                 if (!nft_set_elem_active(ext, genmask))
9955                         continue;
9956
9957                 ret = nft_check_loops(ctx, ext);
9958                 if (ret < 0)
9959                         return ret;
9960         }
9961
9962         return ret;
9963 }
9964
9965 static int nf_tables_check_loops(const struct nft_ctx *ctx,
9966                                  const struct nft_chain *chain)
9967 {
9968         const struct nft_rule *rule;
9969         const struct nft_expr *expr, *last;
9970         struct nft_set *set;
9971         struct nft_set_binding *binding;
9972         struct nft_set_iter iter;
9973
9974         if (ctx->chain == chain)
9975                 return -ELOOP;
9976
9977         list_for_each_entry(rule, &chain->rules, list) {
9978                 nft_rule_for_each_expr(expr, last, rule) {
9979                         struct nft_immediate_expr *priv;
9980                         const struct nft_data *data;
9981                         int err;
9982
9983                         if (strcmp(expr->ops->type->name, "immediate"))
9984                                 continue;
9985
9986                         priv = nft_expr_priv(expr);
9987                         if (priv->dreg != NFT_REG_VERDICT)
9988                                 continue;
9989
9990                         data = &priv->data;
9991                         switch (data->verdict.code) {
9992                         case NFT_JUMP:
9993                         case NFT_GOTO:
9994                                 err = nf_tables_check_loops(ctx,
9995                                                         data->verdict.chain);
9996                                 if (err < 0)
9997                                         return err;
9998                                 break;
9999                         default:
10000                                 break;
10001                         }
10002                 }
10003         }
10004
10005         list_for_each_entry(set, &ctx->table->sets, list) {
10006                 if (!nft_is_active_next(ctx->net, set))
10007                         continue;
10008                 if (!(set->flags & NFT_SET_MAP) ||
10009                     set->dtype != NFT_DATA_VERDICT)
10010                         continue;
10011
10012                 list_for_each_entry(binding, &set->bindings, list) {
10013                         if (!(binding->flags & NFT_SET_MAP) ||
10014                             binding->chain != chain)
10015                                 continue;
10016
10017                         iter.genmask    = nft_genmask_next(ctx->net);
10018                         iter.skip       = 0;
10019                         iter.count      = 0;
10020                         iter.err        = 0;
10021                         iter.fn         = nf_tables_loop_check_setelem;
10022
10023                         set->ops->walk(ctx, set, &iter);
10024                         if (!iter.err)
10025                                 iter.err = nft_set_catchall_loops(ctx, set);
10026
10027                         if (iter.err < 0)
10028                                 return iter.err;
10029                 }
10030         }
10031
10032         return 0;
10033 }
10034
10035 /**
10036  *      nft_parse_u32_check - fetch u32 attribute and check for maximum value
10037  *
10038  *      @attr: netlink attribute to fetch value from
10039  *      @max: maximum value to be stored in dest
10040  *      @dest: pointer to the variable
10041  *
10042  *      Parse, check and store a given u32 netlink attribute into variable.
10043  *      This function returns -ERANGE if the value goes over maximum value.
10044  *      Otherwise a 0 is returned and the attribute value is stored in the
10045  *      destination variable.
10046  */
10047 int nft_parse_u32_check(const struct nlattr *attr, int max, u32 *dest)
10048 {
10049         u32 val;
10050
10051         val = ntohl(nla_get_be32(attr));
10052         if (val > max)
10053                 return -ERANGE;
10054
10055         *dest = val;
10056         return 0;
10057 }
10058 EXPORT_SYMBOL_GPL(nft_parse_u32_check);
10059
10060 static int nft_parse_register(const struct nlattr *attr, u32 *preg)
10061 {
10062         unsigned int reg;
10063
10064         reg = ntohl(nla_get_be32(attr));
10065         switch (reg) {
10066         case NFT_REG_VERDICT...NFT_REG_4:
10067                 *preg = reg * NFT_REG_SIZE / NFT_REG32_SIZE;
10068                 break;
10069         case NFT_REG32_00...NFT_REG32_15:
10070                 *preg = reg + NFT_REG_SIZE / NFT_REG32_SIZE - NFT_REG32_00;
10071                 break;
10072         default:
10073                 return -ERANGE;
10074         }
10075
10076         return 0;
10077 }
10078
10079 /**
10080  *      nft_dump_register - dump a register value to a netlink attribute
10081  *
10082  *      @skb: socket buffer
10083  *      @attr: attribute number
10084  *      @reg: register number
10085  *
10086  *      Construct a netlink attribute containing the register number. For
10087  *      compatibility reasons, register numbers being a multiple of 4 are
10088  *      translated to the corresponding 128 bit register numbers.
10089  */
10090 int nft_dump_register(struct sk_buff *skb, unsigned int attr, unsigned int reg)
10091 {
10092         if (reg % (NFT_REG_SIZE / NFT_REG32_SIZE) == 0)
10093                 reg = reg / (NFT_REG_SIZE / NFT_REG32_SIZE);
10094         else
10095                 reg = reg - NFT_REG_SIZE / NFT_REG32_SIZE + NFT_REG32_00;
10096
10097         return nla_put_be32(skb, attr, htonl(reg));
10098 }
10099 EXPORT_SYMBOL_GPL(nft_dump_register);
10100
10101 static int nft_validate_register_load(enum nft_registers reg, unsigned int len)
10102 {
10103         if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
10104                 return -EINVAL;
10105         if (len == 0)
10106                 return -EINVAL;
10107         if (reg * NFT_REG32_SIZE + len > sizeof_field(struct nft_regs, data))
10108                 return -ERANGE;
10109
10110         return 0;
10111 }
10112
10113 int nft_parse_register_load(const struct nlattr *attr, u8 *sreg, u32 len)
10114 {
10115         u32 reg;
10116         int err;
10117
10118         err = nft_parse_register(attr, &reg);
10119         if (err < 0)
10120                 return err;
10121
10122         err = nft_validate_register_load(reg, len);
10123         if (err < 0)
10124                 return err;
10125
10126         *sreg = reg;
10127         return 0;
10128 }
10129 EXPORT_SYMBOL_GPL(nft_parse_register_load);
10130
10131 static int nft_validate_register_store(const struct nft_ctx *ctx,
10132                                        enum nft_registers reg,
10133                                        const struct nft_data *data,
10134                                        enum nft_data_types type,
10135                                        unsigned int len)
10136 {
10137         int err;
10138
10139         switch (reg) {
10140         case NFT_REG_VERDICT:
10141                 if (type != NFT_DATA_VERDICT)
10142                         return -EINVAL;
10143
10144                 if (data != NULL &&
10145                     (data->verdict.code == NFT_GOTO ||
10146                      data->verdict.code == NFT_JUMP)) {
10147                         err = nf_tables_check_loops(ctx, data->verdict.chain);
10148                         if (err < 0)
10149                                 return err;
10150                 }
10151
10152                 return 0;
10153         default:
10154                 if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
10155                         return -EINVAL;
10156                 if (len == 0)
10157                         return -EINVAL;
10158                 if (reg * NFT_REG32_SIZE + len >
10159                     sizeof_field(struct nft_regs, data))
10160                         return -ERANGE;
10161
10162                 if (data != NULL && type != NFT_DATA_VALUE)
10163                         return -EINVAL;
10164                 return 0;
10165         }
10166 }
10167
10168 int nft_parse_register_store(const struct nft_ctx *ctx,
10169                              const struct nlattr *attr, u8 *dreg,
10170                              const struct nft_data *data,
10171                              enum nft_data_types type, unsigned int len)
10172 {
10173         int err;
10174         u32 reg;
10175
10176         err = nft_parse_register(attr, &reg);
10177         if (err < 0)
10178                 return err;
10179
10180         err = nft_validate_register_store(ctx, reg, data, type, len);
10181         if (err < 0)
10182                 return err;
10183
10184         *dreg = reg;
10185         return 0;
10186 }
10187 EXPORT_SYMBOL_GPL(nft_parse_register_store);
10188
10189 static const struct nla_policy nft_verdict_policy[NFTA_VERDICT_MAX + 1] = {
10190         [NFTA_VERDICT_CODE]     = { .type = NLA_U32 },
10191         [NFTA_VERDICT_CHAIN]    = { .type = NLA_STRING,
10192                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
10193         [NFTA_VERDICT_CHAIN_ID] = { .type = NLA_U32 },
10194 };
10195
10196 static int nft_verdict_init(const struct nft_ctx *ctx, struct nft_data *data,
10197                             struct nft_data_desc *desc, const struct nlattr *nla)
10198 {
10199         u8 genmask = nft_genmask_next(ctx->net);
10200         struct nlattr *tb[NFTA_VERDICT_MAX + 1];
10201         struct nft_chain *chain;
10202         int err;
10203
10204         err = nla_parse_nested_deprecated(tb, NFTA_VERDICT_MAX, nla,
10205                                           nft_verdict_policy, NULL);
10206         if (err < 0)
10207                 return err;
10208
10209         if (!tb[NFTA_VERDICT_CODE])
10210                 return -EINVAL;
10211         data->verdict.code = ntohl(nla_get_be32(tb[NFTA_VERDICT_CODE]));
10212
10213         switch (data->verdict.code) {
10214         default:
10215                 switch (data->verdict.code & NF_VERDICT_MASK) {
10216                 case NF_ACCEPT:
10217                 case NF_DROP:
10218                 case NF_QUEUE:
10219                         break;
10220                 default:
10221                         return -EINVAL;
10222                 }
10223                 fallthrough;
10224         case NFT_CONTINUE:
10225         case NFT_BREAK:
10226         case NFT_RETURN:
10227                 break;
10228         case NFT_JUMP:
10229         case NFT_GOTO:
10230                 if (tb[NFTA_VERDICT_CHAIN]) {
10231                         chain = nft_chain_lookup(ctx->net, ctx->table,
10232                                                  tb[NFTA_VERDICT_CHAIN],
10233                                                  genmask);
10234                 } else if (tb[NFTA_VERDICT_CHAIN_ID]) {
10235                         chain = nft_chain_lookup_byid(ctx->net, ctx->table,
10236                                                       tb[NFTA_VERDICT_CHAIN_ID]);
10237                         if (IS_ERR(chain))
10238                                 return PTR_ERR(chain);
10239                 } else {
10240                         return -EINVAL;
10241                 }
10242
10243                 if (IS_ERR(chain))
10244                         return PTR_ERR(chain);
10245                 if (nft_is_base_chain(chain))
10246                         return -EOPNOTSUPP;
10247                 if (nft_chain_is_bound(chain))
10248                         return -EINVAL;
10249                 if (desc->flags & NFT_DATA_DESC_SETELEM &&
10250                     chain->flags & NFT_CHAIN_BINDING)
10251                         return -EINVAL;
10252
10253                 chain->use++;
10254                 data->verdict.chain = chain;
10255                 break;
10256         }
10257
10258         desc->len = sizeof(data->verdict);
10259
10260         return 0;
10261 }
10262
10263 static void nft_verdict_uninit(const struct nft_data *data)
10264 {
10265         struct nft_chain *chain;
10266         struct nft_rule *rule;
10267
10268         switch (data->verdict.code) {
10269         case NFT_JUMP:
10270         case NFT_GOTO:
10271                 chain = data->verdict.chain;
10272                 chain->use--;
10273
10274                 if (!nft_chain_is_bound(chain))
10275                         break;
10276
10277                 chain->table->use--;
10278                 list_for_each_entry(rule, &chain->rules, list)
10279                         chain->use--;
10280
10281                 nft_chain_del(chain);
10282                 break;
10283         }
10284 }
10285
10286 int nft_verdict_dump(struct sk_buff *skb, int type, const struct nft_verdict *v)
10287 {
10288         struct nlattr *nest;
10289
10290         nest = nla_nest_start_noflag(skb, type);
10291         if (!nest)
10292                 goto nla_put_failure;
10293
10294         if (nla_put_be32(skb, NFTA_VERDICT_CODE, htonl(v->code)))
10295                 goto nla_put_failure;
10296
10297         switch (v->code) {
10298         case NFT_JUMP:
10299         case NFT_GOTO:
10300                 if (nla_put_string(skb, NFTA_VERDICT_CHAIN,
10301                                    v->chain->name))
10302                         goto nla_put_failure;
10303         }
10304         nla_nest_end(skb, nest);
10305         return 0;
10306
10307 nla_put_failure:
10308         return -1;
10309 }
10310
10311 static int nft_value_init(const struct nft_ctx *ctx,
10312                           struct nft_data *data, struct nft_data_desc *desc,
10313                           const struct nlattr *nla)
10314 {
10315         unsigned int len;
10316
10317         len = nla_len(nla);
10318         if (len == 0)
10319                 return -EINVAL;
10320         if (len > desc->size)
10321                 return -EOVERFLOW;
10322         if (desc->len) {
10323                 if (len != desc->len)
10324                         return -EINVAL;
10325         } else {
10326                 desc->len = len;
10327         }
10328
10329         nla_memcpy(data->data, nla, len);
10330
10331         return 0;
10332 }
10333
10334 static int nft_value_dump(struct sk_buff *skb, const struct nft_data *data,
10335                           unsigned int len)
10336 {
10337         return nla_put(skb, NFTA_DATA_VALUE, len, data->data);
10338 }
10339
10340 static const struct nla_policy nft_data_policy[NFTA_DATA_MAX + 1] = {
10341         [NFTA_DATA_VALUE]       = { .type = NLA_BINARY },
10342         [NFTA_DATA_VERDICT]     = { .type = NLA_NESTED },
10343 };
10344
10345 /**
10346  *      nft_data_init - parse nf_tables data netlink attributes
10347  *
10348  *      @ctx: context of the expression using the data
10349  *      @data: destination struct nft_data
10350  *      @desc: data description
10351  *      @nla: netlink attribute containing data
10352  *
10353  *      Parse the netlink data attributes and initialize a struct nft_data.
10354  *      The type and length of data are returned in the data description.
10355  *
10356  *      The caller can indicate that it only wants to accept data of type
10357  *      NFT_DATA_VALUE by passing NULL for the ctx argument.
10358  */
10359 int nft_data_init(const struct nft_ctx *ctx, struct nft_data *data,
10360                   struct nft_data_desc *desc, const struct nlattr *nla)
10361 {
10362         struct nlattr *tb[NFTA_DATA_MAX + 1];
10363         int err;
10364
10365         if (WARN_ON_ONCE(!desc->size))
10366                 return -EINVAL;
10367
10368         err = nla_parse_nested_deprecated(tb, NFTA_DATA_MAX, nla,
10369                                           nft_data_policy, NULL);
10370         if (err < 0)
10371                 return err;
10372
10373         if (tb[NFTA_DATA_VALUE]) {
10374                 if (desc->type != NFT_DATA_VALUE)
10375                         return -EINVAL;
10376
10377                 err = nft_value_init(ctx, data, desc, tb[NFTA_DATA_VALUE]);
10378         } else if (tb[NFTA_DATA_VERDICT] && ctx != NULL) {
10379                 if (desc->type != NFT_DATA_VERDICT)
10380                         return -EINVAL;
10381
10382                 err = nft_verdict_init(ctx, data, desc, tb[NFTA_DATA_VERDICT]);
10383         } else {
10384                 err = -EINVAL;
10385         }
10386
10387         return err;
10388 }
10389 EXPORT_SYMBOL_GPL(nft_data_init);
10390
10391 /**
10392  *      nft_data_release - release a nft_data item
10393  *
10394  *      @data: struct nft_data to release
10395  *      @type: type of data
10396  *
10397  *      Release a nft_data item. NFT_DATA_VALUE types can be silently discarded,
10398  *      all others need to be released by calling this function.
10399  */
10400 void nft_data_release(const struct nft_data *data, enum nft_data_types type)
10401 {
10402         if (type < NFT_DATA_VERDICT)
10403                 return;
10404         switch (type) {
10405         case NFT_DATA_VERDICT:
10406                 return nft_verdict_uninit(data);
10407         default:
10408                 WARN_ON(1);
10409         }
10410 }
10411 EXPORT_SYMBOL_GPL(nft_data_release);
10412
10413 int nft_data_dump(struct sk_buff *skb, int attr, const struct nft_data *data,
10414                   enum nft_data_types type, unsigned int len)
10415 {
10416         struct nlattr *nest;
10417         int err;
10418
10419         nest = nla_nest_start_noflag(skb, attr);
10420         if (nest == NULL)
10421                 return -1;
10422
10423         switch (type) {
10424         case NFT_DATA_VALUE:
10425                 err = nft_value_dump(skb, data, len);
10426                 break;
10427         case NFT_DATA_VERDICT:
10428                 err = nft_verdict_dump(skb, NFTA_DATA_VERDICT, &data->verdict);
10429                 break;
10430         default:
10431                 err = -EINVAL;
10432                 WARN_ON(1);
10433         }
10434
10435         nla_nest_end(skb, nest);
10436         return err;
10437 }
10438 EXPORT_SYMBOL_GPL(nft_data_dump);
10439
10440 int __nft_release_basechain(struct nft_ctx *ctx)
10441 {
10442         struct nft_rule *rule, *nr;
10443
10444         if (WARN_ON(!nft_is_base_chain(ctx->chain)))
10445                 return 0;
10446
10447         nf_tables_unregister_hook(ctx->net, ctx->chain->table, ctx->chain);
10448         list_for_each_entry_safe(rule, nr, &ctx->chain->rules, list) {
10449                 list_del(&rule->list);
10450                 ctx->chain->use--;
10451                 nf_tables_rule_release(ctx, rule);
10452         }
10453         nft_chain_del(ctx->chain);
10454         ctx->table->use--;
10455         nf_tables_chain_destroy(ctx);
10456
10457         return 0;
10458 }
10459 EXPORT_SYMBOL_GPL(__nft_release_basechain);
10460
10461 static void __nft_release_hook(struct net *net, struct nft_table *table)
10462 {
10463         struct nft_flowtable *flowtable;
10464         struct nft_chain *chain;
10465
10466         list_for_each_entry(chain, &table->chains, list)
10467                 __nf_tables_unregister_hook(net, table, chain, true);
10468         list_for_each_entry(flowtable, &table->flowtables, list)
10469                 __nft_unregister_flowtable_net_hooks(net, &flowtable->hook_list,
10470                                                      true);
10471 }
10472
10473 static void __nft_release_hooks(struct net *net)
10474 {
10475         struct nftables_pernet *nft_net = nft_pernet(net);
10476         struct nft_table *table;
10477
10478         list_for_each_entry(table, &nft_net->tables, list) {
10479                 if (nft_table_has_owner(table))
10480                         continue;
10481
10482                 __nft_release_hook(net, table);
10483         }
10484 }
10485
10486 static void __nft_release_table(struct net *net, struct nft_table *table)
10487 {
10488         struct nft_flowtable *flowtable, *nf;
10489         struct nft_chain *chain, *nc;
10490         struct nft_object *obj, *ne;
10491         struct nft_rule *rule, *nr;
10492         struct nft_set *set, *ns;
10493         struct nft_ctx ctx = {
10494                 .net    = net,
10495                 .family = NFPROTO_NETDEV,
10496         };
10497
10498         ctx.family = table->family;
10499         ctx.table = table;
10500         list_for_each_entry(chain, &table->chains, list) {
10501                 ctx.chain = chain;
10502                 list_for_each_entry_safe(rule, nr, &chain->rules, list) {
10503                         list_del(&rule->list);
10504                         chain->use--;
10505                         nf_tables_rule_release(&ctx, rule);
10506                 }
10507         }
10508         list_for_each_entry_safe(flowtable, nf, &table->flowtables, list) {
10509                 list_del(&flowtable->list);
10510                 table->use--;
10511                 nf_tables_flowtable_destroy(flowtable);
10512         }
10513         list_for_each_entry_safe(set, ns, &table->sets, list) {
10514                 list_del(&set->list);
10515                 table->use--;
10516                 nft_set_destroy(&ctx, set);
10517         }
10518         list_for_each_entry_safe(obj, ne, &table->objects, list) {
10519                 nft_obj_del(obj);
10520                 table->use--;
10521                 nft_obj_destroy(&ctx, obj);
10522         }
10523         list_for_each_entry_safe(chain, nc, &table->chains, list) {
10524                 ctx.chain = chain;
10525                 nft_chain_del(chain);
10526                 table->use--;
10527                 nf_tables_chain_destroy(&ctx);
10528         }
10529         nf_tables_table_destroy(&ctx);
10530 }
10531
10532 static void __nft_release_tables(struct net *net)
10533 {
10534         struct nftables_pernet *nft_net = nft_pernet(net);
10535         struct nft_table *table, *nt;
10536
10537         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
10538                 if (nft_table_has_owner(table))
10539                         continue;
10540
10541                 list_del(&table->list);
10542
10543                 __nft_release_table(net, table);
10544         }
10545 }
10546
10547 static int nft_rcv_nl_event(struct notifier_block *this, unsigned long event,
10548                             void *ptr)
10549 {
10550         struct nft_table *table, *to_delete[8];
10551         struct nftables_pernet *nft_net;
10552         struct netlink_notify *n = ptr;
10553         struct net *net = n->net;
10554         unsigned int deleted;
10555         bool restart = false;
10556
10557         if (event != NETLINK_URELEASE || n->protocol != NETLINK_NETFILTER)
10558                 return NOTIFY_DONE;
10559
10560         nft_net = nft_pernet(net);
10561         deleted = 0;
10562         mutex_lock(&nft_net->commit_mutex);
10563         if (!list_empty(&nf_tables_destroy_list))
10564                 rcu_barrier();
10565 again:
10566         list_for_each_entry(table, &nft_net->tables, list) {
10567                 if (nft_table_has_owner(table) &&
10568                     n->portid == table->nlpid) {
10569                         __nft_release_hook(net, table);
10570                         list_del_rcu(&table->list);
10571                         to_delete[deleted++] = table;
10572                         if (deleted >= ARRAY_SIZE(to_delete))
10573                                 break;
10574                 }
10575         }
10576         if (deleted) {
10577                 restart = deleted >= ARRAY_SIZE(to_delete);
10578                 synchronize_rcu();
10579                 while (deleted)
10580                         __nft_release_table(net, to_delete[--deleted]);
10581
10582                 if (restart)
10583                         goto again;
10584         }
10585         mutex_unlock(&nft_net->commit_mutex);
10586
10587         return NOTIFY_DONE;
10588 }
10589
10590 static struct notifier_block nft_nl_notifier = {
10591         .notifier_call  = nft_rcv_nl_event,
10592 };
10593
10594 static int __net_init nf_tables_init_net(struct net *net)
10595 {
10596         struct nftables_pernet *nft_net = nft_pernet(net);
10597
10598         INIT_LIST_HEAD(&nft_net->tables);
10599         INIT_LIST_HEAD(&nft_net->commit_list);
10600         INIT_LIST_HEAD(&nft_net->module_list);
10601         INIT_LIST_HEAD(&nft_net->notify_list);
10602         mutex_init(&nft_net->commit_mutex);
10603         nft_net->base_seq = 1;
10604
10605         return 0;
10606 }
10607
10608 static void __net_exit nf_tables_pre_exit_net(struct net *net)
10609 {
10610         struct nftables_pernet *nft_net = nft_pernet(net);
10611
10612         mutex_lock(&nft_net->commit_mutex);
10613         __nft_release_hooks(net);
10614         mutex_unlock(&nft_net->commit_mutex);
10615 }
10616
10617 static void __net_exit nf_tables_exit_net(struct net *net)
10618 {
10619         struct nftables_pernet *nft_net = nft_pernet(net);
10620
10621         mutex_lock(&nft_net->commit_mutex);
10622         if (!list_empty(&nft_net->commit_list) ||
10623             !list_empty(&nft_net->module_list))
10624                 __nf_tables_abort(net, NFNL_ABORT_NONE);
10625         __nft_release_tables(net);
10626         mutex_unlock(&nft_net->commit_mutex);
10627         WARN_ON_ONCE(!list_empty(&nft_net->tables));
10628         WARN_ON_ONCE(!list_empty(&nft_net->module_list));
10629         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
10630 }
10631
10632 static struct pernet_operations nf_tables_net_ops = {
10633         .init           = nf_tables_init_net,
10634         .pre_exit       = nf_tables_pre_exit_net,
10635         .exit           = nf_tables_exit_net,
10636         .id             = &nf_tables_net_id,
10637         .size           = sizeof(struct nftables_pernet),
10638 };
10639
10640 static int __init nf_tables_module_init(void)
10641 {
10642         int err;
10643
10644         err = register_pernet_subsys(&nf_tables_net_ops);
10645         if (err < 0)
10646                 return err;
10647
10648         err = nft_chain_filter_init();
10649         if (err < 0)
10650                 goto err_chain_filter;
10651
10652         err = nf_tables_core_module_init();
10653         if (err < 0)
10654                 goto err_core_module;
10655
10656         err = register_netdevice_notifier(&nf_tables_flowtable_notifier);
10657         if (err < 0)
10658                 goto err_netdev_notifier;
10659
10660         err = rhltable_init(&nft_objname_ht, &nft_objname_ht_params);
10661         if (err < 0)
10662                 goto err_rht_objname;
10663
10664         err = nft_offload_init();
10665         if (err < 0)
10666                 goto err_offload;
10667
10668         err = netlink_register_notifier(&nft_nl_notifier);
10669         if (err < 0)
10670                 goto err_netlink_notifier;
10671
10672         /* must be last */
10673         err = nfnetlink_subsys_register(&nf_tables_subsys);
10674         if (err < 0)
10675                 goto err_nfnl_subsys;
10676
10677         nft_chain_route_init();
10678
10679         return err;
10680
10681 err_nfnl_subsys:
10682         netlink_unregister_notifier(&nft_nl_notifier);
10683 err_netlink_notifier:
10684         nft_offload_exit();
10685 err_offload:
10686         rhltable_destroy(&nft_objname_ht);
10687 err_rht_objname:
10688         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
10689 err_netdev_notifier:
10690         nf_tables_core_module_exit();
10691 err_core_module:
10692         nft_chain_filter_fini();
10693 err_chain_filter:
10694         unregister_pernet_subsys(&nf_tables_net_ops);
10695         return err;
10696 }
10697
10698 static void __exit nf_tables_module_exit(void)
10699 {
10700         nfnetlink_subsys_unregister(&nf_tables_subsys);
10701         netlink_unregister_notifier(&nft_nl_notifier);
10702         nft_offload_exit();
10703         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
10704         nft_chain_filter_fini();
10705         nft_chain_route_fini();
10706         unregister_pernet_subsys(&nf_tables_net_ops);
10707         cancel_work_sync(&trans_destroy_work);
10708         rcu_barrier();
10709         rhltable_destroy(&nft_objname_ht);
10710         nf_tables_core_module_exit();
10711 }
10712
10713 module_init(nf_tables_module_init);
10714 module_exit(nf_tables_module_exit);
10715
10716 MODULE_LICENSE("GPL");
10717 MODULE_AUTHOR("Patrick McHardy <kaber@trash.net>");
10718 MODULE_ALIAS_NFNL_SUBSYS(NFNL_SUBSYS_NFTABLES);