net/netfilter/nf_tables_api.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2007-2009 Patrick McHardy <kaber@trash.net>
   4  *
   5  * Development of this code funded by Astaro AG (http://www.astaro.com/)
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/init.h>
  10 #include <linux/list.h>
  11 #include <linux/skbuff.h>
  12 #include <linux/netlink.h>
  13 #include <linux/vmalloc.h>
  14 #include <linux/rhashtable.h>
  15 #include <linux/audit.h>
  16 #include <linux/netfilter.h>
  17 #include <linux/netfilter/nfnetlink.h>
  18 #include <linux/netfilter/nf_tables.h>
  19 #include <net/netfilter/nf_flow_table.h>
  20 #include <net/netfilter/nf_tables_core.h>
  21 #include <net/netfilter/nf_tables.h>
  22 #include <net/netfilter/nf_tables_offload.h>
  23 #include <net/net_namespace.h>
  24 #include <net/sock.h>
  25
  26 #define NFT_MODULE_AUTOLOAD_LIMIT (MODULE_NAME_LEN - sizeof("nft-expr-255-"))
  27
  28 unsigned int nf_tables_net_id __read_mostly;
  29
  30 static LIST_HEAD(nf_tables_expressions);
  31 static LIST_HEAD(nf_tables_objects);
  32 static LIST_HEAD(nf_tables_flowtables);
  33 static LIST_HEAD(nf_tables_destroy_list);
  34 static DEFINE_SPINLOCK(nf_tables_destroy_list_lock);
  35
  36 enum {
  37         NFT_VALIDATE_SKIP       = 0,
  38         NFT_VALIDATE_NEED,
  39         NFT_VALIDATE_DO,
  40 };
  41
  42 static struct rhltable nft_objname_ht;
  43
  44 static u32 nft_chain_hash(const void *data, u32 len, u32 seed);
  45 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed);
  46 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *, const void *);
  47
  48 static u32 nft_objname_hash(const void *data, u32 len, u32 seed);
  49 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed);
  50 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *, const void *);
  51
  52 static const struct rhashtable_params nft_chain_ht_params = {
  53         .head_offset            = offsetof(struct nft_chain, rhlhead),
  54         .key_offset             = offsetof(struct nft_chain, name),
  55         .hashfn                 = nft_chain_hash,
  56         .obj_hashfn             = nft_chain_hash_obj,
  57         .obj_cmpfn              = nft_chain_hash_cmp,
  58         .automatic_shrinking    = true,
  59 };
  60
  61 static const struct rhashtable_params nft_objname_ht_params = {
  62         .head_offset            = offsetof(struct nft_object, rhlhead),
  63         .key_offset             = offsetof(struct nft_object, key),
  64         .hashfn                 = nft_objname_hash,
  65         .obj_hashfn             = nft_objname_hash_obj,
  66         .obj_cmpfn              = nft_objname_hash_cmp,
  67         .automatic_shrinking    = true,
  68 };
  69
  70 struct nft_audit_data {
  71         struct nft_table *table;
  72         int entries;
  73         int op;
  74         struct list_head list;
  75 };
  76
  77 static const u8 nft2audit_op[NFT_MSG_MAX] = { // enum nf_tables_msg_types
  78         [NFT_MSG_NEWTABLE]      = AUDIT_NFT_OP_TABLE_REGISTER,
  79         [NFT_MSG_GETTABLE]      = AUDIT_NFT_OP_INVALID,
  80         [NFT_MSG_DELTABLE]      = AUDIT_NFT_OP_TABLE_UNREGISTER,
  81         [NFT_MSG_NEWCHAIN]      = AUDIT_NFT_OP_CHAIN_REGISTER,
  82         [NFT_MSG_GETCHAIN]      = AUDIT_NFT_OP_INVALID,
  83         [NFT_MSG_DELCHAIN]      = AUDIT_NFT_OP_CHAIN_UNREGISTER,
  84         [NFT_MSG_NEWRULE]       = AUDIT_NFT_OP_RULE_REGISTER,
  85         [NFT_MSG_GETRULE]       = AUDIT_NFT_OP_INVALID,
  86         [NFT_MSG_DELRULE]       = AUDIT_NFT_OP_RULE_UNREGISTER,
  87         [NFT_MSG_NEWSET]        = AUDIT_NFT_OP_SET_REGISTER,
  88         [NFT_MSG_GETSET]        = AUDIT_NFT_OP_INVALID,
  89         [NFT_MSG_DELSET]        = AUDIT_NFT_OP_SET_UNREGISTER,
  90         [NFT_MSG_NEWSETELEM]    = AUDIT_NFT_OP_SETELEM_REGISTER,
  91         [NFT_MSG_GETSETELEM]    = AUDIT_NFT_OP_INVALID,
  92         [NFT_MSG_DELSETELEM]    = AUDIT_NFT_OP_SETELEM_UNREGISTER,
  93         [NFT_MSG_NEWGEN]        = AUDIT_NFT_OP_GEN_REGISTER,
  94         [NFT_MSG_GETGEN]        = AUDIT_NFT_OP_INVALID,
  95         [NFT_MSG_TRACE]         = AUDIT_NFT_OP_INVALID,
  96         [NFT_MSG_NEWOBJ]        = AUDIT_NFT_OP_OBJ_REGISTER,
  97         [NFT_MSG_GETOBJ]        = AUDIT_NFT_OP_INVALID,
  98         [NFT_MSG_DELOBJ]        = AUDIT_NFT_OP_OBJ_UNREGISTER,
  99         [NFT_MSG_GETOBJ_RESET]  = AUDIT_NFT_OP_OBJ_RESET,
 100         [NFT_MSG_NEWFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_REGISTER,
 101         [NFT_MSG_GETFLOWTABLE]  = AUDIT_NFT_OP_INVALID,
 102         [NFT_MSG_DELFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_UNREGISTER,
 103 };
 104
 105 static void nft_validate_state_update(struct net *net, u8 new_validate_state)
 106 {
 107         struct nftables_pernet *nft_net = nft_pernet(net);
 108
 109         switch (nft_net->validate_state) {
 110         case NFT_VALIDATE_SKIP:
 111                 WARN_ON_ONCE(new_validate_state == NFT_VALIDATE_DO);
 112                 break;
 113         case NFT_VALIDATE_NEED:
 114                 break;
 115         case NFT_VALIDATE_DO:
 116                 if (new_validate_state == NFT_VALIDATE_NEED)
 117                         return;
 118         }
 119
 120         nft_net->validate_state = new_validate_state;
 121 }
 122 static void nf_tables_trans_destroy_work(struct work_struct *w);
 123 static DECLARE_WORK(trans_destroy_work, nf_tables_trans_destroy_work);
 124
 125 static void nft_ctx_init(struct nft_ctx *ctx,
 126                          struct net *net,
 127                          const struct sk_buff *skb,
 128                          const struct nlmsghdr *nlh,
 129                          u8 family,
 130                          struct nft_table *table,
 131                          struct nft_chain *chain,
 132                          const struct nlattr * const *nla)
 133 {
 134         ctx->net        = net;
 135         ctx->family     = family;
 136         ctx->level      = 0;
 137         ctx->table      = table;
 138         ctx->chain      = chain;
 139         ctx->nla        = nla;
 140         ctx->portid     = NETLINK_CB(skb).portid;
 141         ctx->report     = nlmsg_report(nlh);
 142         ctx->flags      = nlh->nlmsg_flags;
 143         ctx->seq        = nlh->nlmsg_seq;
 144 }
 145
 146 static struct nft_trans *nft_trans_alloc_gfp(const struct nft_ctx *ctx,
 147                                              int msg_type, u32 size, gfp_t gfp)
 148 {
 149         struct nft_trans *trans;
 150
 151         trans = kzalloc(sizeof(struct nft_trans) + size, gfp);
 152         if (trans == NULL)
 153                 return NULL;
 154
 155         INIT_LIST_HEAD(&trans->list);
 156         trans->msg_type = msg_type;
 157         trans->ctx      = *ctx;
 158
 159         return trans;
 160 }
 161
 162 static struct nft_trans *nft_trans_alloc(const struct nft_ctx *ctx,
 163                                          int msg_type, u32 size)
 164 {
 165         return nft_trans_alloc_gfp(ctx, msg_type, size, GFP_KERNEL);
 166 }
 167
 168 static void nft_trans_destroy(struct nft_trans *trans)
 169 {
 170         list_del(&trans->list);
 171         kfree(trans);
 172 }
 173
 174 static void nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set)
 175 {
 176         struct nftables_pernet *nft_net;
 177         struct net *net = ctx->net;
 178         struct nft_trans *trans;
 179
 180         if (!nft_set_is_anonymous(set))
 181                 return;
 182
 183         nft_net = nft_pernet(net);
 184         list_for_each_entry_reverse(trans, &nft_net->commit_list, list) {
 185                 switch (trans->msg_type) {
 186                 case NFT_MSG_NEWSET:
 187                         if (nft_trans_set(trans) == set)
 188                                 nft_trans_set_bound(trans) = true;
 189                         break;
 190                 case NFT_MSG_NEWSETELEM:
 191                         if (nft_trans_elem_set(trans) == set)
 192                                 nft_trans_elem_set_bound(trans) = true;
 193                         break;
 194                 }
 195         }
 196 }
 197
 198 static int nft_netdev_register_hooks(struct net *net,
 199                                      struct list_head *hook_list)
 200 {
 201         struct nft_hook *hook;
 202         int err, j;
 203
 204         j = 0;
 205         list_for_each_entry(hook, hook_list, list) {
 206                 err = nf_register_net_hook(net, &hook->ops);
 207                 if (err < 0)
 208                         goto err_register;
 209
 210                 j++;
 211         }
 212         return 0;
 213
 214 err_register:
 215         list_for_each_entry(hook, hook_list, list) {
 216                 if (j-- <= 0)
 217                         break;
 218
 219                 nf_unregister_net_hook(net, &hook->ops);
 220         }
 221         return err;
 222 }
 223
 224 static void nft_netdev_unregister_hooks(struct net *net,
 225                                         struct list_head *hook_list,
 226                                         bool release_netdev)
 227 {
 228         struct nft_hook *hook, *next;
 229
 230         list_for_each_entry_safe(hook, next, hook_list, list) {
 231                 nf_unregister_net_hook(net, &hook->ops);
 232                 if (release_netdev) {
 233                         list_del(&hook->list);
 234                         kfree_rcu(hook, rcu);
 235                 }
 236         }
 237 }
 238
 239 static int nf_tables_register_hook(struct net *net,
 240                                    const struct nft_table *table,
 241                                    struct nft_chain *chain)
 242 {
 243         struct nft_base_chain *basechain;
 244         const struct nf_hook_ops *ops;
 245
 246         if (table->flags & NFT_TABLE_F_DORMANT ||
 247             !nft_is_base_chain(chain))
 248                 return 0;
 249
 250         basechain = nft_base_chain(chain);
 251         ops = &basechain->ops;
 252
 253         if (basechain->type->ops_register)
 254                 return basechain->type->ops_register(net, ops);
 255
 256         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 257                 return nft_netdev_register_hooks(net, &basechain->hook_list);
 258
 259         return nf_register_net_hook(net, &basechain->ops);
 260 }
 261
 262 static void __nf_tables_unregister_hook(struct net *net,
 263                                         const struct nft_table *table,
 264                                         struct nft_chain *chain,
 265                                         bool release_netdev)
 266 {
 267         struct nft_base_chain *basechain;
 268         const struct nf_hook_ops *ops;
 269
 270         if (table->flags & NFT_TABLE_F_DORMANT ||
 271             !nft_is_base_chain(chain))
 272                 return;
 273         basechain = nft_base_chain(chain);
 274         ops = &basechain->ops;
 275
 276         if (basechain->type->ops_unregister)
 277                 return basechain->type->ops_unregister(net, ops);
 278
 279         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 280                 nft_netdev_unregister_hooks(net, &basechain->hook_list,
 281                                             release_netdev);
 282         else
 283                 nf_unregister_net_hook(net, &basechain->ops);
 284 }
 285
 286 static void nf_tables_unregister_hook(struct net *net,
 287                                       const struct nft_table *table,
 288                                       struct nft_chain *chain)
 289 {
 290         return __nf_tables_unregister_hook(net, table, chain, false);
 291 }
 292
 293 static void nft_trans_commit_list_add_tail(struct net *net, struct nft_trans *trans)
 294 {
 295         struct nftables_pernet *nft_net = nft_pernet(net);
 296
 297         list_add_tail(&trans->list, &nft_net->commit_list);
 298 }
 299
 300 static int nft_trans_table_add(struct nft_ctx *ctx, int msg_type)
 301 {
 302         struct nft_trans *trans;
 303
 304         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_table));
 305         if (trans == NULL)
 306                 return -ENOMEM;
 307
 308         if (msg_type == NFT_MSG_NEWTABLE)
 309                 nft_activate_next(ctx->net, ctx->table);
 310
 311         nft_trans_commit_list_add_tail(ctx->net, trans);
 312         return 0;
 313 }
 314
 315 static int nft_deltable(struct nft_ctx *ctx)
 316 {
 317         int err;
 318
 319         err = nft_trans_table_add(ctx, NFT_MSG_DELTABLE);
 320         if (err < 0)
 321                 return err;
 322
 323         nft_deactivate_next(ctx->net, ctx->table);
 324         return err;
 325 }
 326
 327 static struct nft_trans *nft_trans_chain_add(struct nft_ctx *ctx, int msg_type)
 328 {
 329         struct nft_trans *trans;
 330
 331         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_chain));
 332         if (trans == NULL)
 333                 return ERR_PTR(-ENOMEM);
 334
 335         if (msg_type == NFT_MSG_NEWCHAIN) {
 336                 nft_activate_next(ctx->net, ctx->chain);
 337
 338                 if (ctx->nla[NFTA_CHAIN_ID]) {
 339                         nft_trans_chain_id(trans) =
 340                                 ntohl(nla_get_be32(ctx->nla[NFTA_CHAIN_ID]));
 341                 }
 342         }
 343
 344         nft_trans_commit_list_add_tail(ctx->net, trans);
 345         return trans;
 346 }
 347
 348 static int nft_delchain(struct nft_ctx *ctx)
 349 {
 350         struct nft_trans *trans;
 351
 352         trans = nft_trans_chain_add(ctx, NFT_MSG_DELCHAIN);
 353         if (IS_ERR(trans))
 354                 return PTR_ERR(trans);
 355
 356         ctx->table->use--;
 357         nft_deactivate_next(ctx->net, ctx->chain);
 358
 359         return 0;
 360 }
 361
 362 static void nft_rule_expr_activate(const struct nft_ctx *ctx,
 363                                    struct nft_rule *rule)
 364 {
 365         struct nft_expr *expr;
 366
 367         expr = nft_expr_first(rule);
 368         while (nft_expr_more(rule, expr)) {
 369                 if (expr->ops->activate)
 370                         expr->ops->activate(ctx, expr);
 371
 372                 expr = nft_expr_next(expr);
 373         }
 374 }
 375
 376 static void nft_rule_expr_deactivate(const struct nft_ctx *ctx,
 377                                      struct nft_rule *rule,
 378                                      enum nft_trans_phase phase)
 379 {
 380         struct nft_expr *expr;
 381
 382         expr = nft_expr_first(rule);
 383         while (nft_expr_more(rule, expr)) {
 384                 if (expr->ops->deactivate)
 385                         expr->ops->deactivate(ctx, expr, phase);
 386
 387                 expr = nft_expr_next(expr);
 388         }
 389 }
 390
 391 static int
 392 nf_tables_delrule_deactivate(struct nft_ctx *ctx, struct nft_rule *rule)
 393 {
 394         /* You cannot delete the same rule twice */
 395         if (nft_is_active_next(ctx->net, rule)) {
 396                 nft_deactivate_next(ctx->net, rule);
 397                 ctx->chain->use--;
 398                 return 0;
 399         }
 400         return -ENOENT;
 401 }
 402
 403 static struct nft_trans *nft_trans_rule_add(struct nft_ctx *ctx, int msg_type,
 404                                             struct nft_rule *rule)
 405 {
 406         struct nft_trans *trans;
 407
 408         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_rule));
 409         if (trans == NULL)
 410                 return NULL;
 411
 412         if (msg_type == NFT_MSG_NEWRULE && ctx->nla[NFTA_RULE_ID] != NULL) {
 413                 nft_trans_rule_id(trans) =
 414                         ntohl(nla_get_be32(ctx->nla[NFTA_RULE_ID]));
 415         }
 416         nft_trans_rule(trans) = rule;
 417         nft_trans_commit_list_add_tail(ctx->net, trans);
 418
 419         return trans;
 420 }
 421
 422 static int nft_delrule(struct nft_ctx *ctx, struct nft_rule *rule)
 423 {
 424         struct nft_flow_rule *flow;
 425         struct nft_trans *trans;
 426         int err;
 427
 428         trans = nft_trans_rule_add(ctx, NFT_MSG_DELRULE, rule);
 429         if (trans == NULL)
 430                 return -ENOMEM;
 431
 432         if (ctx->chain->flags & NFT_CHAIN_HW_OFFLOAD) {
 433                 flow = nft_flow_rule_create(ctx->net, rule);
 434                 if (IS_ERR(flow)) {
 435                         nft_trans_destroy(trans);
 436                         return PTR_ERR(flow);
 437                 }
 438
 439                 nft_trans_flow_rule(trans) = flow;
 440         }
 441
 442         err = nf_tables_delrule_deactivate(ctx, rule);
 443         if (err < 0) {
 444                 nft_trans_destroy(trans);
 445                 return err;
 446         }
 447         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_PREPARE);
 448
 449         return 0;
 450 }
 451
 452 static int nft_delrule_by_chain(struct nft_ctx *ctx)
 453 {
 454         struct nft_rule *rule;
 455         int err;
 456
 457         list_for_each_entry(rule, &ctx->chain->rules, list) {
 458                 if (!nft_is_active_next(ctx->net, rule))
 459                         continue;
 460
 461                 err = nft_delrule(ctx, rule);
 462                 if (err < 0)
 463                         return err;
 464         }
 465         return 0;
 466 }
 467
 468 static int nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
 469                              struct nft_set *set)
 470 {
 471         struct nft_trans *trans;
 472
 473         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_set));
 474         if (trans == NULL)
 475                 return -ENOMEM;
 476
 477         if (msg_type == NFT_MSG_NEWSET && ctx->nla[NFTA_SET_ID] != NULL) {
 478                 nft_trans_set_id(trans) =
 479                         ntohl(nla_get_be32(ctx->nla[NFTA_SET_ID]));
 480                 nft_activate_next(ctx->net, set);
 481         }
 482         nft_trans_set(trans) = set;
 483         nft_trans_commit_list_add_tail(ctx->net, trans);
 484
 485         return 0;
 486 }
 487
 488 static int nft_delset(const struct nft_ctx *ctx, struct nft_set *set)
 489 {
 490         int err;
 491
 492         err = nft_trans_set_add(ctx, NFT_MSG_DELSET, set);
 493         if (err < 0)
 494                 return err;
 495
 496         nft_deactivate_next(ctx->net, set);
 497         ctx->table->use--;
 498
 499         return err;
 500 }
 501
 502 static int nft_trans_obj_add(struct nft_ctx *ctx, int msg_type,
 503                              struct nft_object *obj)
 504 {
 505         struct nft_trans *trans;
 506
 507         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_obj));
 508         if (trans == NULL)
 509                 return -ENOMEM;
 510
 511         if (msg_type == NFT_MSG_NEWOBJ)
 512                 nft_activate_next(ctx->net, obj);
 513
 514         nft_trans_obj(trans) = obj;
 515         nft_trans_commit_list_add_tail(ctx->net, trans);
 516
 517         return 0;
 518 }
 519
 520 static int nft_delobj(struct nft_ctx *ctx, struct nft_object *obj)
 521 {
 522         int err;
 523
 524         err = nft_trans_obj_add(ctx, NFT_MSG_DELOBJ, obj);
 525         if (err < 0)
 526                 return err;
 527
 528         nft_deactivate_next(ctx->net, obj);
 529         ctx->table->use--;
 530
 531         return err;
 532 }
 533
 534 static int nft_trans_flowtable_add(struct nft_ctx *ctx, int msg_type,
 535                                    struct nft_flowtable *flowtable)
 536 {
 537         struct nft_trans *trans;
 538
 539         trans = nft_trans_alloc(ctx, msg_type,
 540                                 sizeof(struct nft_trans_flowtable));
 541         if (trans == NULL)
 542                 return -ENOMEM;
 543
 544         if (msg_type == NFT_MSG_NEWFLOWTABLE)
 545                 nft_activate_next(ctx->net, flowtable);
 546
 547         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
 548         nft_trans_flowtable(trans) = flowtable;
 549         nft_trans_commit_list_add_tail(ctx->net, trans);
 550
 551         return 0;
 552 }
 553
 554 static int nft_delflowtable(struct nft_ctx *ctx,
 555                             struct nft_flowtable *flowtable)
 556 {
 557         int err;
 558
 559         err = nft_trans_flowtable_add(ctx, NFT_MSG_DELFLOWTABLE, flowtable);
 560         if (err < 0)
 561                 return err;
 562
 563         nft_deactivate_next(ctx->net, flowtable);
 564         ctx->table->use--;
 565
 566         return err;
 567 }
 568
 569 static void __nft_reg_track_clobber(struct nft_regs_track *track, u8 dreg)
 570 {
 571         int i;
 572
 573         for (i = track->regs[dreg].num_reg; i > 0; i--)
 574                 __nft_reg_track_cancel(track, dreg - i);
 575 }
 576
 577 static void __nft_reg_track_update(struct nft_regs_track *track,
 578                                    const struct nft_expr *expr,
 579                                    u8 dreg, u8 num_reg)
 580 {
 581         track->regs[dreg].selector = expr;
 582         track->regs[dreg].bitwise = NULL;
 583         track->regs[dreg].num_reg = num_reg;
 584 }
 585
 586 void nft_reg_track_update(struct nft_regs_track *track,
 587                           const struct nft_expr *expr, u8 dreg, u8 len)
 588 {
 589         unsigned int regcount;
 590         int i;
 591
 592         __nft_reg_track_clobber(track, dreg);
 593
 594         regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
 595         for (i = 0; i < regcount; i++, dreg++)
 596                 __nft_reg_track_update(track, expr, dreg, i);
 597 }
 598 EXPORT_SYMBOL_GPL(nft_reg_track_update);
 599
 600 void nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg, u8 len)
 601 {
 602         unsigned int regcount;
 603         int i;
 604
 605         __nft_reg_track_clobber(track, dreg);
 606
 607         regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
 608         for (i = 0; i < regcount; i++, dreg++)
 609                 __nft_reg_track_cancel(track, dreg);
 610 }
 611 EXPORT_SYMBOL_GPL(nft_reg_track_cancel);
 612
 613 void __nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg)
 614 {
 615         track->regs[dreg].selector = NULL;
 616         track->regs[dreg].bitwise = NULL;
 617         track->regs[dreg].num_reg = 0;
 618 }
 619 EXPORT_SYMBOL_GPL(__nft_reg_track_cancel);
 620
 621 /*
 622  * Tables
 623  */
 624
 625 static struct nft_table *nft_table_lookup(const struct net *net,
 626                                           const struct nlattr *nla,
 627                                           u8 family, u8 genmask, u32 nlpid)
 628 {
 629         struct nftables_pernet *nft_net;
 630         struct nft_table *table;
 631
 632         if (nla == NULL)
 633                 return ERR_PTR(-EINVAL);
 634
 635         nft_net = nft_pernet(net);
 636         list_for_each_entry_rcu(table, &nft_net->tables, list,
 637                                 lockdep_is_held(&nft_net->commit_mutex)) {
 638                 if (!nla_strcmp(nla, table->name) &&
 639                     table->family == family &&
 640                     nft_active_genmask(table, genmask)) {
 641                         if (nft_table_has_owner(table) &&
 642                             nlpid && table->nlpid != nlpid)
 643                                 return ERR_PTR(-EPERM);
 644
 645                         return table;
 646                 }
 647         }
 648
 649         return ERR_PTR(-ENOENT);
 650 }
 651
 652 static struct nft_table *nft_table_lookup_byhandle(const struct net *net,
 653                                                    const struct nlattr *nla,
 654                                                    u8 genmask, u32 nlpid)
 655 {
 656         struct nftables_pernet *nft_net;
 657         struct nft_table *table;
 658
 659         nft_net = nft_pernet(net);
 660         list_for_each_entry(table, &nft_net->tables, list) {
 661                 if (be64_to_cpu(nla_get_be64(nla)) == table->handle &&
 662                     nft_active_genmask(table, genmask)) {
 663                         if (nft_table_has_owner(table) &&
 664                             nlpid && table->nlpid != nlpid)
 665                                 return ERR_PTR(-EPERM);
 666
 667                         return table;
 668                 }
 669         }
 670
 671         return ERR_PTR(-ENOENT);
 672 }
 673
 674 static inline u64 nf_tables_alloc_handle(struct nft_table *table)
 675 {
 676         return ++table->hgenerator;
 677 }
 678
 679 static const struct nft_chain_type *chain_type[NFPROTO_NUMPROTO][NFT_CHAIN_T_MAX];
 680
 681 static const struct nft_chain_type *
 682 __nft_chain_type_get(u8 family, enum nft_chain_types type)
 683 {
 684         if (family >= NFPROTO_NUMPROTO ||
 685             type >= NFT_CHAIN_T_MAX)
 686                 return NULL;
 687
 688         return chain_type[family][type];
 689 }
 690
 691 static const struct nft_chain_type *
 692 __nf_tables_chain_type_lookup(const struct nlattr *nla, u8 family)
 693 {
 694         const struct nft_chain_type *type;
 695         int i;
 696
 697         for (i = 0; i < NFT_CHAIN_T_MAX; i++) {
 698                 type = __nft_chain_type_get(family, i);
 699                 if (!type)
 700                         continue;
 701                 if (!nla_strcmp(nla, type->name))
 702                         return type;
 703         }
 704         return NULL;
 705 }
 706
 707 struct nft_module_request {
 708         struct list_head        list;
 709         char                    module[MODULE_NAME_LEN];
 710         bool                    done;
 711 };
 712
 713 #ifdef CONFIG_MODULES
 714 __printf(2, 3) int nft_request_module(struct net *net, const char *fmt,
 715                                       ...)
 716 {
 717         char module_name[MODULE_NAME_LEN];
 718         struct nftables_pernet *nft_net;
 719         struct nft_module_request *req;
 720         va_list args;
 721         int ret;
 722
 723         va_start(args, fmt);
 724         ret = vsnprintf(module_name, MODULE_NAME_LEN, fmt, args);
 725         va_end(args);
 726         if (ret >= MODULE_NAME_LEN)
 727                 return 0;
 728
 729         nft_net = nft_pernet(net);
 730         list_for_each_entry(req, &nft_net->module_list, list) {
 731                 if (!strcmp(req->module, module_name)) {
 732                         if (req->done)
 733                                 return 0;
 734
 735                         /* A request to load this module already exists. */
 736                         return -EAGAIN;
 737                 }
 738         }
 739
 740         req = kmalloc(sizeof(*req), GFP_KERNEL);
 741         if (!req)
 742                 return -ENOMEM;
 743
 744         req->done = false;
 745         strlcpy(req->module, module_name, MODULE_NAME_LEN);
 746         list_add_tail(&req->list, &nft_net->module_list);
 747
 748         return -EAGAIN;
 749 }
 750 EXPORT_SYMBOL_GPL(nft_request_module);
 751 #endif
 752
 753 static void lockdep_nfnl_nft_mutex_not_held(void)
 754 {
 755 #ifdef CONFIG_PROVE_LOCKING
 756         if (debug_locks)
 757                 WARN_ON_ONCE(lockdep_nfnl_is_held(NFNL_SUBSYS_NFTABLES));
 758 #endif
 759 }
 760
 761 static const struct nft_chain_type *
 762 nf_tables_chain_type_lookup(struct net *net, const struct nlattr *nla,
 763                             u8 family, bool autoload)
 764 {
 765         const struct nft_chain_type *type;
 766
 767         type = __nf_tables_chain_type_lookup(nla, family);
 768         if (type != NULL)
 769                 return type;
 770
 771         lockdep_nfnl_nft_mutex_not_held();
 772 #ifdef CONFIG_MODULES
 773         if (autoload) {
 774                 if (nft_request_module(net, "nft-chain-%u-%.*s", family,
 775                                        nla_len(nla),
 776                                        (const char *)nla_data(nla)) == -EAGAIN)
 777                         return ERR_PTR(-EAGAIN);
 778         }
 779 #endif
 780         return ERR_PTR(-ENOENT);
 781 }
 782
 783 static __be16 nft_base_seq(const struct net *net)
 784 {
 785         struct nftables_pernet *nft_net = nft_pernet(net);
 786
 787         return htons(nft_net->base_seq & 0xffff);
 788 }
 789
 790 static const struct nla_policy nft_table_policy[NFTA_TABLE_MAX + 1] = {
 791         [NFTA_TABLE_NAME]       = { .type = NLA_STRING,
 792                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
 793         [NFTA_TABLE_FLAGS]      = { .type = NLA_U32 },
 794         [NFTA_TABLE_HANDLE]     = { .type = NLA_U64 },
 795         [NFTA_TABLE_USERDATA]   = { .type = NLA_BINARY,
 796                                     .len = NFT_USERDATA_MAXLEN }
 797 };
 798
 799 static int nf_tables_fill_table_info(struct sk_buff *skb, struct net *net,
 800                                      u32 portid, u32 seq, int event, u32 flags,
 801                                      int family, const struct nft_table *table)
 802 {
 803         struct nlmsghdr *nlh;
 804
 805         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
 806         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
 807                            NFNETLINK_V0, nft_base_seq(net));
 808         if (!nlh)
 809                 goto nla_put_failure;
 810
 811         if (nla_put_string(skb, NFTA_TABLE_NAME, table->name) ||
 812             nla_put_be32(skb, NFTA_TABLE_FLAGS,
 813                          htonl(table->flags & NFT_TABLE_F_MASK)) ||
 814             nla_put_be32(skb, NFTA_TABLE_USE, htonl(table->use)) ||
 815             nla_put_be64(skb, NFTA_TABLE_HANDLE, cpu_to_be64(table->handle),
 816                          NFTA_TABLE_PAD))
 817                 goto nla_put_failure;
 818         if (nft_table_has_owner(table) &&
 819             nla_put_be32(skb, NFTA_TABLE_OWNER, htonl(table->nlpid)))
 820                 goto nla_put_failure;
 821
 822         if (table->udata) {
 823                 if (nla_put(skb, NFTA_TABLE_USERDATA, table->udlen, table->udata))
 824                         goto nla_put_failure;
 825         }
 826
 827         nlmsg_end(skb, nlh);
 828         return 0;
 829
 830 nla_put_failure:
 831         nlmsg_trim(skb, nlh);
 832         return -1;
 833 }
 834
 835 struct nftnl_skb_parms {
 836         bool report;
 837 };
 838 #define NFT_CB(skb)     (*(struct nftnl_skb_parms*)&((skb)->cb))
 839
 840 static void nft_notify_enqueue(struct sk_buff *skb, bool report,
 841                                struct list_head *notify_list)
 842 {
 843         NFT_CB(skb).report = report;
 844         list_add_tail(&skb->list, notify_list);
 845 }
 846
 847 static void nf_tables_table_notify(const struct nft_ctx *ctx, int event)
 848 {
 849         struct nftables_pernet *nft_net;
 850         struct sk_buff *skb;
 851         u16 flags = 0;
 852         int err;
 853
 854         if (!ctx->report &&
 855             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
 856                 return;
 857
 858         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
 859         if (skb == NULL)
 860                 goto err;
 861
 862         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
 863                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
 864
 865         err = nf_tables_fill_table_info(skb, ctx->net, ctx->portid, ctx->seq,
 866                                         event, flags, ctx->family, ctx->table);
 867         if (err < 0) {
 868                 kfree_skb(skb);
 869                 goto err;
 870         }
 871
 872         nft_net = nft_pernet(ctx->net);
 873         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
 874         return;
 875 err:
 876         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
 877 }
 878
 879 static int nf_tables_dump_tables(struct sk_buff *skb,
 880                                  struct netlink_callback *cb)
 881 {
 882         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
 883         struct nftables_pernet *nft_net;
 884         const struct nft_table *table;
 885         unsigned int idx = 0, s_idx = cb->args[0];
 886         struct net *net = sock_net(skb->sk);
 887         int family = nfmsg->nfgen_family;
 888
 889         rcu_read_lock();
 890         nft_net = nft_pernet(net);
 891         cb->seq = READ_ONCE(nft_net->base_seq);
 892
 893         list_for_each_entry_rcu(table, &nft_net->tables, list) {
 894                 if (family != NFPROTO_UNSPEC && family != table->family)
 895                         continue;
 896
 897                 if (idx < s_idx)
 898                         goto cont;
 899                 if (idx > s_idx)
 900                         memset(&cb->args[1], 0,
 901                                sizeof(cb->args) - sizeof(cb->args[0]));
 902                 if (!nft_is_active(net, table))
 903                         continue;
 904                 if (nf_tables_fill_table_info(skb, net,
 905                                               NETLINK_CB(cb->skb).portid,
 906                                               cb->nlh->nlmsg_seq,
 907                                               NFT_MSG_NEWTABLE, NLM_F_MULTI,
 908                                               table->family, table) < 0)
 909                         goto done;
 910
 911                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
 912 cont:
 913                 idx++;
 914         }
 915 done:
 916         rcu_read_unlock();
 917         cb->args[0] = idx;
 918         return skb->len;
 919 }
 920
 921 static int nft_netlink_dump_start_rcu(struct sock *nlsk, struct sk_buff *skb,
 922                                       const struct nlmsghdr *nlh,
 923                                       struct netlink_dump_control *c)
 924 {
 925         int err;
 926
 927         if (!try_module_get(THIS_MODULE))
 928                 return -EINVAL;
 929
 930         rcu_read_unlock();
 931         err = netlink_dump_start(nlsk, skb, nlh, c);
 932         rcu_read_lock();
 933         module_put(THIS_MODULE);
 934
 935         return err;
 936 }
 937
 938 /* called with rcu_read_lock held */
 939 static int nf_tables_gettable(struct sk_buff *skb, const struct nfnl_info *info,
 940                               const struct nlattr * const nla[])
 941 {
 942         struct netlink_ext_ack *extack = info->extack;
 943         u8 genmask = nft_genmask_cur(info->net);
 944         u8 family = info->nfmsg->nfgen_family;
 945         const struct nft_table *table;
 946         struct net *net = info->net;
 947         struct sk_buff *skb2;
 948         int err;
 949
 950         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
 951                 struct netlink_dump_control c = {
 952                         .dump = nf_tables_dump_tables,
 953                         .module = THIS_MODULE,
 954                 };
 955
 956                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
 957         }
 958
 959         table = nft_table_lookup(net, nla[NFTA_TABLE_NAME], family, genmask, 0);
 960         if (IS_ERR(table)) {
 961                 NL_SET_BAD_ATTR(extack, nla[NFTA_TABLE_NAME]);
 962                 return PTR_ERR(table);
 963         }
 964
 965         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
 966         if (!skb2)
 967                 return -ENOMEM;
 968
 969         err = nf_tables_fill_table_info(skb2, net, NETLINK_CB(skb).portid,
 970                                         info->nlh->nlmsg_seq, NFT_MSG_NEWTABLE,
 971                                         0, family, table);
 972         if (err < 0)
 973                 goto err_fill_table_info;
 974
 975         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
 976
 977 err_fill_table_info:
 978         kfree_skb(skb2);
 979         return err;
 980 }
 981
 982 static void nft_table_disable(struct net *net, struct nft_table *table, u32 cnt)
 983 {
 984         struct nft_chain *chain;
 985         u32 i = 0;
 986
 987         list_for_each_entry(chain, &table->chains, list) {
 988                 if (!nft_is_active_next(net, chain))
 989                         continue;
 990                 if (!nft_is_base_chain(chain))
 991                         continue;
 992
 993                 if (cnt && i++ == cnt)
 994                         break;
 995
 996                 nf_tables_unregister_hook(net, table, chain);
 997         }
 998 }
 999
1000 static int nf_tables_table_enable(struct net *net, struct nft_table *table)
1001 {
1002         struct nft_chain *chain;
1003         int err, i = 0;
1004
1005         list_for_each_entry(chain, &table->chains, list) {
1006                 if (!nft_is_active_next(net, chain))
1007                         continue;
1008                 if (!nft_is_base_chain(chain))
1009                         continue;
1010
1011                 err = nf_tables_register_hook(net, table, chain);
1012                 if (err < 0)
1013                         goto err_register_hooks;
1014
1015                 i++;
1016         }
1017         return 0;
1018
1019 err_register_hooks:
1020         if (i)
1021                 nft_table_disable(net, table, i);
1022         return err;
1023 }
1024
1025 static void nf_tables_table_disable(struct net *net, struct nft_table *table)
1026 {
1027         table->flags &= ~NFT_TABLE_F_DORMANT;
1028         nft_table_disable(net, table, 0);
1029         table->flags |= NFT_TABLE_F_DORMANT;
1030 }
1031
1032 #define __NFT_TABLE_F_INTERNAL          (NFT_TABLE_F_MASK + 1)
1033 #define __NFT_TABLE_F_WAS_DORMANT       (__NFT_TABLE_F_INTERNAL << 0)
1034 #define __NFT_TABLE_F_WAS_AWAKEN        (__NFT_TABLE_F_INTERNAL << 1)
1035 #define __NFT_TABLE_F_UPDATE            (__NFT_TABLE_F_WAS_DORMANT | \
1036                                          __NFT_TABLE_F_WAS_AWAKEN)
1037
1038 static int nf_tables_updtable(struct nft_ctx *ctx)
1039 {
1040         struct nft_trans *trans;
1041         u32 flags;
1042         int ret;
1043
1044         if (!ctx->nla[NFTA_TABLE_FLAGS])
1045                 return 0;
1046
1047         flags = ntohl(nla_get_be32(ctx->nla[NFTA_TABLE_FLAGS]));
1048         if (flags & ~NFT_TABLE_F_MASK)
1049                 return -EOPNOTSUPP;
1050
1051         if (flags == ctx->table->flags)
1052                 return 0;
1053
1054         if ((nft_table_has_owner(ctx->table) &&
1055              !(flags & NFT_TABLE_F_OWNER)) ||
1056             (!nft_table_has_owner(ctx->table) &&
1057              flags & NFT_TABLE_F_OWNER))
1058                 return -EOPNOTSUPP;
1059
1060         trans = nft_trans_alloc(ctx, NFT_MSG_NEWTABLE,
1061                                 sizeof(struct nft_trans_table));
1062         if (trans == NULL)
1063                 return -ENOMEM;
1064
1065         if ((flags & NFT_TABLE_F_DORMANT) &&
1066             !(ctx->table->flags & NFT_TABLE_F_DORMANT)) {
1067                 ctx->table->flags |= NFT_TABLE_F_DORMANT;
1068                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE))
1069                         ctx->table->flags |= __NFT_TABLE_F_WAS_AWAKEN;
1070         } else if (!(flags & NFT_TABLE_F_DORMANT) &&
1071                    ctx->table->flags & NFT_TABLE_F_DORMANT) {
1072                 ctx->table->flags &= ~NFT_TABLE_F_DORMANT;
1073                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE)) {
1074                         ret = nf_tables_table_enable(ctx->net, ctx->table);
1075                         if (ret < 0)
1076                                 goto err_register_hooks;
1077
1078                         ctx->table->flags |= __NFT_TABLE_F_WAS_DORMANT;
1079                 }
1080         }
1081
1082         nft_trans_table_update(trans) = true;
1083         nft_trans_commit_list_add_tail(ctx->net, trans);
1084
1085         return 0;
1086
1087 err_register_hooks:
1088         nft_trans_destroy(trans);
1089         return ret;
1090 }
1091
1092 static u32 nft_chain_hash(const void *data, u32 len, u32 seed)
1093 {
1094         const char *name = data;
1095
1096         return jhash(name, strlen(name), seed);
1097 }
1098
1099 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed)
1100 {
1101         const struct nft_chain *chain = data;
1102
1103         return nft_chain_hash(chain->name, 0, seed);
1104 }
1105
1106 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *arg,
1107                               const void *ptr)
1108 {
1109         const struct nft_chain *chain = ptr;
1110         const char *name = arg->key;
1111
1112         return strcmp(chain->name, name);
1113 }
1114
1115 static u32 nft_objname_hash(const void *data, u32 len, u32 seed)
1116 {
1117         const struct nft_object_hash_key *k = data;
1118
1119         seed ^= hash_ptr(k->table, 32);
1120
1121         return jhash(k->name, strlen(k->name), seed);
1122 }
1123
1124 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed)
1125 {
1126         const struct nft_object *obj = data;
1127
1128         return nft_objname_hash(&obj->key, 0, seed);
1129 }
1130
1131 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *arg,
1132                                 const void *ptr)
1133 {
1134         const struct nft_object_hash_key *k = arg->key;
1135         const struct nft_object *obj = ptr;
1136
1137         if (obj->key.table != k->table)
1138                 return -1;
1139
1140         return strcmp(obj->key.name, k->name);
1141 }
1142
1143 static bool nft_supported_family(u8 family)
1144 {
1145         return false
1146 #ifdef CONFIG_NF_TABLES_INET
1147                 || family == NFPROTO_INET
1148 #endif
1149 #ifdef CONFIG_NF_TABLES_IPV4
1150                 || family == NFPROTO_IPV4
1151 #endif
1152 #ifdef CONFIG_NF_TABLES_ARP
1153                 || family == NFPROTO_ARP
1154 #endif
1155 #ifdef CONFIG_NF_TABLES_NETDEV
1156                 || family == NFPROTO_NETDEV
1157 #endif
1158 #if IS_ENABLED(CONFIG_NF_TABLES_BRIDGE)
1159                 || family == NFPROTO_BRIDGE
1160 #endif
1161 #ifdef CONFIG_NF_TABLES_IPV6
1162                 || family == NFPROTO_IPV6
1163 #endif
1164                 ;
1165 }
1166
1167 static int nf_tables_newtable(struct sk_buff *skb, const struct nfnl_info *info,
1168                               const struct nlattr * const nla[])
1169 {
1170         struct nftables_pernet *nft_net = nft_pernet(info->net);
1171         struct netlink_ext_ack *extack = info->extack;
1172         u8 genmask = nft_genmask_next(info->net);
1173         u8 family = info->nfmsg->nfgen_family;
1174         struct net *net = info->net;
1175         const struct nlattr *attr;
1176         struct nft_table *table;
1177         struct nft_ctx ctx;
1178         u32 flags = 0;
1179         int err;
1180
1181         if (!nft_supported_family(family))
1182                 return -EOPNOTSUPP;
1183
1184         lockdep_assert_held(&nft_net->commit_mutex);
1185         attr = nla[NFTA_TABLE_NAME];
1186         table = nft_table_lookup(net, attr, family, genmask,
1187                                  NETLINK_CB(skb).portid);
1188         if (IS_ERR(table)) {
1189                 if (PTR_ERR(table) != -ENOENT)
1190                         return PTR_ERR(table);
1191         } else {
1192                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
1193                         NL_SET_BAD_ATTR(extack, attr);
1194                         return -EEXIST;
1195                 }
1196                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
1197                         return -EOPNOTSUPP;
1198
1199                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1200
1201                 return nf_tables_updtable(&ctx);
1202         }
1203
1204         if (nla[NFTA_TABLE_FLAGS]) {
1205                 flags = ntohl(nla_get_be32(nla[NFTA_TABLE_FLAGS]));
1206                 if (flags & ~NFT_TABLE_F_MASK)
1207                         return -EOPNOTSUPP;
1208         }
1209
1210         err = -ENOMEM;
1211         table = kzalloc(sizeof(*table), GFP_KERNEL_ACCOUNT);
1212         if (table == NULL)
1213                 goto err_kzalloc;
1214
1215         table->name = nla_strdup(attr, GFP_KERNEL_ACCOUNT);
1216         if (table->name == NULL)
1217                 goto err_strdup;
1218
1219         if (nla[NFTA_TABLE_USERDATA]) {
1220                 table->udata = nla_memdup(nla[NFTA_TABLE_USERDATA], GFP_KERNEL_ACCOUNT);
1221                 if (table->udata == NULL)
1222                         goto err_table_udata;
1223
1224                 table->udlen = nla_len(nla[NFTA_TABLE_USERDATA]);
1225         }
1226
1227         err = rhltable_init(&table->chains_ht, &nft_chain_ht_params);
1228         if (err)
1229                 goto err_chain_ht;
1230
1231         INIT_LIST_HEAD(&table->chains);
1232         INIT_LIST_HEAD(&table->sets);
1233         INIT_LIST_HEAD(&table->objects);
1234         INIT_LIST_HEAD(&table->flowtables);
1235         table->family = family;
1236         table->flags = flags;
1237         table->handle = ++nft_net->table_handle;
1238         if (table->flags & NFT_TABLE_F_OWNER)
1239                 table->nlpid = NETLINK_CB(skb).portid;
1240
1241         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1242         err = nft_trans_table_add(&ctx, NFT_MSG_NEWTABLE);
1243         if (err < 0)
1244                 goto err_trans;
1245
1246         list_add_tail_rcu(&table->list, &nft_net->tables);
1247         return 0;
1248 err_trans:
1249         rhltable_destroy(&table->chains_ht);
1250 err_chain_ht:
1251         kfree(table->udata);
1252 err_table_udata:
1253         kfree(table->name);
1254 err_strdup:
1255         kfree(table);
1256 err_kzalloc:
1257         return err;
1258 }
1259
1260 static int nft_flush_table(struct nft_ctx *ctx)
1261 {
1262         struct nft_flowtable *flowtable, *nft;
1263         struct nft_chain *chain, *nc;
1264         struct nft_object *obj, *ne;
1265         struct nft_set *set, *ns;
1266         int err;
1267
1268         list_for_each_entry(chain, &ctx->table->chains, list) {
1269                 if (!nft_is_active_next(ctx->net, chain))
1270                         continue;
1271
1272                 if (nft_chain_is_bound(chain))
1273                         continue;
1274
1275                 ctx->chain = chain;
1276
1277                 err = nft_delrule_by_chain(ctx);
1278                 if (err < 0)
1279                         goto out;
1280         }
1281
1282         list_for_each_entry_safe(set, ns, &ctx->table->sets, list) {
1283                 if (!nft_is_active_next(ctx->net, set))
1284                         continue;
1285
1286                 if (nft_set_is_anonymous(set) &&
1287                     !list_empty(&set->bindings))
1288                         continue;
1289
1290                 err = nft_delset(ctx, set);
1291                 if (err < 0)
1292                         goto out;
1293         }
1294
1295         list_for_each_entry_safe(flowtable, nft, &ctx->table->flowtables, list) {
1296                 if (!nft_is_active_next(ctx->net, flowtable))
1297                         continue;
1298
1299                 err = nft_delflowtable(ctx, flowtable);
1300                 if (err < 0)
1301                         goto out;
1302         }
1303
1304         list_for_each_entry_safe(obj, ne, &ctx->table->objects, list) {
1305                 if (!nft_is_active_next(ctx->net, obj))
1306                         continue;
1307
1308                 err = nft_delobj(ctx, obj);
1309                 if (err < 0)
1310                         goto out;
1311         }
1312
1313         list_for_each_entry_safe(chain, nc, &ctx->table->chains, list) {
1314                 if (!nft_is_active_next(ctx->net, chain))
1315                         continue;
1316
1317                 if (nft_chain_is_bound(chain))
1318                         continue;
1319
1320                 ctx->chain = chain;
1321
1322                 err = nft_delchain(ctx);
1323                 if (err < 0)
1324                         goto out;
1325         }
1326
1327         err = nft_deltable(ctx);
1328 out:
1329         return err;
1330 }
1331
1332 static int nft_flush(struct nft_ctx *ctx, int family)
1333 {
1334         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
1335         const struct nlattr * const *nla = ctx->nla;
1336         struct nft_table *table, *nt;
1337         int err = 0;
1338
1339         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
1340                 if (family != AF_UNSPEC && table->family != family)
1341                         continue;
1342
1343                 ctx->family = table->family;
1344
1345                 if (!nft_is_active_next(ctx->net, table))
1346                         continue;
1347
1348                 if (nft_table_has_owner(table) && table->nlpid != ctx->portid)
1349                         continue;
1350
1351                 if (nla[NFTA_TABLE_NAME] &&
1352                     nla_strcmp(nla[NFTA_TABLE_NAME], table->name) != 0)
1353                         continue;
1354
1355                 ctx->table = table;
1356
1357                 err = nft_flush_table(ctx);
1358                 if (err < 0)
1359                         goto out;
1360         }
1361 out:
1362         return err;
1363 }
1364
1365 static int nf_tables_deltable(struct sk_buff *skb, const struct nfnl_info *info,
1366                               const struct nlattr * const nla[])
1367 {
1368         struct netlink_ext_ack *extack = info->extack;
1369         u8 genmask = nft_genmask_next(info->net);
1370         u8 family = info->nfmsg->nfgen_family;
1371         struct net *net = info->net;
1372         const struct nlattr *attr;
1373         struct nft_table *table;
1374         struct nft_ctx ctx;
1375
1376         nft_ctx_init(&ctx, net, skb, info->nlh, 0, NULL, NULL, nla);
1377         if (family == AF_UNSPEC ||
1378             (!nla[NFTA_TABLE_NAME] && !nla[NFTA_TABLE_HANDLE]))
1379                 return nft_flush(&ctx, family);
1380
1381         if (nla[NFTA_TABLE_HANDLE]) {
1382                 attr = nla[NFTA_TABLE_HANDLE];
1383                 table = nft_table_lookup_byhandle(net, attr, genmask,
1384                                                   NETLINK_CB(skb).portid);
1385         } else {
1386                 attr = nla[NFTA_TABLE_NAME];
1387                 table = nft_table_lookup(net, attr, family, genmask,
1388                                          NETLINK_CB(skb).portid);
1389         }
1390
1391         if (IS_ERR(table)) {
1392                 NL_SET_BAD_ATTR(extack, attr);
1393                 return PTR_ERR(table);
1394         }
1395
1396         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
1397             table->use > 0)
1398                 return -EBUSY;
1399
1400         ctx.family = family;
1401         ctx.table = table;
1402
1403         return nft_flush_table(&ctx);
1404 }
1405
1406 static void nf_tables_table_destroy(struct nft_ctx *ctx)
1407 {
1408         if (WARN_ON(ctx->table->use > 0))
1409                 return;
1410
1411         rhltable_destroy(&ctx->table->chains_ht);
1412         kfree(ctx->table->name);
1413         kfree(ctx->table->udata);
1414         kfree(ctx->table);
1415 }
1416
1417 void nft_register_chain_type(const struct nft_chain_type *ctype)
1418 {
1419         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1420         if (WARN_ON(__nft_chain_type_get(ctype->family, ctype->type))) {
1421                 nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1422                 return;
1423         }
1424         chain_type[ctype->family][ctype->type] = ctype;
1425         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1426 }
1427 EXPORT_SYMBOL_GPL(nft_register_chain_type);
1428
1429 void nft_unregister_chain_type(const struct nft_chain_type *ctype)
1430 {
1431         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1432         chain_type[ctype->family][ctype->type] = NULL;
1433         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1434 }
1435 EXPORT_SYMBOL_GPL(nft_unregister_chain_type);
1436
1437 /*
1438  * Chains
1439  */
1440
1441 static struct nft_chain *
1442 nft_chain_lookup_byhandle(const struct nft_table *table, u64 handle, u8 genmask)
1443 {
1444         struct nft_chain *chain;
1445
1446         list_for_each_entry(chain, &table->chains, list) {
1447                 if (chain->handle == handle &&
1448                     nft_active_genmask(chain, genmask))
1449                         return chain;
1450         }
1451
1452         return ERR_PTR(-ENOENT);
1453 }
1454
1455 static bool lockdep_commit_lock_is_held(const struct net *net)
1456 {
1457 #ifdef CONFIG_PROVE_LOCKING
1458         struct nftables_pernet *nft_net = nft_pernet(net);
1459
1460         return lockdep_is_held(&nft_net->commit_mutex);
1461 #else
1462         return true;
1463 #endif
1464 }
1465
1466 static struct nft_chain *nft_chain_lookup(struct net *net,
1467                                           struct nft_table *table,
1468                                           const struct nlattr *nla, u8 genmask)
1469 {
1470         char search[NFT_CHAIN_MAXNAMELEN + 1];
1471         struct rhlist_head *tmp, *list;
1472         struct nft_chain *chain;
1473
1474         if (nla == NULL)
1475                 return ERR_PTR(-EINVAL);
1476
1477         nla_strscpy(search, nla, sizeof(search));
1478
1479         WARN_ON(!rcu_read_lock_held() &&
1480                 !lockdep_commit_lock_is_held(net));
1481
1482         chain = ERR_PTR(-ENOENT);
1483         rcu_read_lock();
1484         list = rhltable_lookup(&table->chains_ht, search, nft_chain_ht_params);
1485         if (!list)
1486                 goto out_unlock;
1487
1488         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
1489                 if (nft_active_genmask(chain, genmask))
1490                         goto out_unlock;
1491         }
1492         chain = ERR_PTR(-ENOENT);
1493 out_unlock:
1494         rcu_read_unlock();
1495         return chain;
1496 }
1497
1498 static const struct nla_policy nft_chain_policy[NFTA_CHAIN_MAX + 1] = {
1499         [NFTA_CHAIN_TABLE]      = { .type = NLA_STRING,
1500                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
1501         [NFTA_CHAIN_HANDLE]     = { .type = NLA_U64 },
1502         [NFTA_CHAIN_NAME]       = { .type = NLA_STRING,
1503                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
1504         [NFTA_CHAIN_HOOK]       = { .type = NLA_NESTED },
1505         [NFTA_CHAIN_POLICY]     = { .type = NLA_U32 },
1506         [NFTA_CHAIN_TYPE]       = { .type = NLA_STRING,
1507                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
1508         [NFTA_CHAIN_COUNTERS]   = { .type = NLA_NESTED },
1509         [NFTA_CHAIN_FLAGS]      = { .type = NLA_U32 },
1510         [NFTA_CHAIN_ID]         = { .type = NLA_U32 },
1511         [NFTA_CHAIN_USERDATA]   = { .type = NLA_BINARY,
1512                                     .len = NFT_USERDATA_MAXLEN },
1513 };
1514
1515 static const struct nla_policy nft_hook_policy[NFTA_HOOK_MAX + 1] = {
1516         [NFTA_HOOK_HOOKNUM]     = { .type = NLA_U32 },
1517         [NFTA_HOOK_PRIORITY]    = { .type = NLA_U32 },
1518         [NFTA_HOOK_DEV]         = { .type = NLA_STRING,
1519                                     .len = IFNAMSIZ - 1 },
1520 };
1521
1522 static int nft_dump_stats(struct sk_buff *skb, struct nft_stats __percpu *stats)
1523 {
1524         struct nft_stats *cpu_stats, total;
1525         struct nlattr *nest;
1526         unsigned int seq;
1527         u64 pkts, bytes;
1528         int cpu;
1529
1530         if (!stats)
1531                 return 0;
1532
1533         memset(&total, 0, sizeof(total));
1534         for_each_possible_cpu(cpu) {
1535                 cpu_stats = per_cpu_ptr(stats, cpu);
1536                 do {
1537                         seq = u64_stats_fetch_begin_irq(&cpu_stats->syncp);
1538                         pkts = cpu_stats->pkts;
1539                         bytes = cpu_stats->bytes;
1540                 } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, seq));
1541                 total.pkts += pkts;
1542                 total.bytes += bytes;
1543         }
1544         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_COUNTERS);
1545         if (nest == NULL)
1546                 goto nla_put_failure;
1547
1548         if (nla_put_be64(skb, NFTA_COUNTER_PACKETS, cpu_to_be64(total.pkts),
1549                          NFTA_COUNTER_PAD) ||
1550             nla_put_be64(skb, NFTA_COUNTER_BYTES, cpu_to_be64(total.bytes),
1551                          NFTA_COUNTER_PAD))
1552                 goto nla_put_failure;
1553
1554         nla_nest_end(skb, nest);
1555         return 0;
1556
1557 nla_put_failure:
1558         return -ENOSPC;
1559 }
1560
1561 static int nft_dump_basechain_hook(struct sk_buff *skb, int family,
1562                                    const struct nft_base_chain *basechain)
1563 {
1564         const struct nf_hook_ops *ops = &basechain->ops;
1565         struct nft_hook *hook, *first = NULL;
1566         struct nlattr *nest, *nest_devs;
1567         int n = 0;
1568
1569         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_HOOK);
1570         if (nest == NULL)
1571                 goto nla_put_failure;
1572         if (nla_put_be32(skb, NFTA_HOOK_HOOKNUM, htonl(ops->hooknum)))
1573                 goto nla_put_failure;
1574         if (nla_put_be32(skb, NFTA_HOOK_PRIORITY, htonl(ops->priority)))
1575                 goto nla_put_failure;
1576
1577         if (nft_base_chain_netdev(family, ops->hooknum)) {
1578                 nest_devs = nla_nest_start_noflag(skb, NFTA_HOOK_DEVS);
1579                 list_for_each_entry(hook, &basechain->hook_list, list) {
1580                         if (!first)
1581                                 first = hook;
1582
1583                         if (nla_put_string(skb, NFTA_DEVICE_NAME,
1584                                            hook->ops.dev->name))
1585                                 goto nla_put_failure;
1586                         n++;
1587                 }
1588                 nla_nest_end(skb, nest_devs);
1589
1590                 if (n == 1 &&
1591                     nla_put_string(skb, NFTA_HOOK_DEV, first->ops.dev->name))
1592                         goto nla_put_failure;
1593         }
1594         nla_nest_end(skb, nest);
1595
1596         return 0;
1597 nla_put_failure:
1598         return -1;
1599 }
1600
1601 static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net,
1602                                      u32 portid, u32 seq, int event, u32 flags,
1603                                      int family, const struct nft_table *table,
1604                                      const struct nft_chain *chain)
1605 {
1606         struct nlmsghdr *nlh;
1607
1608         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
1609         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
1610                            NFNETLINK_V0, nft_base_seq(net));
1611         if (!nlh)
1612                 goto nla_put_failure;
1613
1614         if (nla_put_string(skb, NFTA_CHAIN_TABLE, table->name))
1615                 goto nla_put_failure;
1616         if (nla_put_be64(skb, NFTA_CHAIN_HANDLE, cpu_to_be64(chain->handle),
1617                          NFTA_CHAIN_PAD))
1618                 goto nla_put_failure;
1619         if (nla_put_string(skb, NFTA_CHAIN_NAME, chain->name))
1620                 goto nla_put_failure;
1621
1622         if (nft_is_base_chain(chain)) {
1623                 const struct nft_base_chain *basechain = nft_base_chain(chain);
1624                 struct nft_stats __percpu *stats;
1625
1626                 if (nft_dump_basechain_hook(skb, family, basechain))
1627                         goto nla_put_failure;
1628
1629                 if (nla_put_be32(skb, NFTA_CHAIN_POLICY,
1630                                  htonl(basechain->policy)))
1631                         goto nla_put_failure;
1632
1633                 if (nla_put_string(skb, NFTA_CHAIN_TYPE, basechain->type->name))
1634                         goto nla_put_failure;
1635
1636                 stats = rcu_dereference_check(basechain->stats,
1637                                               lockdep_commit_lock_is_held(net));
1638                 if (nft_dump_stats(skb, stats))
1639                         goto nla_put_failure;
1640         }
1641
1642         if (chain->flags &&
1643             nla_put_be32(skb, NFTA_CHAIN_FLAGS, htonl(chain->flags)))
1644                 goto nla_put_failure;
1645
1646         if (nla_put_be32(skb, NFTA_CHAIN_USE, htonl(chain->use)))
1647                 goto nla_put_failure;
1648
1649         if (chain->udata &&
1650             nla_put(skb, NFTA_CHAIN_USERDATA, chain->udlen, chain->udata))
1651                 goto nla_put_failure;
1652
1653         nlmsg_end(skb, nlh);
1654         return 0;
1655
1656 nla_put_failure:
1657         nlmsg_trim(skb, nlh);
1658         return -1;
1659 }
1660
1661 static void nf_tables_chain_notify(const struct nft_ctx *ctx, int event)
1662 {
1663         struct nftables_pernet *nft_net;
1664         struct sk_buff *skb;
1665         u16 flags = 0;
1666         int err;
1667
1668         if (!ctx->report &&
1669             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
1670                 return;
1671
1672         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
1673         if (skb == NULL)
1674                 goto err;
1675
1676         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
1677                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
1678
1679         err = nf_tables_fill_chain_info(skb, ctx->net, ctx->portid, ctx->seq,
1680                                         event, flags, ctx->family, ctx->table,
1681                                         ctx->chain);
1682         if (err < 0) {
1683                 kfree_skb(skb);
1684                 goto err;
1685         }
1686
1687         nft_net = nft_pernet(ctx->net);
1688         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
1689         return;
1690 err:
1691         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
1692 }
1693
1694 static int nf_tables_dump_chains(struct sk_buff *skb,
1695                                  struct netlink_callback *cb)
1696 {
1697         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
1698         unsigned int idx = 0, s_idx = cb->args[0];
1699         struct net *net = sock_net(skb->sk);
1700         int family = nfmsg->nfgen_family;
1701         struct nftables_pernet *nft_net;
1702         const struct nft_table *table;
1703         const struct nft_chain *chain;
1704
1705         rcu_read_lock();
1706         nft_net = nft_pernet(net);
1707         cb->seq = READ_ONCE(nft_net->base_seq);
1708
1709         list_for_each_entry_rcu(table, &nft_net->tables, list) {
1710                 if (family != NFPROTO_UNSPEC && family != table->family)
1711                         continue;
1712
1713                 list_for_each_entry_rcu(chain, &table->chains, list) {
1714                         if (idx < s_idx)
1715                                 goto cont;
1716                         if (idx > s_idx)
1717                                 memset(&cb->args[1], 0,
1718                                        sizeof(cb->args) - sizeof(cb->args[0]));
1719                         if (!nft_is_active(net, chain))
1720                                 continue;
1721                         if (nf_tables_fill_chain_info(skb, net,
1722                                                       NETLINK_CB(cb->skb).portid,
1723                                                       cb->nlh->nlmsg_seq,
1724                                                       NFT_MSG_NEWCHAIN,
1725                                                       NLM_F_MULTI,
1726                                                       table->family, table,
1727                                                       chain) < 0)
1728                                 goto done;
1729
1730                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
1731 cont:
1732                         idx++;
1733                 }
1734         }
1735 done:
1736         rcu_read_unlock();
1737         cb->args[0] = idx;
1738         return skb->len;
1739 }
1740
1741 /* called with rcu_read_lock held */
1742 static int nf_tables_getchain(struct sk_buff *skb, const struct nfnl_info *info,
1743                               const struct nlattr * const nla[])
1744 {
1745         struct netlink_ext_ack *extack = info->extack;
1746         u8 genmask = nft_genmask_cur(info->net);
1747         u8 family = info->nfmsg->nfgen_family;
1748         const struct nft_chain *chain;
1749         struct net *net = info->net;
1750         struct nft_table *table;
1751         struct sk_buff *skb2;
1752         int err;
1753
1754         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
1755                 struct netlink_dump_control c = {
1756                         .dump = nf_tables_dump_chains,
1757                         .module = THIS_MODULE,
1758                 };
1759
1760                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
1761         }
1762
1763         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask, 0);
1764         if (IS_ERR(table)) {
1765                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
1766                 return PTR_ERR(table);
1767         }
1768
1769         chain = nft_chain_lookup(net, table, nla[NFTA_CHAIN_NAME], genmask);
1770         if (IS_ERR(chain)) {
1771                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
1772                 return PTR_ERR(chain);
1773         }
1774
1775         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
1776         if (!skb2)
1777                 return -ENOMEM;
1778
1779         err = nf_tables_fill_chain_info(skb2, net, NETLINK_CB(skb).portid,
1780                                         info->nlh->nlmsg_seq, NFT_MSG_NEWCHAIN,
1781                                         0, family, table, chain);
1782         if (err < 0)
1783                 goto err_fill_chain_info;
1784
1785         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
1786
1787 err_fill_chain_info:
1788         kfree_skb(skb2);
1789         return err;
1790 }
1791
1792 static const struct nla_policy nft_counter_policy[NFTA_COUNTER_MAX + 1] = {
1793         [NFTA_COUNTER_PACKETS]  = { .type = NLA_U64 },
1794         [NFTA_COUNTER_BYTES]    = { .type = NLA_U64 },
1795 };
1796
1797 static struct nft_stats __percpu *nft_stats_alloc(const struct nlattr *attr)
1798 {
1799         struct nlattr *tb[NFTA_COUNTER_MAX+1];
1800         struct nft_stats __percpu *newstats;
1801         struct nft_stats *stats;
1802         int err;
1803
1804         err = nla_parse_nested_deprecated(tb, NFTA_COUNTER_MAX, attr,
1805                                           nft_counter_policy, NULL);
1806         if (err < 0)
1807                 return ERR_PTR(err);
1808
1809         if (!tb[NFTA_COUNTER_BYTES] || !tb[NFTA_COUNTER_PACKETS])
1810                 return ERR_PTR(-EINVAL);
1811
1812         newstats = netdev_alloc_pcpu_stats(struct nft_stats);
1813         if (newstats == NULL)
1814                 return ERR_PTR(-ENOMEM);
1815
1816         /* Restore old counters on this cpu, no problem. Per-cpu statistics
1817          * are not exposed to userspace.
1818          */
1819         preempt_disable();
1820         stats = this_cpu_ptr(newstats);
1821         stats->bytes = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_BYTES]));
1822         stats->pkts = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_PACKETS]));
1823         preempt_enable();
1824
1825         return newstats;
1826 }
1827
1828 static void nft_chain_stats_replace(struct nft_trans *trans)
1829 {
1830         struct nft_base_chain *chain = nft_base_chain(trans->ctx.chain);
1831
1832         if (!nft_trans_chain_stats(trans))
1833                 return;
1834
1835         nft_trans_chain_stats(trans) =
1836                 rcu_replace_pointer(chain->stats, nft_trans_chain_stats(trans),
1837                                     lockdep_commit_lock_is_held(trans->ctx.net));
1838
1839         if (!nft_trans_chain_stats(trans))
1840                 static_branch_inc(&nft_counters_enabled);
1841 }
1842
1843 static void nf_tables_chain_free_chain_rules(struct nft_chain *chain)
1844 {
1845         struct nft_rule_blob *g0 = rcu_dereference_raw(chain->blob_gen_0);
1846         struct nft_rule_blob *g1 = rcu_dereference_raw(chain->blob_gen_1);
1847
1848         if (g0 != g1)
1849                 kvfree(g1);
1850         kvfree(g0);
1851
1852         /* should be NULL either via abort or via successful commit */
1853         WARN_ON_ONCE(chain->blob_next);
1854         kvfree(chain->blob_next);
1855 }
1856
1857 void nf_tables_chain_destroy(struct nft_ctx *ctx)
1858 {
1859         struct nft_chain *chain = ctx->chain;
1860         struct nft_hook *hook, *next;
1861
1862         if (WARN_ON(chain->use > 0))
1863                 return;
1864
1865         /* no concurrent access possible anymore */
1866         nf_tables_chain_free_chain_rules(chain);
1867
1868         if (nft_is_base_chain(chain)) {
1869                 struct nft_base_chain *basechain = nft_base_chain(chain);
1870
1871                 if (nft_base_chain_netdev(ctx->family, basechain->ops.hooknum)) {
1872                         list_for_each_entry_safe(hook, next,
1873                                                  &basechain->hook_list, list) {
1874                                 list_del_rcu(&hook->list);
1875                                 kfree_rcu(hook, rcu);
1876                         }
1877                 }
1878                 module_put(basechain->type->owner);
1879                 if (rcu_access_pointer(basechain->stats)) {
1880                         static_branch_dec(&nft_counters_enabled);
1881                         free_percpu(rcu_dereference_raw(basechain->stats));
1882                 }
1883                 kfree(chain->name);
1884                 kfree(chain->udata);
1885                 kfree(basechain);
1886         } else {
1887                 kfree(chain->name);
1888                 kfree(chain->udata);
1889                 kfree(chain);
1890         }
1891 }
1892
1893 static struct nft_hook *nft_netdev_hook_alloc(struct net *net,
1894                                               const struct nlattr *attr)
1895 {
1896         struct net_device *dev;
1897         char ifname[IFNAMSIZ];
1898         struct nft_hook *hook;
1899         int err;
1900
1901         hook = kmalloc(sizeof(struct nft_hook), GFP_KERNEL_ACCOUNT);
1902         if (!hook) {
1903                 err = -ENOMEM;
1904                 goto err_hook_alloc;
1905         }
1906
1907         nla_strscpy(ifname, attr, IFNAMSIZ);
1908         /* nf_tables_netdev_event() is called under rtnl_mutex, this is
1909          * indirectly serializing all the other holders of the commit_mutex with
1910          * the rtnl_mutex.
1911          */
1912         dev = __dev_get_by_name(net, ifname);
1913         if (!dev) {
1914                 err = -ENOENT;
1915                 goto err_hook_dev;
1916         }
1917         hook->ops.dev = dev;
1918
1919         return hook;
1920
1921 err_hook_dev:
1922         kfree(hook);
1923 err_hook_alloc:
1924         return ERR_PTR(err);
1925 }
1926
1927 static struct nft_hook *nft_hook_list_find(struct list_head *hook_list,
1928                                            const struct nft_hook *this)
1929 {
1930         struct nft_hook *hook;
1931
1932         list_for_each_entry(hook, hook_list, list) {
1933                 if (this->ops.dev == hook->ops.dev)
1934                         return hook;
1935         }
1936
1937         return NULL;
1938 }
1939
1940 static int nf_tables_parse_netdev_hooks(struct net *net,
1941                                         const struct nlattr *attr,
1942                                         struct list_head *hook_list)
1943 {
1944         struct nft_hook *hook, *next;
1945         const struct nlattr *tmp;
1946         int rem, n = 0, err;
1947
1948         nla_for_each_nested(tmp, attr, rem) {
1949                 if (nla_type(tmp) != NFTA_DEVICE_NAME) {
1950                         err = -EINVAL;
1951                         goto err_hook;
1952                 }
1953
1954                 hook = nft_netdev_hook_alloc(net, tmp);
1955                 if (IS_ERR(hook)) {
1956                         err = PTR_ERR(hook);
1957                         goto err_hook;
1958                 }
1959                 if (nft_hook_list_find(hook_list, hook)) {
1960                         kfree(hook);
1961                         err = -EEXIST;
1962                         goto err_hook;
1963                 }
1964                 list_add_tail(&hook->list, hook_list);
1965                 n++;
1966
1967                 if (n == NFT_NETDEVICE_MAX) {
1968                         err = -EFBIG;
1969                         goto err_hook;
1970                 }
1971         }
1972
1973         return 0;
1974
1975 err_hook:
1976         list_for_each_entry_safe(hook, next, hook_list, list) {
1977                 list_del(&hook->list);
1978                 kfree(hook);
1979         }
1980         return err;
1981 }
1982
1983 struct nft_chain_hook {
1984         u32                             num;
1985         s32                             priority;
1986         const struct nft_chain_type     *type;
1987         struct list_head                list;
1988 };
1989
1990 static int nft_chain_parse_netdev(struct net *net,
1991                                   struct nlattr *tb[],
1992                                   struct list_head *hook_list)
1993 {
1994         struct nft_hook *hook;
1995         int err;
1996
1997         if (tb[NFTA_HOOK_DEV]) {
1998                 hook = nft_netdev_hook_alloc(net, tb[NFTA_HOOK_DEV]);
1999                 if (IS_ERR(hook))
2000                         return PTR_ERR(hook);
2001
2002                 list_add_tail(&hook->list, hook_list);
2003         } else if (tb[NFTA_HOOK_DEVS]) {
2004                 err = nf_tables_parse_netdev_hooks(net, tb[NFTA_HOOK_DEVS],
2005                                                    hook_list);
2006                 if (err < 0)
2007                         return err;
2008
2009                 if (list_empty(hook_list))
2010                         return -EINVAL;
2011         } else {
2012                 return -EINVAL;
2013         }
2014
2015         return 0;
2016 }
2017
2018 static int nft_chain_parse_hook(struct net *net,
2019                                 const struct nlattr * const nla[],
2020                                 struct nft_chain_hook *hook, u8 family,
2021                                 struct netlink_ext_ack *extack, bool autoload)
2022 {
2023         struct nftables_pernet *nft_net = nft_pernet(net);
2024         struct nlattr *ha[NFTA_HOOK_MAX + 1];
2025         const struct nft_chain_type *type;
2026         int err;
2027
2028         lockdep_assert_held(&nft_net->commit_mutex);
2029         lockdep_nfnl_nft_mutex_not_held();
2030
2031         err = nla_parse_nested_deprecated(ha, NFTA_HOOK_MAX,
2032                                           nla[NFTA_CHAIN_HOOK],
2033                                           nft_hook_policy, NULL);
2034         if (err < 0)
2035                 return err;
2036
2037         if (ha[NFTA_HOOK_HOOKNUM] == NULL ||
2038             ha[NFTA_HOOK_PRIORITY] == NULL)
2039                 return -EINVAL;
2040
2041         hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
2042         hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
2043
2044         type = __nft_chain_type_get(family, NFT_CHAIN_T_DEFAULT);
2045         if (!type)
2046                 return -EOPNOTSUPP;
2047
2048         if (nla[NFTA_CHAIN_TYPE]) {
2049                 type = nf_tables_chain_type_lookup(net, nla[NFTA_CHAIN_TYPE],
2050                                                    family, autoload);
2051                 if (IS_ERR(type)) {
2052                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2053                         return PTR_ERR(type);
2054                 }
2055         }
2056         if (hook->num >= NFT_MAX_HOOKS || !(type->hook_mask & (1 << hook->num)))
2057                 return -EOPNOTSUPP;
2058
2059         if (type->type == NFT_CHAIN_T_NAT &&
2060             hook->priority <= NF_IP_PRI_CONNTRACK)
2061                 return -EOPNOTSUPP;
2062
2063         if (!try_module_get(type->owner)) {
2064                 if (nla[NFTA_CHAIN_TYPE])
2065                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2066                 return -ENOENT;
2067         }
2068
2069         hook->type = type;
2070
2071         INIT_LIST_HEAD(&hook->list);
2072         if (nft_base_chain_netdev(family, hook->num)) {
2073                 err = nft_chain_parse_netdev(net, ha, &hook->list);
2074                 if (err < 0) {
2075                         module_put(type->owner);
2076                         return err;
2077                 }
2078         } else if (ha[NFTA_HOOK_DEV] || ha[NFTA_HOOK_DEVS]) {
2079                 module_put(type->owner);
2080                 return -EOPNOTSUPP;
2081         }
2082
2083         return 0;
2084 }
2085
2086 static void nft_chain_release_hook(struct nft_chain_hook *hook)
2087 {
2088         struct nft_hook *h, *next;
2089
2090         list_for_each_entry_safe(h, next, &hook->list, list) {
2091                 list_del(&h->list);
2092                 kfree(h);
2093         }
2094         module_put(hook->type->owner);
2095 }
2096
2097 struct nft_rules_old {
2098         struct rcu_head h;
2099         struct nft_rule_blob *blob;
2100 };
2101
2102 static void nft_last_rule(struct nft_rule_blob *blob, const void *ptr)
2103 {
2104         struct nft_rule_dp *prule;
2105
2106         prule = (struct nft_rule_dp *)ptr;
2107         prule->is_last = 1;
2108         /* blob size does not include the trailer rule */
2109 }
2110
2111 static struct nft_rule_blob *nf_tables_chain_alloc_rules(unsigned int size)
2112 {
2113         struct nft_rule_blob *blob;
2114
2115         /* size must include room for the last rule */
2116         if (size < offsetof(struct nft_rule_dp, data))
2117                 return NULL;
2118
2119         size += sizeof(struct nft_rule_blob) + sizeof(struct nft_rules_old);
2120         if (size > INT_MAX)
2121                 return NULL;
2122
2123         blob = kvmalloc(size, GFP_KERNEL_ACCOUNT);
2124         if (!blob)
2125                 return NULL;
2126
2127         blob->size = 0;
2128         nft_last_rule(blob, blob->data);
2129
2130         return blob;
2131 }
2132
2133 static void nft_basechain_hook_init(struct nf_hook_ops *ops, u8 family,
2134                                     const struct nft_chain_hook *hook,
2135                                     struct nft_chain *chain)
2136 {
2137         ops->pf                 = family;
2138         ops->hooknum            = hook->num;
2139         ops->priority           = hook->priority;
2140         ops->priv               = chain;
2141         ops->hook               = hook->type->hooks[ops->hooknum];
2142         ops->hook_ops_type      = NF_HOOK_OP_NF_TABLES;
2143 }
2144
2145 static int nft_basechain_init(struct nft_base_chain *basechain, u8 family,
2146                               struct nft_chain_hook *hook, u32 flags)
2147 {
2148         struct nft_chain *chain;
2149         struct nft_hook *h;
2150
2151         basechain->type = hook->type;
2152         INIT_LIST_HEAD(&basechain->hook_list);
2153         chain = &basechain->chain;
2154
2155         if (nft_base_chain_netdev(family, hook->num)) {
2156                 list_splice_init(&hook->list, &basechain->hook_list);
2157                 list_for_each_entry(h, &basechain->hook_list, list)
2158                         nft_basechain_hook_init(&h->ops, family, hook, chain);
2159
2160                 basechain->ops.hooknum  = hook->num;
2161                 basechain->ops.priority = hook->priority;
2162         } else {
2163                 nft_basechain_hook_init(&basechain->ops, family, hook, chain);
2164         }
2165
2166         chain->flags |= NFT_CHAIN_BASE | flags;
2167         basechain->policy = NF_ACCEPT;
2168         if (chain->flags & NFT_CHAIN_HW_OFFLOAD &&
2169             !nft_chain_offload_support(basechain))
2170                 return -EOPNOTSUPP;
2171
2172         flow_block_init(&basechain->flow_block);
2173
2174         return 0;
2175 }
2176
2177 static int nft_chain_add(struct nft_table *table, struct nft_chain *chain)
2178 {
2179         int err;
2180
2181         err = rhltable_insert_key(&table->chains_ht, chain->name,
2182                                   &chain->rhlhead, nft_chain_ht_params);
2183         if (err)
2184                 return err;
2185
2186         list_add_tail_rcu(&chain->list, &table->chains);
2187
2188         return 0;
2189 }
2190
2191 static u64 chain_id;
2192
2193 static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 genmask,
2194                               u8 policy, u32 flags,
2195                               struct netlink_ext_ack *extack)
2196 {
2197         const struct nlattr * const *nla = ctx->nla;
2198         struct nft_stats __percpu *stats = NULL;
2199         struct nft_table *table = ctx->table;
2200         struct nft_base_chain *basechain;
2201         struct net *net = ctx->net;
2202         char name[NFT_NAME_MAXLEN];
2203         struct nft_rule_blob *blob;
2204         struct nft_trans *trans;
2205         struct nft_chain *chain;
2206         unsigned int data_size;
2207         int err;
2208
2209         if (table->use == UINT_MAX)
2210                 return -EOVERFLOW;
2211
2212         if (nla[NFTA_CHAIN_HOOK]) {
2213                 struct nft_chain_hook hook;
2214
2215                 if (flags & NFT_CHAIN_BINDING)
2216                         return -EOPNOTSUPP;
2217
2218                 err = nft_chain_parse_hook(net, nla, &hook, family, extack,
2219                                            true);
2220                 if (err < 0)
2221                         return err;
2222
2223                 basechain = kzalloc(sizeof(*basechain), GFP_KERNEL_ACCOUNT);
2224                 if (basechain == NULL) {
2225                         nft_chain_release_hook(&hook);
2226                         return -ENOMEM;
2227                 }
2228                 chain = &basechain->chain;
2229
2230                 if (nla[NFTA_CHAIN_COUNTERS]) {
2231                         stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2232                         if (IS_ERR(stats)) {
2233                                 nft_chain_release_hook(&hook);
2234                                 kfree(basechain);
2235                                 return PTR_ERR(stats);
2236                         }
2237                         rcu_assign_pointer(basechain->stats, stats);
2238                 }
2239
2240                 err = nft_basechain_init(basechain, family, &hook, flags);
2241                 if (err < 0) {
2242                         nft_chain_release_hook(&hook);
2243                         kfree(basechain);
2244                         return err;
2245                 }
2246         } else {
2247                 if (flags & NFT_CHAIN_BASE)
2248                         return -EINVAL;
2249                 if (flags & NFT_CHAIN_HW_OFFLOAD)
2250                         return -EOPNOTSUPP;
2251
2252                 chain = kzalloc(sizeof(*chain), GFP_KERNEL_ACCOUNT);
2253                 if (chain == NULL)
2254                         return -ENOMEM;
2255
2256                 chain->flags = flags;
2257         }
2258         ctx->chain = chain;
2259
2260         INIT_LIST_HEAD(&chain->rules);
2261         chain->handle = nf_tables_alloc_handle(table);
2262         chain->table = table;
2263
2264         if (nla[NFTA_CHAIN_NAME]) {
2265                 chain->name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
2266         } else {
2267                 if (!(flags & NFT_CHAIN_BINDING)) {
2268                         err = -EINVAL;
2269                         goto err_destroy_chain;
2270                 }
2271
2272                 snprintf(name, sizeof(name), "__chain%llu", ++chain_id);
2273                 chain->name = kstrdup(name, GFP_KERNEL_ACCOUNT);
2274         }
2275
2276         if (!chain->name) {
2277                 err = -ENOMEM;
2278                 goto err_destroy_chain;
2279         }
2280
2281         if (nla[NFTA_CHAIN_USERDATA]) {
2282                 chain->udata = nla_memdup(nla[NFTA_CHAIN_USERDATA], GFP_KERNEL_ACCOUNT);
2283                 if (chain->udata == NULL) {
2284                         err = -ENOMEM;
2285                         goto err_destroy_chain;
2286                 }
2287                 chain->udlen = nla_len(nla[NFTA_CHAIN_USERDATA]);
2288         }
2289
2290         data_size = offsetof(struct nft_rule_dp, data); /* last rule */
2291         blob = nf_tables_chain_alloc_rules(data_size);
2292         if (!blob) {
2293                 err = -ENOMEM;
2294                 goto err_destroy_chain;
2295         }
2296
2297         RCU_INIT_POINTER(chain->blob_gen_0, blob);
2298         RCU_INIT_POINTER(chain->blob_gen_1, blob);
2299
2300         err = nf_tables_register_hook(net, table, chain);
2301         if (err < 0)
2302                 goto err_destroy_chain;
2303
2304         trans = nft_trans_chain_add(ctx, NFT_MSG_NEWCHAIN);
2305         if (IS_ERR(trans)) {
2306                 err = PTR_ERR(trans);
2307                 goto err_unregister_hook;
2308         }
2309
2310         nft_trans_chain_policy(trans) = NFT_CHAIN_POLICY_UNSET;
2311         if (nft_is_base_chain(chain))
2312                 nft_trans_chain_policy(trans) = policy;
2313
2314         err = nft_chain_add(table, chain);
2315         if (err < 0) {
2316                 nft_trans_destroy(trans);
2317                 goto err_unregister_hook;
2318         }
2319
2320         if (stats)
2321                 static_branch_inc(&nft_counters_enabled);
2322
2323         table->use++;
2324
2325         return 0;
2326 err_unregister_hook:
2327         nf_tables_unregister_hook(net, table, chain);
2328 err_destroy_chain:
2329         nf_tables_chain_destroy(ctx);
2330
2331         return err;
2332 }
2333
2334 static bool nft_hook_list_equal(struct list_head *hook_list1,
2335                                 struct list_head *hook_list2)
2336 {
2337         struct nft_hook *hook;
2338         int n = 0, m = 0;
2339
2340         n = 0;
2341         list_for_each_entry(hook, hook_list2, list) {
2342                 if (!nft_hook_list_find(hook_list1, hook))
2343                         return false;
2344
2345                 n++;
2346         }
2347         list_for_each_entry(hook, hook_list1, list)
2348                 m++;
2349
2350         return n == m;
2351 }
2352
2353 static int nf_tables_updchain(struct nft_ctx *ctx, u8 genmask, u8 policy,
2354                               u32 flags, const struct nlattr *attr,
2355                               struct netlink_ext_ack *extack)
2356 {
2357         const struct nlattr * const *nla = ctx->nla;
2358         struct nft_table *table = ctx->table;
2359         struct nft_chain *chain = ctx->chain;
2360         struct nft_base_chain *basechain;
2361         struct nft_stats *stats = NULL;
2362         struct nft_chain_hook hook;
2363         struct nf_hook_ops *ops;
2364         struct nft_trans *trans;
2365         int err;
2366
2367         if (chain->flags ^ flags)
2368                 return -EOPNOTSUPP;
2369
2370         if (nla[NFTA_CHAIN_HOOK]) {
2371                 if (!nft_is_base_chain(chain)) {
2372                         NL_SET_BAD_ATTR(extack, attr);
2373                         return -EEXIST;
2374                 }
2375                 err = nft_chain_parse_hook(ctx->net, nla, &hook, ctx->family,
2376                                            extack, false);
2377                 if (err < 0)
2378                         return err;
2379
2380                 basechain = nft_base_chain(chain);
2381                 if (basechain->type != hook.type) {
2382                         nft_chain_release_hook(&hook);
2383                         NL_SET_BAD_ATTR(extack, attr);
2384                         return -EEXIST;
2385                 }
2386
2387                 if (nft_base_chain_netdev(ctx->family, hook.num)) {
2388                         if (!nft_hook_list_equal(&basechain->hook_list,
2389                                                  &hook.list)) {
2390                                 nft_chain_release_hook(&hook);
2391                                 NL_SET_BAD_ATTR(extack, attr);
2392                                 return -EEXIST;
2393                         }
2394                 } else {
2395                         ops = &basechain->ops;
2396                         if (ops->hooknum != hook.num ||
2397                             ops->priority != hook.priority) {
2398                                 nft_chain_release_hook(&hook);
2399                                 NL_SET_BAD_ATTR(extack, attr);
2400                                 return -EEXIST;
2401                         }
2402                 }
2403                 nft_chain_release_hook(&hook);
2404         }
2405
2406         if (nla[NFTA_CHAIN_HANDLE] &&
2407             nla[NFTA_CHAIN_NAME]) {
2408                 struct nft_chain *chain2;
2409
2410                 chain2 = nft_chain_lookup(ctx->net, table,
2411                                           nla[NFTA_CHAIN_NAME], genmask);
2412                 if (!IS_ERR(chain2)) {
2413                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2414                         return -EEXIST;
2415                 }
2416         }
2417
2418         if (nla[NFTA_CHAIN_COUNTERS]) {
2419                 if (!nft_is_base_chain(chain))
2420                         return -EOPNOTSUPP;
2421
2422                 stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2423                 if (IS_ERR(stats))
2424                         return PTR_ERR(stats);
2425         }
2426
2427         err = -ENOMEM;
2428         trans = nft_trans_alloc(ctx, NFT_MSG_NEWCHAIN,
2429                                 sizeof(struct nft_trans_chain));
2430         if (trans == NULL)
2431                 goto err;
2432
2433         nft_trans_chain_stats(trans) = stats;
2434         nft_trans_chain_update(trans) = true;
2435
2436         if (nla[NFTA_CHAIN_POLICY])
2437                 nft_trans_chain_policy(trans) = policy;
2438         else
2439                 nft_trans_chain_policy(trans) = -1;
2440
2441         if (nla[NFTA_CHAIN_HANDLE] &&
2442             nla[NFTA_CHAIN_NAME]) {
2443                 struct nftables_pernet *nft_net = nft_pernet(ctx->net);
2444                 struct nft_trans *tmp;
2445                 char *name;
2446
2447                 err = -ENOMEM;
2448                 name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
2449                 if (!name)
2450                         goto err;
2451
2452                 err = -EEXIST;
2453                 list_for_each_entry(tmp, &nft_net->commit_list, list) {
2454                         if (tmp->msg_type == NFT_MSG_NEWCHAIN &&
2455                             tmp->ctx.table == table &&
2456                             nft_trans_chain_update(tmp) &&
2457                             nft_trans_chain_name(tmp) &&
2458                             strcmp(name, nft_trans_chain_name(tmp)) == 0) {
2459                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2460                                 kfree(name);
2461                                 goto err;
2462                         }
2463                 }
2464
2465                 nft_trans_chain_name(trans) = name;
2466         }
2467         nft_trans_commit_list_add_tail(ctx->net, trans);
2468
2469         return 0;
2470 err:
2471         free_percpu(stats);
2472         kfree(trans);
2473         return err;
2474 }
2475
2476 static struct nft_chain *nft_chain_lookup_byid(const struct net *net,
2477                                                const struct nft_table *table,
2478                                                const struct nlattr *nla)
2479 {
2480         struct nftables_pernet *nft_net = nft_pernet(net);
2481         u32 id = ntohl(nla_get_be32(nla));
2482         struct nft_trans *trans;
2483
2484         list_for_each_entry(trans, &nft_net->commit_list, list) {
2485                 struct nft_chain *chain = trans->ctx.chain;
2486
2487                 if (trans->msg_type == NFT_MSG_NEWCHAIN &&
2488                     chain->table == table &&
2489                     id == nft_trans_chain_id(trans))
2490                         return chain;
2491         }
2492         return ERR_PTR(-ENOENT);
2493 }
2494
2495 static int nf_tables_newchain(struct sk_buff *skb, const struct nfnl_info *info,
2496                               const struct nlattr * const nla[])
2497 {
2498         struct nftables_pernet *nft_net = nft_pernet(info->net);
2499         struct netlink_ext_ack *extack = info->extack;
2500         u8 genmask = nft_genmask_next(info->net);
2501         u8 family = info->nfmsg->nfgen_family;
2502         struct nft_chain *chain = NULL;
2503         struct net *net = info->net;
2504         const struct nlattr *attr;
2505         struct nft_table *table;
2506         u8 policy = NF_ACCEPT;
2507         struct nft_ctx ctx;
2508         u64 handle = 0;
2509         u32 flags = 0;
2510
2511         lockdep_assert_held(&nft_net->commit_mutex);
2512
2513         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
2514                                  NETLINK_CB(skb).portid);
2515         if (IS_ERR(table)) {
2516                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2517                 return PTR_ERR(table);
2518         }
2519
2520         chain = NULL;
2521         attr = nla[NFTA_CHAIN_NAME];
2522
2523         if (nla[NFTA_CHAIN_HANDLE]) {
2524                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_CHAIN_HANDLE]));
2525                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2526                 if (IS_ERR(chain)) {
2527                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_HANDLE]);
2528                         return PTR_ERR(chain);
2529                 }
2530                 attr = nla[NFTA_CHAIN_HANDLE];
2531         } else if (nla[NFTA_CHAIN_NAME]) {
2532                 chain = nft_chain_lookup(net, table, attr, genmask);
2533                 if (IS_ERR(chain)) {
2534                         if (PTR_ERR(chain) != -ENOENT) {
2535                                 NL_SET_BAD_ATTR(extack, attr);
2536                                 return PTR_ERR(chain);
2537                         }
2538                         chain = NULL;
2539                 }
2540         } else if (!nla[NFTA_CHAIN_ID]) {
2541                 return -EINVAL;
2542         }
2543
2544         if (nla[NFTA_CHAIN_POLICY]) {
2545                 if (chain != NULL &&
2546                     !nft_is_base_chain(chain)) {
2547                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2548                         return -EOPNOTSUPP;
2549                 }
2550
2551                 if (chain == NULL &&
2552                     nla[NFTA_CHAIN_HOOK] == NULL) {
2553                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2554                         return -EOPNOTSUPP;
2555                 }
2556
2557                 policy = ntohl(nla_get_be32(nla[NFTA_CHAIN_POLICY]));
2558                 switch (policy) {
2559                 case NF_DROP:
2560                 case NF_ACCEPT:
2561                         break;
2562                 default:
2563                         return -EINVAL;
2564                 }
2565         }
2566
2567         if (nla[NFTA_CHAIN_FLAGS])
2568                 flags = ntohl(nla_get_be32(nla[NFTA_CHAIN_FLAGS]));
2569         else if (chain)
2570                 flags = chain->flags;
2571
2572         if (flags & ~NFT_CHAIN_FLAGS)
2573                 return -EOPNOTSUPP;
2574
2575         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
2576
2577         if (chain != NULL) {
2578                 if (chain->flags & NFT_CHAIN_BINDING)
2579                         return -EINVAL;
2580
2581                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
2582                         NL_SET_BAD_ATTR(extack, attr);
2583                         return -EEXIST;
2584                 }
2585                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
2586                         return -EOPNOTSUPP;
2587
2588                 flags |= chain->flags & NFT_CHAIN_BASE;
2589                 return nf_tables_updchain(&ctx, genmask, policy, flags, attr,
2590                                           extack);
2591         }
2592
2593         return nf_tables_addchain(&ctx, family, genmask, policy, flags, extack);
2594 }
2595
2596 static int nf_tables_delchain(struct sk_buff *skb, const struct nfnl_info *info,
2597                               const struct nlattr * const nla[])
2598 {
2599         struct netlink_ext_ack *extack = info->extack;
2600         u8 genmask = nft_genmask_next(info->net);
2601         u8 family = info->nfmsg->nfgen_family;
2602         struct net *net = info->net;
2603         const struct nlattr *attr;
2604         struct nft_table *table;
2605         struct nft_chain *chain;
2606         struct nft_rule *rule;
2607         struct nft_ctx ctx;
2608         u64 handle;
2609         u32 use;
2610         int err;
2611
2612         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
2613                                  NETLINK_CB(skb).portid);
2614         if (IS_ERR(table)) {
2615                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2616                 return PTR_ERR(table);
2617         }
2618
2619         if (nla[NFTA_CHAIN_HANDLE]) {
2620                 attr = nla[NFTA_CHAIN_HANDLE];
2621                 handle = be64_to_cpu(nla_get_be64(attr));
2622                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2623         } else {
2624                 attr = nla[NFTA_CHAIN_NAME];
2625                 chain = nft_chain_lookup(net, table, attr, genmask);
2626         }
2627         if (IS_ERR(chain)) {
2628                 NL_SET_BAD_ATTR(extack, attr);
2629                 return PTR_ERR(chain);
2630         }
2631
2632         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
2633             chain->use > 0)
2634                 return -EBUSY;
2635
2636         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
2637
2638         use = chain->use;
2639         list_for_each_entry(rule, &chain->rules, list) {
2640                 if (!nft_is_active_next(net, rule))
2641                         continue;
2642                 use--;
2643
2644                 err = nft_delrule(&ctx, rule);
2645                 if (err < 0)
2646                         return err;
2647         }
2648
2649         /* There are rules and elements that are still holding references to us,
2650          * we cannot do a recursive removal in this case.
2651          */
2652         if (use > 0) {
2653                 NL_SET_BAD_ATTR(extack, attr);
2654                 return -EBUSY;
2655         }
2656
2657         return nft_delchain(&ctx);
2658 }
2659
2660 /*
2661  * Expressions
2662  */
2663
2664 /**
2665  *      nft_register_expr - register nf_tables expr type
2666  *      @type: expr type
2667  *
2668  *      Registers the expr type for use with nf_tables. Returns zero on
2669  *      success or a negative errno code otherwise.
2670  */
2671 int nft_register_expr(struct nft_expr_type *type)
2672 {
2673         nfnl_lock(NFNL_SUBSYS_NFTABLES);
2674         if (type->family == NFPROTO_UNSPEC)
2675                 list_add_tail_rcu(&type->list, &nf_tables_expressions);
2676         else
2677                 list_add_rcu(&type->list, &nf_tables_expressions);
2678         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
2679         return 0;
2680 }
2681 EXPORT_SYMBOL_GPL(nft_register_expr);
2682
2683 /**
2684  *      nft_unregister_expr - unregister nf_tables expr type
2685  *      @type: expr type
2686  *
2687  *      Unregisters the expr typefor use with nf_tables.
2688  */
2689 void nft_unregister_expr(struct nft_expr_type *type)
2690 {
2691         nfnl_lock(NFNL_SUBSYS_NFTABLES);
2692         list_del_rcu(&type->list);
2693         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
2694 }
2695 EXPORT_SYMBOL_GPL(nft_unregister_expr);
2696
2697 static const struct nft_expr_type *__nft_expr_type_get(u8 family,
2698                                                        struct nlattr *nla)
2699 {
2700         const struct nft_expr_type *type, *candidate = NULL;
2701
2702         list_for_each_entry(type, &nf_tables_expressions, list) {
2703                 if (!nla_strcmp(nla, type->name)) {
2704                         if (!type->family && !candidate)
2705                                 candidate = type;
2706                         else if (type->family == family)
2707                                 candidate = type;
2708                 }
2709         }
2710         return candidate;
2711 }
2712
2713 #ifdef CONFIG_MODULES
2714 static int nft_expr_type_request_module(struct net *net, u8 family,
2715                                         struct nlattr *nla)
2716 {
2717         if (nft_request_module(net, "nft-expr-%u-%.*s", family,
2718                                nla_len(nla), (char *)nla_data(nla)) == -EAGAIN)
2719                 return -EAGAIN;
2720
2721         return 0;
2722 }
2723 #endif
2724
2725 static const struct nft_expr_type *nft_expr_type_get(struct net *net,
2726                                                      u8 family,
2727                                                      struct nlattr *nla)
2728 {
2729         const struct nft_expr_type *type;
2730
2731         if (nla == NULL)
2732                 return ERR_PTR(-EINVAL);
2733
2734         type = __nft_expr_type_get(family, nla);
2735         if (type != NULL && try_module_get(type->owner))
2736                 return type;
2737
2738         lockdep_nfnl_nft_mutex_not_held();
2739 #ifdef CONFIG_MODULES
2740         if (type == NULL) {
2741                 if (nft_expr_type_request_module(net, family, nla) == -EAGAIN)
2742                         return ERR_PTR(-EAGAIN);
2743
2744                 if (nft_request_module(net, "nft-expr-%.*s",
2745                                        nla_len(nla),
2746                                        (char *)nla_data(nla)) == -EAGAIN)
2747                         return ERR_PTR(-EAGAIN);
2748         }
2749 #endif
2750         return ERR_PTR(-ENOENT);
2751 }
2752
2753 static const struct nla_policy nft_expr_policy[NFTA_EXPR_MAX + 1] = {
2754         [NFTA_EXPR_NAME]        = { .type = NLA_STRING,
2755                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
2756         [NFTA_EXPR_DATA]        = { .type = NLA_NESTED },
2757 };
2758
2759 static int nf_tables_fill_expr_info(struct sk_buff *skb,
2760                                     const struct nft_expr *expr)
2761 {
2762         if (nla_put_string(skb, NFTA_EXPR_NAME, expr->ops->type->name))
2763                 goto nla_put_failure;
2764
2765         if (expr->ops->dump) {
2766                 struct nlattr *data = nla_nest_start_noflag(skb,
2767                                                             NFTA_EXPR_DATA);
2768                 if (data == NULL)
2769                         goto nla_put_failure;
2770                 if (expr->ops->dump(skb, expr) < 0)
2771                         goto nla_put_failure;
2772                 nla_nest_end(skb, data);
2773         }
2774
2775         return skb->len;
2776
2777 nla_put_failure:
2778         return -1;
2779 };
2780
2781 int nft_expr_dump(struct sk_buff *skb, unsigned int attr,
2782                   const struct nft_expr *expr)
2783 {
2784         struct nlattr *nest;
2785
2786         nest = nla_nest_start_noflag(skb, attr);
2787         if (!nest)
2788                 goto nla_put_failure;
2789         if (nf_tables_fill_expr_info(skb, expr) < 0)
2790                 goto nla_put_failure;
2791         nla_nest_end(skb, nest);
2792         return 0;
2793
2794 nla_put_failure:
2795         return -1;
2796 }
2797
2798 struct nft_expr_info {
2799         const struct nft_expr_ops       *ops;
2800         const struct nlattr             *attr;
2801         struct nlattr                   *tb[NFT_EXPR_MAXATTR + 1];
2802 };
2803
2804 static int nf_tables_expr_parse(const struct nft_ctx *ctx,
2805                                 const struct nlattr *nla,
2806                                 struct nft_expr_info *info)
2807 {
2808         const struct nft_expr_type *type;
2809         const struct nft_expr_ops *ops;
2810         struct nlattr *tb[NFTA_EXPR_MAX + 1];
2811         int err;
2812
2813         err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
2814                                           nft_expr_policy, NULL);
2815         if (err < 0)
2816                 return err;
2817
2818         type = nft_expr_type_get(ctx->net, ctx->family, tb[NFTA_EXPR_NAME]);
2819         if (IS_ERR(type))
2820                 return PTR_ERR(type);
2821
2822         if (tb[NFTA_EXPR_DATA]) {
2823                 err = nla_parse_nested_deprecated(info->tb, type->maxattr,
2824                                                   tb[NFTA_EXPR_DATA],
2825                                                   type->policy, NULL);
2826                 if (err < 0)
2827                         goto err1;
2828         } else
2829                 memset(info->tb, 0, sizeof(info->tb[0]) * (type->maxattr + 1));
2830
2831         if (type->select_ops != NULL) {
2832                 ops = type->select_ops(ctx,
2833                                        (const struct nlattr * const *)info->tb);
2834                 if (IS_ERR(ops)) {
2835                         err = PTR_ERR(ops);
2836 #ifdef CONFIG_MODULES
2837                         if (err == -EAGAIN)
2838                                 if (nft_expr_type_request_module(ctx->net,
2839                                                                  ctx->family,
2840                                                                  tb[NFTA_EXPR_NAME]) != -EAGAIN)
2841                                         err = -ENOENT;
2842 #endif
2843                         goto err1;
2844                 }
2845         } else
2846                 ops = type->ops;
2847
2848         info->attr = nla;
2849         info->ops = ops;
2850
2851         return 0;
2852
2853 err1:
2854         module_put(type->owner);
2855         return err;
2856 }
2857
2858 static int nf_tables_newexpr(const struct nft_ctx *ctx,
2859                              const struct nft_expr_info *expr_info,
2860                              struct nft_expr *expr)
2861 {
2862         const struct nft_expr_ops *ops = expr_info->ops;
2863         int err;
2864
2865         expr->ops = ops;
2866         if (ops->init) {
2867                 err = ops->init(ctx, expr, (const struct nlattr **)expr_info->tb);
2868                 if (err < 0)
2869                         goto err1;
2870         }
2871
2872         return 0;
2873 err1:
2874         expr->ops = NULL;
2875         return err;
2876 }
2877
2878 static void nf_tables_expr_destroy(const struct nft_ctx *ctx,
2879                                    struct nft_expr *expr)
2880 {
2881         const struct nft_expr_type *type = expr->ops->type;
2882
2883         if (expr->ops->destroy)
2884                 expr->ops->destroy(ctx, expr);
2885         module_put(type->owner);
2886 }
2887
2888 static struct nft_expr *nft_expr_init(const struct nft_ctx *ctx,
2889                                       const struct nlattr *nla)
2890 {
2891         struct nft_expr_info expr_info;
2892         struct nft_expr *expr;
2893         struct module *owner;
2894         int err;
2895
2896         err = nf_tables_expr_parse(ctx, nla, &expr_info);
2897         if (err < 0)
2898                 goto err_expr_parse;
2899
2900         err = -EOPNOTSUPP;
2901         if (!(expr_info.ops->type->flags & NFT_EXPR_STATEFUL))
2902                 goto err_expr_stateful;
2903
2904         err = -ENOMEM;
2905         expr = kzalloc(expr_info.ops->size, GFP_KERNEL_ACCOUNT);
2906         if (expr == NULL)
2907                 goto err_expr_stateful;
2908
2909         err = nf_tables_newexpr(ctx, &expr_info, expr);
2910         if (err < 0)
2911                 goto err_expr_new;
2912
2913         return expr;
2914 err_expr_new:
2915         kfree(expr);
2916 err_expr_stateful:
2917         owner = expr_info.ops->type->owner;
2918         if (expr_info.ops->type->release_ops)
2919                 expr_info.ops->type->release_ops(expr_info.ops);
2920
2921         module_put(owner);
2922 err_expr_parse:
2923         return ERR_PTR(err);
2924 }
2925
2926 int nft_expr_clone(struct nft_expr *dst, struct nft_expr *src)
2927 {
2928         int err;
2929
2930         if (src->ops->clone) {
2931                 dst->ops = src->ops;
2932                 err = src->ops->clone(dst, src);
2933                 if (err < 0)
2934                         return err;
2935         } else {
2936                 memcpy(dst, src, src->ops->size);
2937         }
2938
2939         __module_get(src->ops->type->owner);
2940
2941         return 0;
2942 }
2943
2944 void nft_expr_destroy(const struct nft_ctx *ctx, struct nft_expr *expr)
2945 {
2946         nf_tables_expr_destroy(ctx, expr);
2947         kfree(expr);
2948 }
2949
2950 /*
2951  * Rules
2952  */
2953
2954 static struct nft_rule *__nft_rule_lookup(const struct nft_chain *chain,
2955                                           u64 handle)
2956 {
2957         struct nft_rule *rule;
2958
2959         // FIXME: this sucks
2960         list_for_each_entry_rcu(rule, &chain->rules, list) {
2961                 if (handle == rule->handle)
2962                         return rule;
2963         }
2964
2965         return ERR_PTR(-ENOENT);
2966 }
2967
2968 static struct nft_rule *nft_rule_lookup(const struct nft_chain *chain,
2969                                         const struct nlattr *nla)
2970 {
2971         if (nla == NULL)
2972                 return ERR_PTR(-EINVAL);
2973
2974         return __nft_rule_lookup(chain, be64_to_cpu(nla_get_be64(nla)));
2975 }
2976
2977 static const struct nla_policy nft_rule_policy[NFTA_RULE_MAX + 1] = {
2978         [NFTA_RULE_TABLE]       = { .type = NLA_STRING,
2979                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
2980         [NFTA_RULE_CHAIN]       = { .type = NLA_STRING,
2981                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
2982         [NFTA_RULE_HANDLE]      = { .type = NLA_U64 },
2983         [NFTA_RULE_EXPRESSIONS] = { .type = NLA_NESTED },
2984         [NFTA_RULE_COMPAT]      = { .type = NLA_NESTED },
2985         [NFTA_RULE_POSITION]    = { .type = NLA_U64 },
2986         [NFTA_RULE_USERDATA]    = { .type = NLA_BINARY,
2987                                     .len = NFT_USERDATA_MAXLEN },
2988         [NFTA_RULE_ID]          = { .type = NLA_U32 },
2989         [NFTA_RULE_POSITION_ID] = { .type = NLA_U32 },
2990         [NFTA_RULE_CHAIN_ID]    = { .type = NLA_U32 },
2991 };
2992
2993 static int nf_tables_fill_rule_info(struct sk_buff *skb, struct net *net,
2994                                     u32 portid, u32 seq, int event,
2995                                     u32 flags, int family,
2996                                     const struct nft_table *table,
2997                                     const struct nft_chain *chain,
2998                                     const struct nft_rule *rule, u64 handle)
2999 {
3000         struct nlmsghdr *nlh;
3001         const struct nft_expr *expr, *next;
3002         struct nlattr *list;
3003         u16 type = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
3004
3005         nlh = nfnl_msg_put(skb, portid, seq, type, flags, family, NFNETLINK_V0,
3006                            nft_base_seq(net));
3007         if (!nlh)
3008                 goto nla_put_failure;
3009
3010         if (nla_put_string(skb, NFTA_RULE_TABLE, table->name))
3011                 goto nla_put_failure;
3012         if (nla_put_string(skb, NFTA_RULE_CHAIN, chain->name))
3013                 goto nla_put_failure;
3014         if (nla_put_be64(skb, NFTA_RULE_HANDLE, cpu_to_be64(rule->handle),
3015                          NFTA_RULE_PAD))
3016                 goto nla_put_failure;
3017
3018         if (event != NFT_MSG_DELRULE && handle) {
3019                 if (nla_put_be64(skb, NFTA_RULE_POSITION, cpu_to_be64(handle),
3020                                  NFTA_RULE_PAD))
3021                         goto nla_put_failure;
3022         }
3023
3024         if (chain->flags & NFT_CHAIN_HW_OFFLOAD)
3025                 nft_flow_rule_stats(chain, rule);
3026
3027         list = nla_nest_start_noflag(skb, NFTA_RULE_EXPRESSIONS);
3028         if (list == NULL)
3029                 goto nla_put_failure;
3030         nft_rule_for_each_expr(expr, next, rule) {
3031                 if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr) < 0)
3032                         goto nla_put_failure;
3033         }
3034         nla_nest_end(skb, list);
3035
3036         if (rule->udata) {
3037                 struct nft_userdata *udata = nft_userdata(rule);
3038                 if (nla_put(skb, NFTA_RULE_USERDATA, udata->len + 1,
3039                             udata->data) < 0)
3040                         goto nla_put_failure;
3041         }
3042
3043         nlmsg_end(skb, nlh);
3044         return 0;
3045
3046 nla_put_failure:
3047         nlmsg_trim(skb, nlh);
3048         return -1;
3049 }
3050
3051 static void nf_tables_rule_notify(const struct nft_ctx *ctx,
3052                                   const struct nft_rule *rule, int event)
3053 {
3054         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
3055         const struct nft_rule *prule;
3056         struct sk_buff *skb;
3057         u64 handle = 0;
3058         u16 flags = 0;
3059         int err;
3060
3061         if (!ctx->report &&
3062             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
3063                 return;
3064
3065         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
3066         if (skb == NULL)
3067                 goto err;
3068
3069         if (event == NFT_MSG_NEWRULE &&
3070             !list_is_first(&rule->list, &ctx->chain->rules) &&
3071             !list_is_last(&rule->list, &ctx->chain->rules)) {
3072                 prule = list_prev_entry(rule, list);
3073                 handle = prule->handle;
3074         }
3075         if (ctx->flags & (NLM_F_APPEND | NLM_F_REPLACE))
3076                 flags |= NLM_F_APPEND;
3077         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
3078                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
3079
3080         err = nf_tables_fill_rule_info(skb, ctx->net, ctx->portid, ctx->seq,
3081                                        event, flags, ctx->family, ctx->table,
3082                                        ctx->chain, rule, handle);
3083         if (err < 0) {
3084                 kfree_skb(skb);
3085                 goto err;
3086         }
3087
3088         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
3089         return;
3090 err:
3091         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
3092 }
3093
3094 struct nft_rule_dump_ctx {
3095         char *table;
3096         char *chain;
3097 };
3098
3099 static int __nf_tables_dump_rules(struct sk_buff *skb,
3100                                   unsigned int *idx,
3101                                   struct netlink_callback *cb,
3102                                   const struct nft_table *table,
3103                                   const struct nft_chain *chain)
3104 {
3105         struct net *net = sock_net(skb->sk);
3106         const struct nft_rule *rule, *prule;
3107         unsigned int s_idx = cb->args[0];
3108         u64 handle;
3109
3110         prule = NULL;
3111         list_for_each_entry_rcu(rule, &chain->rules, list) {
3112                 if (!nft_is_active(net, rule))
3113                         goto cont_skip;
3114                 if (*idx < s_idx)
3115                         goto cont;
3116                 if (*idx > s_idx) {
3117                         memset(&cb->args[1], 0,
3118                                         sizeof(cb->args) - sizeof(cb->args[0]));
3119                 }
3120                 if (prule)
3121                         handle = prule->handle;
3122                 else
3123                         handle = 0;
3124
3125                 if (nf_tables_fill_rule_info(skb, net, NETLINK_CB(cb->skb).portid,
3126                                         cb->nlh->nlmsg_seq,
3127                                         NFT_MSG_NEWRULE,
3128                                         NLM_F_MULTI | NLM_F_APPEND,
3129                                         table->family,
3130                                         table, chain, rule, handle) < 0)
3131                         return 1;
3132
3133                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
3134 cont:
3135                 prule = rule;
3136 cont_skip:
3137                 (*idx)++;
3138         }
3139         return 0;
3140 }
3141
3142 static int nf_tables_dump_rules(struct sk_buff *skb,
3143                                 struct netlink_callback *cb)
3144 {
3145         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
3146         const struct nft_rule_dump_ctx *ctx = cb->data;
3147         struct nft_table *table;
3148         const struct nft_chain *chain;
3149         unsigned int idx = 0;
3150         struct net *net = sock_net(skb->sk);
3151         int family = nfmsg->nfgen_family;
3152         struct nftables_pernet *nft_net;
3153
3154         rcu_read_lock();
3155         nft_net = nft_pernet(net);
3156         cb->seq = READ_ONCE(nft_net->base_seq);
3157
3158         list_for_each_entry_rcu(table, &nft_net->tables, list) {
3159                 if (family != NFPROTO_UNSPEC && family != table->family)
3160                         continue;
3161
3162                 if (ctx && ctx->table && strcmp(ctx->table, table->name) != 0)
3163                         continue;
3164
3165                 if (ctx && ctx->table && ctx->chain) {
3166                         struct rhlist_head *list, *tmp;
3167
3168                         list = rhltable_lookup(&table->chains_ht, ctx->chain,
3169                                                nft_chain_ht_params);
3170                         if (!list)
3171                                 goto done;
3172
3173                         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
3174                                 if (!nft_is_active(net, chain))
3175                                         continue;
3176                                 __nf_tables_dump_rules(skb, &idx,
3177                                                        cb, table, chain);
3178                                 break;
3179                         }
3180                         goto done;
3181                 }
3182
3183                 list_for_each_entry_rcu(chain, &table->chains, list) {
3184                         if (__nf_tables_dump_rules(skb, &idx, cb, table, chain))
3185                                 goto done;
3186                 }
3187
3188                 if (ctx && ctx->table)
3189                         break;
3190         }
3191 done:
3192         rcu_read_unlock();
3193
3194         cb->args[0] = idx;
3195         return skb->len;
3196 }
3197
3198 static int nf_tables_dump_rules_start(struct netlink_callback *cb)
3199 {
3200         const struct nlattr * const *nla = cb->data;
3201         struct nft_rule_dump_ctx *ctx = NULL;
3202
3203         if (nla[NFTA_RULE_TABLE] || nla[NFTA_RULE_CHAIN]) {
3204                 ctx = kzalloc(sizeof(*ctx), GFP_ATOMIC);
3205                 if (!ctx)
3206                         return -ENOMEM;
3207
3208                 if (nla[NFTA_RULE_TABLE]) {
3209                         ctx->table = nla_strdup(nla[NFTA_RULE_TABLE],
3210                                                         GFP_ATOMIC);
3211                         if (!ctx->table) {
3212                                 kfree(ctx);
3213                                 return -ENOMEM;
3214                         }
3215                 }
3216                 if (nla[NFTA_RULE_CHAIN]) {
3217                         ctx->chain = nla_strdup(nla[NFTA_RULE_CHAIN],
3218                                                 GFP_ATOMIC);
3219                         if (!ctx->chain) {
3220                                 kfree(ctx->table);
3221                                 kfree(ctx);
3222                                 return -ENOMEM;
3223                         }
3224                 }
3225         }
3226
3227         cb->data = ctx;
3228         return 0;
3229 }
3230
3231 static int nf_tables_dump_rules_done(struct netlink_callback *cb)
3232 {
3233         struct nft_rule_dump_ctx *ctx = cb->data;
3234
3235         if (ctx) {
3236                 kfree(ctx->table);
3237                 kfree(ctx->chain);
3238                 kfree(ctx);
3239         }
3240         return 0;
3241 }
3242
3243 /* called with rcu_read_lock held */
3244 static int nf_tables_getrule(struct sk_buff *skb, const struct nfnl_info *info,
3245                              const struct nlattr * const nla[])
3246 {
3247         struct netlink_ext_ack *extack = info->extack;
3248         u8 genmask = nft_genmask_cur(info->net);
3249         u8 family = info->nfmsg->nfgen_family;
3250         const struct nft_chain *chain;
3251         const struct nft_rule *rule;
3252         struct net *net = info->net;
3253         struct nft_table *table;
3254         struct sk_buff *skb2;
3255         int err;
3256
3257         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
3258                 struct netlink_dump_control c = {
3259                         .start= nf_tables_dump_rules_start,
3260                         .dump = nf_tables_dump_rules,
3261                         .done = nf_tables_dump_rules_done,
3262                         .module = THIS_MODULE,
3263                         .data = (void *)nla,
3264                 };
3265
3266                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
3267         }
3268
3269         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask, 0);
3270         if (IS_ERR(table)) {
3271                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3272                 return PTR_ERR(table);
3273         }
3274
3275         chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN], genmask);
3276         if (IS_ERR(chain)) {
3277                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3278                 return PTR_ERR(chain);
3279         }
3280
3281         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
3282         if (IS_ERR(rule)) {
3283                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3284                 return PTR_ERR(rule);
3285         }
3286
3287         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
3288         if (!skb2)
3289                 return -ENOMEM;
3290
3291         err = nf_tables_fill_rule_info(skb2, net, NETLINK_CB(skb).portid,
3292                                        info->nlh->nlmsg_seq, NFT_MSG_NEWRULE, 0,
3293                                        family, table, chain, rule, 0);
3294         if (err < 0)
3295                 goto err_fill_rule_info;
3296
3297         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
3298
3299 err_fill_rule_info:
3300         kfree_skb(skb2);
3301         return err;
3302 }
3303
3304 static void nf_tables_rule_destroy(const struct nft_ctx *ctx,
3305                                    struct nft_rule *rule)
3306 {
3307         struct nft_expr *expr, *next;
3308
3309         /*
3310          * Careful: some expressions might not be initialized in case this
3311          * is called on error from nf_tables_newrule().
3312          */
3313         expr = nft_expr_first(rule);
3314         while (nft_expr_more(rule, expr)) {
3315                 next = nft_expr_next(expr);
3316                 nf_tables_expr_destroy(ctx, expr);
3317                 expr = next;
3318         }
3319         kfree(rule);
3320 }
3321
3322 void nf_tables_rule_release(const struct nft_ctx *ctx, struct nft_rule *rule)
3323 {
3324         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_RELEASE);
3325         nf_tables_rule_destroy(ctx, rule);
3326 }
3327
3328 int nft_chain_validate(const struct nft_ctx *ctx, const struct nft_chain *chain)
3329 {
3330         struct nft_expr *expr, *last;
3331         const struct nft_data *data;
3332         struct nft_rule *rule;
3333         int err;
3334
3335         if (ctx->level == NFT_JUMP_STACK_SIZE)
3336                 return -EMLINK;
3337
3338         list_for_each_entry(rule, &chain->rules, list) {
3339                 if (!nft_is_active_next(ctx->net, rule))
3340                         continue;
3341
3342                 nft_rule_for_each_expr(expr, last, rule) {
3343                         if (!expr->ops->validate)
3344                                 continue;
3345
3346                         err = expr->ops->validate(ctx, expr, &data);
3347                         if (err < 0)
3348                                 return err;
3349                 }
3350
3351                 cond_resched();
3352         }
3353
3354         return 0;
3355 }
3356 EXPORT_SYMBOL_GPL(nft_chain_validate);
3357
3358 static int nft_table_validate(struct net *net, const struct nft_table *table)
3359 {
3360         struct nft_chain *chain;
3361         struct nft_ctx ctx = {
3362                 .net    = net,
3363                 .family = table->family,
3364         };
3365         int err;
3366
3367         list_for_each_entry(chain, &table->chains, list) {
3368                 if (!nft_is_base_chain(chain))
3369                         continue;
3370
3371                 ctx.chain = chain;
3372                 err = nft_chain_validate(&ctx, chain);
3373                 if (err < 0)
3374                         return err;
3375         }
3376
3377         return 0;
3378 }
3379
3380 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
3381                                              const struct nft_chain *chain,
3382                                              const struct nlattr *nla);
3383
3384 #define NFT_RULE_MAXEXPRS       128
3385
3386 static int nf_tables_newrule(struct sk_buff *skb, const struct nfnl_info *info,
3387                              const struct nlattr * const nla[])
3388 {
3389         struct nftables_pernet *nft_net = nft_pernet(info->net);
3390         struct netlink_ext_ack *extack = info->extack;
3391         unsigned int size, i, n, ulen = 0, usize = 0;
3392         u8 genmask = nft_genmask_next(info->net);
3393         struct nft_rule *rule, *old_rule = NULL;
3394         struct nft_expr_info *expr_info = NULL;
3395         u8 family = info->nfmsg->nfgen_family;
3396         struct nft_flow_rule *flow = NULL;
3397         struct net *net = info->net;
3398         struct nft_userdata *udata;
3399         struct nft_table *table;
3400         struct nft_chain *chain;
3401         struct nft_trans *trans;
3402         u64 handle, pos_handle;
3403         struct nft_expr *expr;
3404         struct nft_ctx ctx;
3405         struct nlattr *tmp;
3406         int err, rem;
3407
3408         lockdep_assert_held(&nft_net->commit_mutex);
3409
3410         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
3411                                  NETLINK_CB(skb).portid);
3412         if (IS_ERR(table)) {
3413                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3414                 return PTR_ERR(table);
3415         }
3416
3417         if (nla[NFTA_RULE_CHAIN]) {
3418                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
3419                                          genmask);
3420                 if (IS_ERR(chain)) {
3421                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3422                         return PTR_ERR(chain);
3423                 }
3424                 if (nft_chain_is_bound(chain))
3425                         return -EOPNOTSUPP;
3426
3427         } else if (nla[NFTA_RULE_CHAIN_ID]) {
3428                 chain = nft_chain_lookup_byid(net, table, nla[NFTA_RULE_CHAIN_ID]);
3429                 if (IS_ERR(chain)) {
3430                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN_ID]);
3431                         return PTR_ERR(chain);
3432                 }
3433         } else {
3434                 return -EINVAL;
3435         }
3436
3437         if (nla[NFTA_RULE_HANDLE]) {
3438                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_HANDLE]));
3439                 rule = __nft_rule_lookup(chain, handle);
3440                 if (IS_ERR(rule)) {
3441                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3442                         return PTR_ERR(rule);
3443                 }
3444
3445                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
3446                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3447                         return -EEXIST;
3448                 }
3449                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
3450                         old_rule = rule;
3451                 else
3452                         return -EOPNOTSUPP;
3453         } else {
3454                 if (!(info->nlh->nlmsg_flags & NLM_F_CREATE) ||
3455                     info->nlh->nlmsg_flags & NLM_F_REPLACE)
3456                         return -EINVAL;
3457                 handle = nf_tables_alloc_handle(table);
3458
3459                 if (chain->use == UINT_MAX)
3460                         return -EOVERFLOW;
3461
3462                 if (nla[NFTA_RULE_POSITION]) {
3463                         pos_handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_POSITION]));
3464                         old_rule = __nft_rule_lookup(chain, pos_handle);
3465                         if (IS_ERR(old_rule)) {
3466                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION]);
3467                                 return PTR_ERR(old_rule);
3468                         }
3469                 } else if (nla[NFTA_RULE_POSITION_ID]) {
3470                         old_rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_POSITION_ID]);
3471                         if (IS_ERR(old_rule)) {
3472                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION_ID]);
3473                                 return PTR_ERR(old_rule);
3474                         }
3475                 }
3476         }
3477
3478         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
3479
3480         n = 0;
3481         size = 0;
3482         if (nla[NFTA_RULE_EXPRESSIONS]) {
3483                 expr_info = kvmalloc_array(NFT_RULE_MAXEXPRS,
3484                                            sizeof(struct nft_expr_info),
3485                                            GFP_KERNEL);
3486                 if (!expr_info)
3487                         return -ENOMEM;
3488
3489                 nla_for_each_nested(tmp, nla[NFTA_RULE_EXPRESSIONS], rem) {
3490                         err = -EINVAL;
3491                         if (nla_type(tmp) != NFTA_LIST_ELEM)
3492                                 goto err_release_expr;
3493                         if (n == NFT_RULE_MAXEXPRS)
3494                                 goto err_release_expr;
3495                         err = nf_tables_expr_parse(&ctx, tmp, &expr_info[n]);
3496                         if (err < 0) {
3497                                 NL_SET_BAD_ATTR(extack, tmp);
3498                                 goto err_release_expr;
3499                         }
3500                         size += expr_info[n].ops->size;
3501                         n++;
3502                 }
3503         }
3504         /* Check for overflow of dlen field */
3505         err = -EFBIG;
3506         if (size >= 1 << 12)
3507                 goto err_release_expr;
3508
3509         if (nla[NFTA_RULE_USERDATA]) {
3510                 ulen = nla_len(nla[NFTA_RULE_USERDATA]);
3511                 if (ulen > 0)
3512                         usize = sizeof(struct nft_userdata) + ulen;
3513         }
3514
3515         err = -ENOMEM;
3516         rule = kzalloc(sizeof(*rule) + size + usize, GFP_KERNEL_ACCOUNT);
3517         if (rule == NULL)
3518                 goto err_release_expr;
3519
3520         nft_activate_next(net, rule);
3521
3522         rule->handle = handle;
3523         rule->dlen   = size;
3524         rule->udata  = ulen ? 1 : 0;
3525
3526         if (ulen) {
3527                 udata = nft_userdata(rule);
3528                 udata->len = ulen - 1;
3529                 nla_memcpy(udata->data, nla[NFTA_RULE_USERDATA], ulen);
3530         }
3531
3532         expr = nft_expr_first(rule);
3533         for (i = 0; i < n; i++) {
3534                 err = nf_tables_newexpr(&ctx, &expr_info[i], expr);
3535                 if (err < 0) {
3536                         NL_SET_BAD_ATTR(extack, expr_info[i].attr);
3537                         goto err_release_rule;
3538                 }
3539
3540                 if (expr_info[i].ops->validate)
3541                         nft_validate_state_update(net, NFT_VALIDATE_NEED);
3542
3543                 expr_info[i].ops = NULL;
3544                 expr = nft_expr_next(expr);
3545         }
3546
3547         if (chain->flags & NFT_CHAIN_HW_OFFLOAD) {
3548                 flow = nft_flow_rule_create(net, rule);
3549                 if (IS_ERR(flow)) {
3550                         err = PTR_ERR(flow);
3551                         goto err_release_rule;
3552                 }
3553         }
3554
3555         if (info->nlh->nlmsg_flags & NLM_F_REPLACE) {
3556                 err = nft_delrule(&ctx, old_rule);
3557                 if (err < 0)
3558                         goto err_destroy_flow_rule;
3559
3560                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
3561                 if (trans == NULL) {
3562                         err = -ENOMEM;
3563                         goto err_destroy_flow_rule;
3564                 }
3565                 list_add_tail_rcu(&rule->list, &old_rule->list);
3566         } else {
3567                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
3568                 if (!trans) {
3569                         err = -ENOMEM;
3570                         goto err_destroy_flow_rule;
3571                 }
3572
3573                 if (info->nlh->nlmsg_flags & NLM_F_APPEND) {
3574                         if (old_rule)
3575                                 list_add_rcu(&rule->list, &old_rule->list);
3576                         else
3577                                 list_add_tail_rcu(&rule->list, &chain->rules);
3578                  } else {
3579                         if (old_rule)
3580                                 list_add_tail_rcu(&rule->list, &old_rule->list);
3581                         else
3582                                 list_add_rcu(&rule->list, &chain->rules);
3583                 }
3584         }
3585         kvfree(expr_info);
3586         chain->use++;
3587
3588         if (flow)
3589                 nft_trans_flow_rule(trans) = flow;
3590
3591         if (nft_net->validate_state == NFT_VALIDATE_DO)
3592                 return nft_table_validate(net, table);
3593
3594         return 0;
3595
3596 err_destroy_flow_rule:
3597         if (flow)
3598                 nft_flow_rule_destroy(flow);
3599 err_release_rule:
3600         nf_tables_rule_release(&ctx, rule);
3601 err_release_expr:
3602         for (i = 0; i < n; i++) {
3603                 if (expr_info[i].ops) {
3604                         module_put(expr_info[i].ops->type->owner);
3605                         if (expr_info[i].ops->type->release_ops)
3606                                 expr_info[i].ops->type->release_ops(expr_info[i].ops);
3607                 }
3608         }
3609         kvfree(expr_info);
3610
3611         return err;
3612 }
3613
3614 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
3615                                              const struct nft_chain *chain,
3616                                              const struct nlattr *nla)
3617 {
3618         struct nftables_pernet *nft_net = nft_pernet(net);
3619         u32 id = ntohl(nla_get_be32(nla));
3620         struct nft_trans *trans;
3621
3622         list_for_each_entry(trans, &nft_net->commit_list, list) {
3623                 struct nft_rule *rule = nft_trans_rule(trans);
3624
3625                 if (trans->msg_type == NFT_MSG_NEWRULE &&
3626                     trans->ctx.chain == chain &&
3627                     id == nft_trans_rule_id(trans))
3628                         return rule;
3629         }
3630         return ERR_PTR(-ENOENT);
3631 }
3632
3633 static int nf_tables_delrule(struct sk_buff *skb, const struct nfnl_info *info,
3634                              const struct nlattr * const nla[])
3635 {
3636         struct netlink_ext_ack *extack = info->extack;
3637         u8 genmask = nft_genmask_next(info->net);
3638         u8 family = info->nfmsg->nfgen_family;
3639         struct nft_chain *chain = NULL;
3640         struct net *net = info->net;
3641         struct nft_table *table;
3642         struct nft_rule *rule;
3643         struct nft_ctx ctx;
3644         int err = 0;
3645
3646         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
3647                                  NETLINK_CB(skb).portid);
3648         if (IS_ERR(table)) {
3649                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3650                 return PTR_ERR(table);
3651         }
3652
3653         if (nla[NFTA_RULE_CHAIN]) {
3654                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
3655                                          genmask);
3656                 if (IS_ERR(chain)) {
3657                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3658                         return PTR_ERR(chain);
3659                 }
3660                 if (nft_chain_is_bound(chain))
3661                         return -EOPNOTSUPP;
3662         }
3663
3664         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
3665
3666         if (chain) {
3667                 if (nla[NFTA_RULE_HANDLE]) {
3668                         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
3669                         if (IS_ERR(rule)) {
3670                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3671                                 return PTR_ERR(rule);
3672                         }
3673
3674                         err = nft_delrule(&ctx, rule);
3675                 } else if (nla[NFTA_RULE_ID]) {
3676                         rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_ID]);
3677                         if (IS_ERR(rule)) {
3678                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_ID]);
3679                                 return PTR_ERR(rule);
3680                         }
3681
3682                         err = nft_delrule(&ctx, rule);
3683                 } else {
3684                         err = nft_delrule_by_chain(&ctx);
3685                 }
3686         } else {
3687                 list_for_each_entry(chain, &table->chains, list) {
3688                         if (!nft_is_active_next(net, chain))
3689                                 continue;
3690
3691                         ctx.chain = chain;
3692                         err = nft_delrule_by_chain(&ctx);
3693                         if (err < 0)
3694                                 break;
3695                 }
3696         }
3697
3698         return err;
3699 }
3700
3701 /*
3702  * Sets
3703  */
3704 static const struct nft_set_type *nft_set_types[] = {
3705         &nft_set_hash_fast_type,
3706         &nft_set_hash_type,
3707         &nft_set_rhash_type,
3708         &nft_set_bitmap_type,
3709         &nft_set_rbtree_type,
3710 #if defined(CONFIG_X86_64) && !defined(CONFIG_UML)
3711         &nft_set_pipapo_avx2_type,
3712 #endif
3713         &nft_set_pipapo_type,
3714 };
3715
3716 #define NFT_SET_FEATURES        (NFT_SET_INTERVAL | NFT_SET_MAP | \
3717                                  NFT_SET_TIMEOUT | NFT_SET_OBJECT | \
3718                                  NFT_SET_EVAL)
3719
3720 static bool nft_set_ops_candidate(const struct nft_set_type *type, u32 flags)
3721 {
3722         return (flags & type->features) == (flags & NFT_SET_FEATURES);
3723 }
3724
3725 /*
3726  * Select a set implementation based on the data characteristics and the
3727  * given policy. The total memory use might not be known if no size is
3728  * given, in that case the amount of memory per element is used.
3729  */
3730 static const struct nft_set_ops *
3731 nft_select_set_ops(const struct nft_ctx *ctx,
3732                    const struct nlattr * const nla[],
3733                    const struct nft_set_desc *desc,
3734                    enum nft_set_policies policy)
3735 {
3736         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
3737         const struct nft_set_ops *ops, *bops;
3738         struct nft_set_estimate est, best;
3739         const struct nft_set_type *type;
3740         u32 flags = 0;
3741         int i;
3742
3743         lockdep_assert_held(&nft_net->commit_mutex);
3744         lockdep_nfnl_nft_mutex_not_held();
3745
3746         if (nla[NFTA_SET_FLAGS] != NULL)
3747                 flags = ntohl(nla_get_be32(nla[NFTA_SET_FLAGS]));
3748
3749         bops        = NULL;
3750         best.size   = ~0;
3751         best.lookup = ~0;
3752         best.space  = ~0;
3753
3754         for (i = 0; i < ARRAY_SIZE(nft_set_types); i++) {
3755                 type = nft_set_types[i];
3756                 ops = &type->ops;
3757
3758                 if (!nft_set_ops_candidate(type, flags))
3759                         continue;
3760                 if (!ops->estimate(desc, flags, &est))
3761                         continue;
3762
3763                 switch (policy) {
3764                 case NFT_SET_POL_PERFORMANCE:
3765                         if (est.lookup < best.lookup)
3766                                 break;
3767                         if (est.lookup == best.lookup &&
3768                             est.space < best.space)
3769                                 break;
3770                         continue;
3771                 case NFT_SET_POL_MEMORY:
3772                         if (!desc->size) {
3773                                 if (est.space < best.space)
3774                                         break;
3775                                 if (est.space == best.space &&
3776                                     est.lookup < best.lookup)
3777                                         break;
3778                         } else if (est.size < best.size || !bops) {
3779                                 break;
3780                         }
3781                         continue;
3782                 default:
3783                         break;
3784                 }
3785
3786                 bops = ops;
3787                 best = est;
3788         }
3789
3790         if (bops != NULL)
3791                 return bops;
3792
3793         return ERR_PTR(-EOPNOTSUPP);
3794 }
3795
3796 static const struct nla_policy nft_set_policy[NFTA_SET_MAX + 1] = {
3797         [NFTA_SET_TABLE]                = { .type = NLA_STRING,
3798                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
3799         [NFTA_SET_NAME]                 = { .type = NLA_STRING,
3800                                             .len = NFT_SET_MAXNAMELEN - 1 },
3801         [NFTA_SET_FLAGS]                = { .type = NLA_U32 },
3802         [NFTA_SET_KEY_TYPE]             = { .type = NLA_U32 },
3803         [NFTA_SET_KEY_LEN]              = { .type = NLA_U32 },
3804         [NFTA_SET_DATA_TYPE]            = { .type = NLA_U32 },
3805         [NFTA_SET_DATA_LEN]             = { .type = NLA_U32 },
3806         [NFTA_SET_POLICY]               = { .type = NLA_U32 },
3807         [NFTA_SET_DESC]                 = { .type = NLA_NESTED },
3808         [NFTA_SET_ID]                   = { .type = NLA_U32 },
3809         [NFTA_SET_TIMEOUT]              = { .type = NLA_U64 },
3810         [NFTA_SET_GC_INTERVAL]          = { .type = NLA_U32 },
3811         [NFTA_SET_USERDATA]             = { .type = NLA_BINARY,
3812                                             .len  = NFT_USERDATA_MAXLEN },
3813         [NFTA_SET_OBJ_TYPE]             = { .type = NLA_U32 },
3814         [NFTA_SET_HANDLE]               = { .type = NLA_U64 },
3815         [NFTA_SET_EXPR]                 = { .type = NLA_NESTED },
3816         [NFTA_SET_EXPRESSIONS]          = { .type = NLA_NESTED },
3817 };
3818
3819 static const struct nla_policy nft_set_desc_policy[NFTA_SET_DESC_MAX + 1] = {
3820         [NFTA_SET_DESC_SIZE]            = { .type = NLA_U32 },
3821         [NFTA_SET_DESC_CONCAT]          = { .type = NLA_NESTED },
3822 };
3823
3824 static struct nft_set *nft_set_lookup(const struct nft_table *table,
3825                                       const struct nlattr *nla, u8 genmask)
3826 {
3827         struct nft_set *set;
3828
3829         if (nla == NULL)
3830                 return ERR_PTR(-EINVAL);
3831
3832         list_for_each_entry_rcu(set, &table->sets, list) {
3833                 if (!nla_strcmp(nla, set->name) &&
3834                     nft_active_genmask(set, genmask))
3835                         return set;
3836         }
3837         return ERR_PTR(-ENOENT);
3838 }
3839
3840 static struct nft_set *nft_set_lookup_byhandle(const struct nft_table *table,
3841                                                const struct nlattr *nla,
3842                                                u8 genmask)
3843 {
3844         struct nft_set *set;
3845
3846         list_for_each_entry(set, &table->sets, list) {
3847                 if (be64_to_cpu(nla_get_be64(nla)) == set->handle &&
3848                     nft_active_genmask(set, genmask))
3849                         return set;
3850         }
3851         return ERR_PTR(-ENOENT);
3852 }
3853
3854 static struct nft_set *nft_set_lookup_byid(const struct net *net,
3855                                            const struct nft_table *table,
3856                                            const struct nlattr *nla, u8 genmask)
3857 {
3858         struct nftables_pernet *nft_net = nft_pernet(net);
3859         u32 id = ntohl(nla_get_be32(nla));
3860         struct nft_trans *trans;
3861
3862         list_for_each_entry(trans, &nft_net->commit_list, list) {
3863                 if (trans->msg_type == NFT_MSG_NEWSET) {
3864                         struct nft_set *set = nft_trans_set(trans);
3865
3866                         if (id == nft_trans_set_id(trans) &&
3867                             set->table == table &&
3868                             nft_active_genmask(set, genmask))
3869                                 return set;
3870                 }
3871         }
3872         return ERR_PTR(-ENOENT);
3873 }
3874
3875 struct nft_set *nft_set_lookup_global(const struct net *net,
3876                                       const struct nft_table *table,
3877                                       const struct nlattr *nla_set_name,
3878                                       const struct nlattr *nla_set_id,
3879                                       u8 genmask)
3880 {
3881         struct nft_set *set;
3882
3883         set = nft_set_lookup(table, nla_set_name, genmask);
3884         if (IS_ERR(set)) {
3885                 if (!nla_set_id)
3886                         return set;
3887
3888                 set = nft_set_lookup_byid(net, table, nla_set_id, genmask);
3889         }
3890         return set;
3891 }
3892 EXPORT_SYMBOL_GPL(nft_set_lookup_global);
3893
3894 static int nf_tables_set_alloc_name(struct nft_ctx *ctx, struct nft_set *set,
3895                                     const char *name)
3896 {
3897         const struct nft_set *i;
3898         const char *p;
3899         unsigned long *inuse;
3900         unsigned int n = 0, min = 0;
3901
3902         p = strchr(name, '%');
3903         if (p != NULL) {
3904                 if (p[1] != 'd' || strchr(p + 2, '%'))
3905                         return -EINVAL;
3906
3907                 inuse = (unsigned long *)get_zeroed_page(GFP_KERNEL);
3908                 if (inuse == NULL)
3909                         return -ENOMEM;
3910 cont:
3911                 list_for_each_entry(i, &ctx->table->sets, list) {
3912                         int tmp;
3913
3914                         if (!nft_is_active_next(ctx->net, i))
3915                                 continue;
3916                         if (!sscanf(i->name, name, &tmp))
3917                                 continue;
3918                         if (tmp < min || tmp >= min + BITS_PER_BYTE * PAGE_SIZE)
3919                                 continue;
3920
3921                         set_bit(tmp - min, inuse);
3922                 }
3923
3924                 n = find_first_zero_bit(inuse, BITS_PER_BYTE * PAGE_SIZE);
3925                 if (n >= BITS_PER_BYTE * PAGE_SIZE) {
3926                         min += BITS_PER_BYTE * PAGE_SIZE;
3927                         memset(inuse, 0, PAGE_SIZE);
3928                         goto cont;
3929                 }
3930                 free_page((unsigned long)inuse);
3931         }
3932
3933         set->name = kasprintf(GFP_KERNEL_ACCOUNT, name, min + n);
3934         if (!set->name)
3935                 return -ENOMEM;
3936
3937         list_for_each_entry(i, &ctx->table->sets, list) {
3938                 if (!nft_is_active_next(ctx->net, i))
3939                         continue;
3940                 if (!strcmp(set->name, i->name)) {
3941                         kfree(set->name);
3942                         set->name = NULL;
3943                         return -ENFILE;
3944                 }
3945         }
3946         return 0;
3947 }
3948
3949 int nf_msecs_to_jiffies64(const struct nlattr *nla, u64 *result)
3950 {
3951         u64 ms = be64_to_cpu(nla_get_be64(nla));
3952         u64 max = (u64)(~((u64)0));
3953
3954         max = div_u64(max, NSEC_PER_MSEC);
3955         if (ms >= max)
3956                 return -ERANGE;
3957
3958         ms *= NSEC_PER_MSEC;
3959         *result = nsecs_to_jiffies64(ms);
3960         return 0;
3961 }
3962
3963 __be64 nf_jiffies64_to_msecs(u64 input)
3964 {
3965         return cpu_to_be64(jiffies64_to_msecs(input));
3966 }
3967
3968 static int nf_tables_fill_set_concat(struct sk_buff *skb,
3969                                      const struct nft_set *set)
3970 {
3971         struct nlattr *concat, *field;
3972         int i;
3973
3974         concat = nla_nest_start_noflag(skb, NFTA_SET_DESC_CONCAT);
3975         if (!concat)
3976                 return -ENOMEM;
3977
3978         for (i = 0; i < set->field_count; i++) {
3979                 field = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
3980                 if (!field)
3981                         return -ENOMEM;
3982
3983                 if (nla_put_be32(skb, NFTA_SET_FIELD_LEN,
3984                                  htonl(set->field_len[i])))
3985                         return -ENOMEM;
3986
3987                 nla_nest_end(skb, field);
3988         }
3989
3990         nla_nest_end(skb, concat);
3991
3992         return 0;
3993 }
3994
3995 static int nf_tables_fill_set(struct sk_buff *skb, const struct nft_ctx *ctx,
3996                               const struct nft_set *set, u16 event, u16 flags)
3997 {
3998         struct nlmsghdr *nlh;
3999         u32 portid = ctx->portid;
4000         struct nlattr *nest;
4001         u32 seq = ctx->seq;
4002         int i;
4003
4004         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
4005         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
4006                            NFNETLINK_V0, nft_base_seq(ctx->net));
4007         if (!nlh)
4008                 goto nla_put_failure;
4009
4010         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
4011                 goto nla_put_failure;
4012         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
4013                 goto nla_put_failure;
4014         if (nla_put_be64(skb, NFTA_SET_HANDLE, cpu_to_be64(set->handle),
4015                          NFTA_SET_PAD))
4016                 goto nla_put_failure;
4017         if (set->flags != 0)
4018                 if (nla_put_be32(skb, NFTA_SET_FLAGS, htonl(set->flags)))
4019                         goto nla_put_failure;
4020
4021         if (nla_put_be32(skb, NFTA_SET_KEY_TYPE, htonl(set->ktype)))
4022                 goto nla_put_failure;
4023         if (nla_put_be32(skb, NFTA_SET_KEY_LEN, htonl(set->klen)))
4024                 goto nla_put_failure;
4025         if (set->flags & NFT_SET_MAP) {
4026                 if (nla_put_be32(skb, NFTA_SET_DATA_TYPE, htonl(set->dtype)))
4027                         goto nla_put_failure;
4028                 if (nla_put_be32(skb, NFTA_SET_DATA_LEN, htonl(set->dlen)))
4029                         goto nla_put_failure;
4030         }
4031         if (set->flags & NFT_SET_OBJECT &&
4032             nla_put_be32(skb, NFTA_SET_OBJ_TYPE, htonl(set->objtype)))
4033                 goto nla_put_failure;
4034
4035         if (set->timeout &&
4036             nla_put_be64(skb, NFTA_SET_TIMEOUT,
4037                          nf_jiffies64_to_msecs(set->timeout),
4038                          NFTA_SET_PAD))
4039                 goto nla_put_failure;
4040         if (set->gc_int &&
4041             nla_put_be32(skb, NFTA_SET_GC_INTERVAL, htonl(set->gc_int)))
4042                 goto nla_put_failure;
4043
4044         if (set->policy != NFT_SET_POL_PERFORMANCE) {
4045                 if (nla_put_be32(skb, NFTA_SET_POLICY, htonl(set->policy)))
4046                         goto nla_put_failure;
4047         }
4048
4049         if (set->udata &&
4050             nla_put(skb, NFTA_SET_USERDATA, set->udlen, set->udata))
4051                 goto nla_put_failure;
4052
4053         nest = nla_nest_start_noflag(skb, NFTA_SET_DESC);
4054         if (!nest)
4055                 goto nla_put_failure;
4056         if (set->size &&
4057             nla_put_be32(skb, NFTA_SET_DESC_SIZE, htonl(set->size)))
4058                 goto nla_put_failure;
4059
4060         if (set->field_count > 1 &&
4061             nf_tables_fill_set_concat(skb, set))
4062                 goto nla_put_failure;
4063
4064         nla_nest_end(skb, nest);
4065
4066         if (set->num_exprs == 1) {
4067                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPR);
4068                 if (nf_tables_fill_expr_info(skb, set->exprs[0]) < 0)
4069                         goto nla_put_failure;
4070
4071                 nla_nest_end(skb, nest);
4072         } else if (set->num_exprs > 1) {
4073                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPRESSIONS);
4074                 if (nest == NULL)
4075                         goto nla_put_failure;
4076
4077                 for (i = 0; i < set->num_exprs; i++) {
4078                         if (nft_expr_dump(skb, NFTA_LIST_ELEM,
4079                                           set->exprs[i]) < 0)
4080                                 goto nla_put_failure;
4081                 }
4082                 nla_nest_end(skb, nest);
4083         }
4084
4085         nlmsg_end(skb, nlh);
4086         return 0;
4087
4088 nla_put_failure:
4089         nlmsg_trim(skb, nlh);
4090         return -1;
4091 }
4092
4093 static void nf_tables_set_notify(const struct nft_ctx *ctx,
4094                                  const struct nft_set *set, int event,
4095                                  gfp_t gfp_flags)
4096 {
4097         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
4098         u32 portid = ctx->portid;
4099         struct sk_buff *skb;
4100         u16 flags = 0;
4101         int err;
4102
4103         if (!ctx->report &&
4104             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
4105                 return;
4106
4107         skb = nlmsg_new(NLMSG_GOODSIZE, gfp_flags);
4108         if (skb == NULL)
4109                 goto err;
4110
4111         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
4112                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
4113
4114         err = nf_tables_fill_set(skb, ctx, set, event, flags);
4115         if (err < 0) {
4116                 kfree_skb(skb);
4117                 goto err;
4118         }
4119
4120         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
4121         return;
4122 err:
4123         nfnetlink_set_err(ctx->net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
4124 }
4125
4126 static int nf_tables_dump_sets(struct sk_buff *skb, struct netlink_callback *cb)
4127 {
4128         const struct nft_set *set;
4129         unsigned int idx, s_idx = cb->args[0];
4130         struct nft_table *table, *cur_table = (struct nft_table *)cb->args[2];
4131         struct net *net = sock_net(skb->sk);
4132         struct nft_ctx *ctx = cb->data, ctx_set;
4133         struct nftables_pernet *nft_net;
4134
4135         if (cb->args[1])
4136                 return skb->len;
4137
4138         rcu_read_lock();
4139         nft_net = nft_pernet(net);
4140         cb->seq = READ_ONCE(nft_net->base_seq);
4141
4142         list_for_each_entry_rcu(table, &nft_net->tables, list) {
4143                 if (ctx->family != NFPROTO_UNSPEC &&
4144                     ctx->family != table->family)
4145                         continue;
4146
4147                 if (ctx->table && ctx->table != table)
4148                         continue;
4149
4150                 if (cur_table) {
4151                         if (cur_table != table)
4152                                 continue;
4153
4154                         cur_table = NULL;
4155                 }
4156                 idx = 0;
4157                 list_for_each_entry_rcu(set, &table->sets, list) {
4158                         if (idx < s_idx)
4159                                 goto cont;
4160                         if (!nft_is_active(net, set))
4161                                 goto cont;
4162
4163                         ctx_set = *ctx;
4164                         ctx_set.table = table;
4165                         ctx_set.family = table->family;
4166
4167                         if (nf_tables_fill_set(skb, &ctx_set, set,
4168                                                NFT_MSG_NEWSET,
4169                                                NLM_F_MULTI) < 0) {
4170                                 cb->args[0] = idx;
4171                                 cb->args[2] = (unsigned long) table;
4172                                 goto done;
4173                         }
4174                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
4175 cont:
4176                         idx++;
4177                 }
4178                 if (s_idx)
4179                         s_idx = 0;
4180         }
4181         cb->args[1] = 1;
4182 done:
4183         rcu_read_unlock();
4184         return skb->len;
4185 }
4186
4187 static int nf_tables_dump_sets_start(struct netlink_callback *cb)
4188 {
4189         struct nft_ctx *ctx_dump = NULL;
4190
4191         ctx_dump = kmemdup(cb->data, sizeof(*ctx_dump), GFP_ATOMIC);
4192         if (ctx_dump == NULL)
4193                 return -ENOMEM;
4194
4195         cb->data = ctx_dump;
4196         return 0;
4197 }
4198
4199 static int nf_tables_dump_sets_done(struct netlink_callback *cb)
4200 {
4201         kfree(cb->data);
4202         return 0;
4203 }
4204
4205 /* called with rcu_read_lock held */
4206 static int nf_tables_getset(struct sk_buff *skb, const struct nfnl_info *info,
4207                             const struct nlattr * const nla[])
4208 {
4209         struct netlink_ext_ack *extack = info->extack;
4210         u8 genmask = nft_genmask_cur(info->net);
4211         u8 family = info->nfmsg->nfgen_family;
4212         struct nft_table *table = NULL;
4213         struct net *net = info->net;
4214         const struct nft_set *set;
4215         struct sk_buff *skb2;
4216         struct nft_ctx ctx;
4217         int err;
4218
4219         if (nla[NFTA_SET_TABLE]) {
4220                 table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
4221                                          genmask, 0);
4222                 if (IS_ERR(table)) {
4223                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
4224                         return PTR_ERR(table);
4225                 }
4226         }
4227
4228         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
4229
4230         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
4231                 struct netlink_dump_control c = {
4232                         .start = nf_tables_dump_sets_start,
4233                         .dump = nf_tables_dump_sets,
4234                         .done = nf_tables_dump_sets_done,
4235                         .data = &ctx,
4236                         .module = THIS_MODULE,
4237                 };
4238
4239                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
4240         }
4241
4242         /* Only accept unspec with dump */
4243         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
4244                 return -EAFNOSUPPORT;
4245         if (!nla[NFTA_SET_TABLE])
4246                 return -EINVAL;
4247
4248         set = nft_set_lookup(table, nla[NFTA_SET_NAME], genmask);
4249         if (IS_ERR(set))
4250                 return PTR_ERR(set);
4251
4252         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
4253         if (skb2 == NULL)
4254                 return -ENOMEM;
4255
4256         err = nf_tables_fill_set(skb2, &ctx, set, NFT_MSG_NEWSET, 0);
4257         if (err < 0)
4258                 goto err_fill_set_info;
4259
4260         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
4261
4262 err_fill_set_info:
4263         kfree_skb(skb2);
4264         return err;
4265 }
4266
4267 static const struct nla_policy nft_concat_policy[NFTA_SET_FIELD_MAX + 1] = {
4268         [NFTA_SET_FIELD_LEN]    = { .type = NLA_U32 },
4269 };
4270
4271 static int nft_set_desc_concat_parse(const struct nlattr *attr,
4272                                      struct nft_set_desc *desc)
4273 {
4274         struct nlattr *tb[NFTA_SET_FIELD_MAX + 1];
4275         u32 len;
4276         int err;
4277
4278         if (desc->field_count >= ARRAY_SIZE(desc->field_len))
4279                 return -E2BIG;
4280
4281         err = nla_parse_nested_deprecated(tb, NFTA_SET_FIELD_MAX, attr,
4282                                           nft_concat_policy, NULL);
4283         if (err < 0)
4284                 return err;
4285
4286         if (!tb[NFTA_SET_FIELD_LEN])
4287                 return -EINVAL;
4288
4289         len = ntohl(nla_get_be32(tb[NFTA_SET_FIELD_LEN]));
4290         if (!len || len > U8_MAX)
4291                 return -EINVAL;
4292
4293         desc->field_len[desc->field_count++] = len;
4294
4295         return 0;
4296 }
4297
4298 static int nft_set_desc_concat(struct nft_set_desc *desc,
4299                                const struct nlattr *nla)
4300 {
4301         struct nlattr *attr;
4302         u32 num_regs = 0;
4303         int rem, err, i;
4304
4305         nla_for_each_nested(attr, nla, rem) {
4306                 if (nla_type(attr) != NFTA_LIST_ELEM)
4307                         return -EINVAL;
4308
4309                 err = nft_set_desc_concat_parse(attr, desc);
4310                 if (err < 0)
4311                         return err;
4312         }
4313
4314         for (i = 0; i < desc->field_count; i++)
4315                 num_regs += DIV_ROUND_UP(desc->field_len[i], sizeof(u32));
4316
4317         if (num_regs > NFT_REG32_COUNT)
4318                 return -E2BIG;
4319
4320         return 0;
4321 }
4322
4323 static int nf_tables_set_desc_parse(struct nft_set_desc *desc,
4324                                     const struct nlattr *nla)
4325 {
4326         struct nlattr *da[NFTA_SET_DESC_MAX + 1];
4327         int err;
4328
4329         err = nla_parse_nested_deprecated(da, NFTA_SET_DESC_MAX, nla,
4330                                           nft_set_desc_policy, NULL);
4331         if (err < 0)
4332                 return err;
4333
4334         if (da[NFTA_SET_DESC_SIZE] != NULL)
4335                 desc->size = ntohl(nla_get_be32(da[NFTA_SET_DESC_SIZE]));
4336         if (da[NFTA_SET_DESC_CONCAT])
4337                 err = nft_set_desc_concat(desc, da[NFTA_SET_DESC_CONCAT]);
4338
4339         return err;
4340 }
4341
4342 static int nf_tables_newset(struct sk_buff *skb, const struct nfnl_info *info,
4343                             const struct nlattr * const nla[])
4344 {
4345         u32 ktype, dtype, flags, policy, gc_int, objtype;
4346         struct netlink_ext_ack *extack = info->extack;
4347         u8 genmask = nft_genmask_next(info->net);
4348         u8 family = info->nfmsg->nfgen_family;
4349         const struct nft_set_ops *ops;
4350         struct nft_expr *expr = NULL;
4351         struct net *net = info->net;
4352         struct nft_set_desc desc;
4353         struct nft_table *table;
4354         unsigned char *udata;
4355         struct nft_set *set;
4356         struct nft_ctx ctx;
4357         size_t alloc_size;
4358         u64 timeout;
4359         char *name;
4360         int err, i;
4361         u16 udlen;
4362         u64 size;
4363
4364         if (nla[NFTA_SET_TABLE] == NULL ||
4365             nla[NFTA_SET_NAME] == NULL ||
4366             nla[NFTA_SET_KEY_LEN] == NULL ||
4367             nla[NFTA_SET_ID] == NULL)
4368                 return -EINVAL;
4369
4370         memset(&desc, 0, sizeof(desc));
4371
4372         ktype = NFT_DATA_VALUE;
4373         if (nla[NFTA_SET_KEY_TYPE] != NULL) {
4374                 ktype = ntohl(nla_get_be32(nla[NFTA_SET_KEY_TYPE]));
4375                 if ((ktype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK)
4376                         return -EINVAL;
4377         }
4378
4379         desc.klen = ntohl(nla_get_be32(nla[NFTA_SET_KEY_LEN]));
4380         if (desc.klen == 0 || desc.klen > NFT_DATA_VALUE_MAXLEN)
4381                 return -EINVAL;
4382
4383         flags = 0;
4384         if (nla[NFTA_SET_FLAGS] != NULL) {
4385                 flags = ntohl(nla_get_be32(nla[NFTA_SET_FLAGS]));
4386                 if (flags & ~(NFT_SET_ANONYMOUS | NFT_SET_CONSTANT |
4387                               NFT_SET_INTERVAL | NFT_SET_TIMEOUT |
4388                               NFT_SET_MAP | NFT_SET_EVAL |
4389                               NFT_SET_OBJECT | NFT_SET_CONCAT | NFT_SET_EXPR))
4390                         return -EOPNOTSUPP;
4391                 /* Only one of these operations is supported */
4392                 if ((flags & (NFT_SET_MAP | NFT_SET_OBJECT)) ==
4393                              (NFT_SET_MAP | NFT_SET_OBJECT))
4394                         return -EOPNOTSUPP;
4395                 if ((flags & (NFT_SET_EVAL | NFT_SET_OBJECT)) ==
4396                              (NFT_SET_EVAL | NFT_SET_OBJECT))
4397                         return -EOPNOTSUPP;
4398         }
4399
4400         dtype = 0;
4401         if (nla[NFTA_SET_DATA_TYPE] != NULL) {
4402                 if (!(flags & NFT_SET_MAP))
4403                         return -EINVAL;
4404
4405                 dtype = ntohl(nla_get_be32(nla[NFTA_SET_DATA_TYPE]));
4406                 if ((dtype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK &&
4407                     dtype != NFT_DATA_VERDICT)
4408                         return -EINVAL;
4409
4410                 if (dtype != NFT_DATA_VERDICT) {
4411                         if (nla[NFTA_SET_DATA_LEN] == NULL)
4412                                 return -EINVAL;
4413                         desc.dlen = ntohl(nla_get_be32(nla[NFTA_SET_DATA_LEN]));
4414                         if (desc.dlen == 0 || desc.dlen > NFT_DATA_VALUE_MAXLEN)
4415                                 return -EINVAL;
4416                 } else
4417                         desc.dlen = sizeof(struct nft_verdict);
4418         } else if (flags & NFT_SET_MAP)
4419                 return -EINVAL;
4420
4421         if (nla[NFTA_SET_OBJ_TYPE] != NULL) {
4422                 if (!(flags & NFT_SET_OBJECT))
4423                         return -EINVAL;
4424
4425                 objtype = ntohl(nla_get_be32(nla[NFTA_SET_OBJ_TYPE]));
4426                 if (objtype == NFT_OBJECT_UNSPEC ||
4427                     objtype > NFT_OBJECT_MAX)
4428                         return -EOPNOTSUPP;
4429         } else if (flags & NFT_SET_OBJECT)
4430                 return -EINVAL;
4431         else
4432                 objtype = NFT_OBJECT_UNSPEC;
4433
4434         timeout = 0;
4435         if (nla[NFTA_SET_TIMEOUT] != NULL) {
4436                 if (!(flags & NFT_SET_TIMEOUT))
4437                         return -EINVAL;
4438
4439                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_TIMEOUT], &timeout);
4440                 if (err)
4441                         return err;
4442         }
4443         gc_int = 0;
4444         if (nla[NFTA_SET_GC_INTERVAL] != NULL) {
4445                 if (!(flags & NFT_SET_TIMEOUT))
4446                         return -EINVAL;
4447                 gc_int = ntohl(nla_get_be32(nla[NFTA_SET_GC_INTERVAL]));
4448         }
4449
4450         policy = NFT_SET_POL_PERFORMANCE;
4451         if (nla[NFTA_SET_POLICY] != NULL)
4452                 policy = ntohl(nla_get_be32(nla[NFTA_SET_POLICY]));
4453
4454         if (nla[NFTA_SET_DESC] != NULL) {
4455                 err = nf_tables_set_desc_parse(&desc, nla[NFTA_SET_DESC]);
4456                 if (err < 0)
4457                         return err;
4458
4459                 if (desc.field_count > 1 && !(flags & NFT_SET_CONCAT))
4460                         return -EINVAL;
4461         } else if (flags & NFT_SET_CONCAT) {
4462                 return -EINVAL;
4463         }
4464
4465         if (nla[NFTA_SET_EXPR] || nla[NFTA_SET_EXPRESSIONS])
4466                 desc.expr = true;
4467
4468         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family, genmask,
4469                                  NETLINK_CB(skb).portid);
4470         if (IS_ERR(table)) {
4471                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
4472                 return PTR_ERR(table);
4473         }
4474
4475         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
4476
4477         set = nft_set_lookup(table, nla[NFTA_SET_NAME], genmask);
4478         if (IS_ERR(set)) {
4479                 if (PTR_ERR(set) != -ENOENT) {
4480                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
4481                         return PTR_ERR(set);
4482                 }
4483         } else {
4484                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
4485                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
4486                         return -EEXIST;
4487                 }
4488                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
4489                         return -EOPNOTSUPP;
4490
4491                 return 0;
4492         }
4493
4494         if (!(info->nlh->nlmsg_flags & NLM_F_CREATE))
4495                 return -ENOENT;
4496
4497         ops = nft_select_set_ops(&ctx, nla, &desc, policy);
4498         if (IS_ERR(ops))
4499                 return PTR_ERR(ops);
4500
4501         udlen = 0;
4502         if (nla[NFTA_SET_USERDATA])
4503                 udlen = nla_len(nla[NFTA_SET_USERDATA]);
4504
4505         size = 0;
4506         if (ops->privsize != NULL)
4507                 size = ops->privsize(nla, &desc);
4508         alloc_size = sizeof(*set) + size + udlen;
4509         if (alloc_size < size || alloc_size > INT_MAX)
4510                 return -ENOMEM;
4511         set = kvzalloc(alloc_size, GFP_KERNEL_ACCOUNT);
4512         if (!set)
4513                 return -ENOMEM;
4514
4515         name = nla_strdup(nla[NFTA_SET_NAME], GFP_KERNEL_ACCOUNT);
4516         if (!name) {
4517                 err = -ENOMEM;
4518                 goto err_set_name;
4519         }
4520
4521         err = nf_tables_set_alloc_name(&ctx, set, name);
4522         kfree(name);
4523         if (err < 0)
4524                 goto err_set_name;
4525
4526         udata = NULL;
4527         if (udlen) {
4528                 udata = set->data + size;
4529                 nla_memcpy(udata, nla[NFTA_SET_USERDATA], udlen);
4530         }
4531
4532         INIT_LIST_HEAD(&set->bindings);
4533         INIT_LIST_HEAD(&set->catchall_list);
4534         set->table = table;
4535         write_pnet(&set->net, net);
4536         set->ops = ops;
4537         set->ktype = ktype;
4538         set->klen = desc.klen;
4539         set->dtype = dtype;
4540         set->objtype = objtype;
4541         set->dlen = desc.dlen;
4542         set->flags = flags;
4543         set->size = desc.size;
4544         set->policy = policy;
4545         set->udlen = udlen;
4546         set->udata = udata;
4547         set->timeout = timeout;
4548         set->gc_int = gc_int;
4549
4550         set->field_count = desc.field_count;
4551         for (i = 0; i < desc.field_count; i++)
4552                 set->field_len[i] = desc.field_len[i];
4553
4554         err = ops->init(set, &desc, nla);
4555         if (err < 0)
4556                 goto err_set_init;
4557
4558         if (nla[NFTA_SET_EXPR]) {
4559                 expr = nft_set_elem_expr_alloc(&ctx, set, nla[NFTA_SET_EXPR]);
4560                 if (IS_ERR(expr)) {
4561                         err = PTR_ERR(expr);
4562                         goto err_set_expr_alloc;
4563                 }
4564                 set->exprs[0] = expr;
4565                 set->num_exprs++;
4566         } else if (nla[NFTA_SET_EXPRESSIONS]) {
4567                 struct nft_expr *expr;
4568                 struct nlattr *tmp;
4569                 int left;
4570
4571                 if (!(flags & NFT_SET_EXPR)) {
4572                         err = -EINVAL;
4573                         goto err_set_expr_alloc;
4574                 }
4575                 i = 0;
4576                 nla_for_each_nested(tmp, nla[NFTA_SET_EXPRESSIONS], left) {
4577                         if (i == NFT_SET_EXPR_MAX) {
4578                                 err = -E2BIG;
4579                                 goto err_set_expr_alloc;
4580                         }
4581                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
4582                                 err = -EINVAL;
4583                                 goto err_set_expr_alloc;
4584                         }
4585                         expr = nft_set_elem_expr_alloc(&ctx, set, tmp);
4586                         if (IS_ERR(expr)) {
4587                                 err = PTR_ERR(expr);
4588                                 goto err_set_expr_alloc;
4589                         }
4590                         set->exprs[i++] = expr;
4591                         set->num_exprs++;
4592                 }
4593         }
4594
4595         set->handle = nf_tables_alloc_handle(table);
4596
4597         err = nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set);
4598         if (err < 0)
4599                 goto err_set_expr_alloc;
4600
4601         list_add_tail_rcu(&set->list, &table->sets);
4602         table->use++;
4603         return 0;
4604
4605 err_set_expr_alloc:
4606         for (i = 0; i < set->num_exprs; i++)
4607                 nft_expr_destroy(&ctx, set->exprs[i]);
4608
4609         ops->destroy(set);
4610 err_set_init:
4611         kfree(set->name);
4612 err_set_name:
4613         kvfree(set);
4614         return err;
4615 }
4616
4617 struct nft_set_elem_catchall {
4618         struct list_head        list;
4619         struct rcu_head         rcu;
4620         void                    *elem;
4621 };
4622
4623 static void nft_set_catchall_destroy(const struct nft_ctx *ctx,
4624                                      struct nft_set *set)
4625 {
4626         struct nft_set_elem_catchall *next, *catchall;
4627
4628         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
4629                 list_del_rcu(&catchall->list);
4630                 nft_set_elem_destroy(set, catchall->elem, true);
4631                 kfree_rcu(catchall, rcu);
4632         }
4633 }
4634
4635 static void nft_set_destroy(const struct nft_ctx *ctx, struct nft_set *set)
4636 {
4637         int i;
4638
4639         if (WARN_ON(set->use > 0))
4640                 return;
4641
4642         for (i = 0; i < set->num_exprs; i++)
4643                 nft_expr_destroy(ctx, set->exprs[i]);
4644
4645         set->ops->destroy(set);
4646         nft_set_catchall_destroy(ctx, set);
4647         kfree(set->name);
4648         kvfree(set);
4649 }
4650
4651 static int nf_tables_delset(struct sk_buff *skb, const struct nfnl_info *info,
4652                             const struct nlattr * const nla[])
4653 {
4654         struct netlink_ext_ack *extack = info->extack;
4655         u8 genmask = nft_genmask_next(info->net);
4656         u8 family = info->nfmsg->nfgen_family;
4657         struct net *net = info->net;
4658         const struct nlattr *attr;
4659         struct nft_table *table;
4660         struct nft_set *set;
4661         struct nft_ctx ctx;
4662
4663         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
4664                 return -EAFNOSUPPORT;
4665
4666         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
4667                                  genmask, NETLINK_CB(skb).portid);
4668         if (IS_ERR(table)) {
4669                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
4670                 return PTR_ERR(table);
4671         }
4672
4673         if (nla[NFTA_SET_HANDLE]) {
4674                 attr = nla[NFTA_SET_HANDLE];
4675                 set = nft_set_lookup_byhandle(table, attr, genmask);
4676         } else {
4677                 attr = nla[NFTA_SET_NAME];
4678                 set = nft_set_lookup(table, attr, genmask);
4679         }
4680
4681         if (IS_ERR(set)) {
4682                 NL_SET_BAD_ATTR(extack, attr);
4683                 return PTR_ERR(set);
4684         }
4685         if (set->use ||
4686             (info->nlh->nlmsg_flags & NLM_F_NONREC &&
4687              atomic_read(&set->nelems) > 0)) {
4688                 NL_SET_BAD_ATTR(extack, attr);
4689                 return -EBUSY;
4690         }
4691
4692         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
4693
4694         return nft_delset(&ctx, set);
4695 }
4696
4697 static int nft_validate_register_store(const struct nft_ctx *ctx,
4698                                        enum nft_registers reg,
4699                                        const struct nft_data *data,
4700                                        enum nft_data_types type,
4701                                        unsigned int len);
4702
4703 static int nft_setelem_data_validate(const struct nft_ctx *ctx,
4704                                      struct nft_set *set,
4705                                      struct nft_set_elem *elem)
4706 {
4707         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
4708         enum nft_registers dreg;
4709
4710         dreg = nft_type_to_reg(set->dtype);
4711         return nft_validate_register_store(ctx, dreg, nft_set_ext_data(ext),
4712                                            set->dtype == NFT_DATA_VERDICT ?
4713                                            NFT_DATA_VERDICT : NFT_DATA_VALUE,
4714                                            set->dlen);
4715 }
4716
4717 static int nf_tables_bind_check_setelem(const struct nft_ctx *ctx,
4718                                         struct nft_set *set,
4719                                         const struct nft_set_iter *iter,
4720                                         struct nft_set_elem *elem)
4721 {
4722         return nft_setelem_data_validate(ctx, set, elem);
4723 }
4724
4725 static int nft_set_catchall_bind_check(const struct nft_ctx *ctx,
4726                                        struct nft_set *set)
4727 {
4728         u8 genmask = nft_genmask_next(ctx->net);
4729         struct nft_set_elem_catchall *catchall;
4730         struct nft_set_elem elem;
4731         struct nft_set_ext *ext;
4732         int ret = 0;
4733
4734         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
4735                 ext = nft_set_elem_ext(set, catchall->elem);
4736                 if (!nft_set_elem_active(ext, genmask))
4737                         continue;
4738
4739                 elem.priv = catchall->elem;
4740                 ret = nft_setelem_data_validate(ctx, set, &elem);
4741                 if (ret < 0)
4742                         break;
4743         }
4744
4745         return ret;
4746 }
4747
4748 int nf_tables_bind_set(const struct nft_ctx *ctx, struct nft_set *set,
4749                        struct nft_set_binding *binding)
4750 {
4751         struct nft_set_binding *i;
4752         struct nft_set_iter iter;
4753
4754         if (set->use == UINT_MAX)
4755                 return -EOVERFLOW;
4756
4757         if (!list_empty(&set->bindings) && nft_set_is_anonymous(set))
4758                 return -EBUSY;
4759
4760         if (binding->flags & NFT_SET_MAP) {
4761                 /* If the set is already bound to the same chain all
4762                  * jumps are already validated for that chain.
4763                  */
4764                 list_for_each_entry(i, &set->bindings, list) {
4765                         if (i->flags & NFT_SET_MAP &&
4766                             i->chain == binding->chain)
4767                                 goto bind;
4768                 }
4769
4770                 iter.genmask    = nft_genmask_next(ctx->net);
4771                 iter.skip       = 0;
4772                 iter.count      = 0;
4773                 iter.err        = 0;
4774                 iter.fn         = nf_tables_bind_check_setelem;
4775
4776                 set->ops->walk(ctx, set, &iter);
4777                 if (!iter.err)
4778                         iter.err = nft_set_catchall_bind_check(ctx, set);
4779
4780                 if (iter.err < 0)
4781                         return iter.err;
4782         }
4783 bind:
4784         binding->chain = ctx->chain;
4785         list_add_tail_rcu(&binding->list, &set->bindings);
4786         nft_set_trans_bind(ctx, set);
4787         set->use++;
4788
4789         return 0;
4790 }
4791 EXPORT_SYMBOL_GPL(nf_tables_bind_set);
4792
4793 static void nf_tables_unbind_set(const struct nft_ctx *ctx, struct nft_set *set,
4794                                  struct nft_set_binding *binding, bool event)
4795 {
4796         list_del_rcu(&binding->list);
4797
4798         if (list_empty(&set->bindings) && nft_set_is_anonymous(set)) {
4799                 list_del_rcu(&set->list);
4800                 if (event)
4801                         nf_tables_set_notify(ctx, set, NFT_MSG_DELSET,
4802                                              GFP_KERNEL);
4803         }
4804 }
4805
4806 void nf_tables_deactivate_set(const struct nft_ctx *ctx, struct nft_set *set,
4807                               struct nft_set_binding *binding,
4808                               enum nft_trans_phase phase)
4809 {
4810         switch (phase) {
4811         case NFT_TRANS_PREPARE:
4812                 set->use--;
4813                 return;
4814         case NFT_TRANS_ABORT:
4815         case NFT_TRANS_RELEASE:
4816                 set->use--;
4817                 fallthrough;
4818         default:
4819                 nf_tables_unbind_set(ctx, set, binding,
4820                                      phase == NFT_TRANS_COMMIT);
4821         }
4822 }
4823 EXPORT_SYMBOL_GPL(nf_tables_deactivate_set);
4824
4825 void nf_tables_destroy_set(const struct nft_ctx *ctx, struct nft_set *set)
4826 {
4827         if (list_empty(&set->bindings) && nft_set_is_anonymous(set))
4828                 nft_set_destroy(ctx, set);
4829 }
4830 EXPORT_SYMBOL_GPL(nf_tables_destroy_set);
4831
4832 const struct nft_set_ext_type nft_set_ext_types[] = {
4833         [NFT_SET_EXT_KEY]               = {
4834                 .align  = __alignof__(u32),
4835         },
4836         [NFT_SET_EXT_DATA]              = {
4837                 .align  = __alignof__(u32),
4838         },
4839         [NFT_SET_EXT_EXPRESSIONS]       = {
4840                 .align  = __alignof__(struct nft_set_elem_expr),
4841         },
4842         [NFT_SET_EXT_OBJREF]            = {
4843                 .len    = sizeof(struct nft_object *),
4844                 .align  = __alignof__(struct nft_object *),
4845         },
4846         [NFT_SET_EXT_FLAGS]             = {
4847                 .len    = sizeof(u8),
4848                 .align  = __alignof__(u8),
4849         },
4850         [NFT_SET_EXT_TIMEOUT]           = {
4851                 .len    = sizeof(u64),
4852                 .align  = __alignof__(u64),
4853         },
4854         [NFT_SET_EXT_EXPIRATION]        = {
4855                 .len    = sizeof(u64),
4856                 .align  = __alignof__(u64),
4857         },
4858         [NFT_SET_EXT_USERDATA]          = {
4859                 .len    = sizeof(struct nft_userdata),
4860                 .align  = __alignof__(struct nft_userdata),
4861         },
4862         [NFT_SET_EXT_KEY_END]           = {
4863                 .align  = __alignof__(u32),
4864         },
4865 };
4866
4867 /*
4868  * Set elements
4869  */
4870
4871 static const struct nla_policy nft_set_elem_policy[NFTA_SET_ELEM_MAX + 1] = {
4872         [NFTA_SET_ELEM_KEY]             = { .type = NLA_NESTED },
4873         [NFTA_SET_ELEM_DATA]            = { .type = NLA_NESTED },
4874         [NFTA_SET_ELEM_FLAGS]           = { .type = NLA_U32 },
4875         [NFTA_SET_ELEM_TIMEOUT]         = { .type = NLA_U64 },
4876         [NFTA_SET_ELEM_EXPIRATION]      = { .type = NLA_U64 },
4877         [NFTA_SET_ELEM_USERDATA]        = { .type = NLA_BINARY,
4878                                             .len = NFT_USERDATA_MAXLEN },
4879         [NFTA_SET_ELEM_EXPR]            = { .type = NLA_NESTED },
4880         [NFTA_SET_ELEM_OBJREF]          = { .type = NLA_STRING,
4881                                             .len = NFT_OBJ_MAXNAMELEN - 1 },
4882         [NFTA_SET_ELEM_KEY_END]         = { .type = NLA_NESTED },
4883         [NFTA_SET_ELEM_EXPRESSIONS]     = { .type = NLA_NESTED },
4884 };
4885
4886 static const struct nla_policy nft_set_elem_list_policy[NFTA_SET_ELEM_LIST_MAX + 1] = {
4887         [NFTA_SET_ELEM_LIST_TABLE]      = { .type = NLA_STRING,
4888                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
4889         [NFTA_SET_ELEM_LIST_SET]        = { .type = NLA_STRING,
4890                                             .len = NFT_SET_MAXNAMELEN - 1 },
4891         [NFTA_SET_ELEM_LIST_ELEMENTS]   = { .type = NLA_NESTED },
4892         [NFTA_SET_ELEM_LIST_SET_ID]     = { .type = NLA_U32 },
4893 };
4894
4895 static int nft_set_elem_expr_dump(struct sk_buff *skb,
4896                                   const struct nft_set *set,
4897                                   const struct nft_set_ext *ext)
4898 {
4899         struct nft_set_elem_expr *elem_expr;
4900         u32 size, num_exprs = 0;
4901         struct nft_expr *expr;
4902         struct nlattr *nest;
4903
4904         elem_expr = nft_set_ext_expr(ext);
4905         nft_setelem_expr_foreach(expr, elem_expr, size)
4906                 num_exprs++;
4907
4908         if (num_exprs == 1) {
4909                 expr = nft_setelem_expr_at(elem_expr, 0);
4910                 if (nft_expr_dump(skb, NFTA_SET_ELEM_EXPR, expr) < 0)
4911                         return -1;
4912
4913                 return 0;
4914         } else if (num_exprs > 1) {
4915                 nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_EXPRESSIONS);
4916                 if (nest == NULL)
4917                         goto nla_put_failure;
4918
4919                 nft_setelem_expr_foreach(expr, elem_expr, size) {
4920                         expr = nft_setelem_expr_at(elem_expr, size);
4921                         if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr) < 0)
4922                                 goto nla_put_failure;
4923                 }
4924                 nla_nest_end(skb, nest);
4925         }
4926         return 0;
4927
4928 nla_put_failure:
4929         return -1;
4930 }
4931
4932 static int nf_tables_fill_setelem(struct sk_buff *skb,
4933                                   const struct nft_set *set,
4934                                   const struct nft_set_elem *elem)
4935 {
4936         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
4937         unsigned char *b = skb_tail_pointer(skb);
4938         struct nlattr *nest;
4939
4940         nest = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
4941         if (nest == NULL)
4942                 goto nla_put_failure;
4943
4944         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
4945             nft_data_dump(skb, NFTA_SET_ELEM_KEY, nft_set_ext_key(ext),
4946                           NFT_DATA_VALUE, set->klen) < 0)
4947                 goto nla_put_failure;
4948
4949         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
4950             nft_data_dump(skb, NFTA_SET_ELEM_KEY_END, nft_set_ext_key_end(ext),
4951                           NFT_DATA_VALUE, set->klen) < 0)
4952                 goto nla_put_failure;
4953
4954         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
4955             nft_data_dump(skb, NFTA_SET_ELEM_DATA, nft_set_ext_data(ext),
4956                           set->dtype == NFT_DATA_VERDICT ? NFT_DATA_VERDICT : NFT_DATA_VALUE,
4957                           set->dlen) < 0)
4958                 goto nla_put_failure;
4959
4960         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS) &&
4961             nft_set_elem_expr_dump(skb, set, ext))
4962                 goto nla_put_failure;
4963
4964         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
4965             nla_put_string(skb, NFTA_SET_ELEM_OBJREF,
4966                            (*nft_set_ext_obj(ext))->key.name) < 0)
4967                 goto nla_put_failure;
4968
4969         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
4970             nla_put_be32(skb, NFTA_SET_ELEM_FLAGS,
4971                          htonl(*nft_set_ext_flags(ext))))
4972                 goto nla_put_failure;
4973
4974         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT) &&
4975             nla_put_be64(skb, NFTA_SET_ELEM_TIMEOUT,
4976                          nf_jiffies64_to_msecs(*nft_set_ext_timeout(ext)),
4977                          NFTA_SET_ELEM_PAD))
4978                 goto nla_put_failure;
4979
4980         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
4981                 u64 expires, now = get_jiffies_64();
4982
4983                 expires = *nft_set_ext_expiration(ext);
4984                 if (time_before64(now, expires))
4985                         expires -= now;
4986                 else
4987                         expires = 0;
4988
4989                 if (nla_put_be64(skb, NFTA_SET_ELEM_EXPIRATION,
4990                                  nf_jiffies64_to_msecs(expires),
4991                                  NFTA_SET_ELEM_PAD))
4992                         goto nla_put_failure;
4993         }
4994
4995         if (nft_set_ext_exists(ext, NFT_SET_EXT_USERDATA)) {
4996                 struct nft_userdata *udata;
4997
4998                 udata = nft_set_ext_userdata(ext);
4999                 if (nla_put(skb, NFTA_SET_ELEM_USERDATA,
5000                             udata->len + 1, udata->data))
5001                         goto nla_put_failure;
5002         }
5003
5004         nla_nest_end(skb, nest);
5005         return 0;
5006
5007 nla_put_failure:
5008         nlmsg_trim(skb, b);
5009         return -EMSGSIZE;
5010 }
5011
5012 struct nft_set_dump_args {
5013         const struct netlink_callback   *cb;
5014         struct nft_set_iter             iter;
5015         struct sk_buff                  *skb;
5016 };
5017
5018 static int nf_tables_dump_setelem(const struct nft_ctx *ctx,
5019                                   struct nft_set *set,
5020                                   const struct nft_set_iter *iter,
5021                                   struct nft_set_elem *elem)
5022 {
5023         struct nft_set_dump_args *args;
5024
5025         args = container_of(iter, struct nft_set_dump_args, iter);
5026         return nf_tables_fill_setelem(args->skb, set, elem);
5027 }
5028
5029 struct nft_set_dump_ctx {
5030         const struct nft_set    *set;
5031         struct nft_ctx          ctx;
5032 };
5033
5034 static int nft_set_catchall_dump(struct net *net, struct sk_buff *skb,
5035                                  const struct nft_set *set)
5036 {
5037         struct nft_set_elem_catchall *catchall;
5038         u8 genmask = nft_genmask_cur(net);
5039         struct nft_set_elem elem;
5040         struct nft_set_ext *ext;
5041         int ret = 0;
5042
5043         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5044                 ext = nft_set_elem_ext(set, catchall->elem);
5045                 if (!nft_set_elem_active(ext, genmask) ||
5046                     nft_set_elem_expired(ext))
5047                         continue;
5048
5049                 elem.priv = catchall->elem;
5050                 ret = nf_tables_fill_setelem(skb, set, &elem);
5051                 break;
5052         }
5053
5054         return ret;
5055 }
5056
5057 static int nf_tables_dump_set(struct sk_buff *skb, struct netlink_callback *cb)
5058 {
5059         struct nft_set_dump_ctx *dump_ctx = cb->data;
5060         struct net *net = sock_net(skb->sk);
5061         struct nftables_pernet *nft_net;
5062         struct nft_table *table;
5063         struct nft_set *set;
5064         struct nft_set_dump_args args;
5065         bool set_found = false;
5066         struct nlmsghdr *nlh;
5067         struct nlattr *nest;
5068         u32 portid, seq;
5069         int event;
5070
5071         rcu_read_lock();
5072         nft_net = nft_pernet(net);
5073         cb->seq = READ_ONCE(nft_net->base_seq);
5074
5075         list_for_each_entry_rcu(table, &nft_net->tables, list) {
5076                 if (dump_ctx->ctx.family != NFPROTO_UNSPEC &&
5077                     dump_ctx->ctx.family != table->family)
5078                         continue;
5079
5080                 if (table != dump_ctx->ctx.table)
5081                         continue;
5082
5083                 list_for_each_entry_rcu(set, &table->sets, list) {
5084                         if (set == dump_ctx->set) {
5085                                 set_found = true;
5086                                 break;
5087                         }
5088                 }
5089                 break;
5090         }
5091
5092         if (!set_found) {
5093                 rcu_read_unlock();
5094                 return -ENOENT;
5095         }
5096
5097         event  = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWSETELEM);
5098         portid = NETLINK_CB(cb->skb).portid;
5099         seq    = cb->nlh->nlmsg_seq;
5100
5101         nlh = nfnl_msg_put(skb, portid, seq, event, NLM_F_MULTI,
5102                            table->family, NFNETLINK_V0, nft_base_seq(net));
5103         if (!nlh)
5104                 goto nla_put_failure;
5105
5106         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_TABLE, table->name))
5107                 goto nla_put_failure;
5108         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_SET, set->name))
5109                 goto nla_put_failure;
5110
5111         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
5112         if (nest == NULL)
5113                 goto nla_put_failure;
5114
5115         args.cb                 = cb;
5116         args.skb                = skb;
5117         args.iter.genmask       = nft_genmask_cur(net);
5118         args.iter.skip          = cb->args[0];
5119         args.iter.count         = 0;
5120         args.iter.err           = 0;
5121         args.iter.fn            = nf_tables_dump_setelem;
5122         set->ops->walk(&dump_ctx->ctx, set, &args.iter);
5123
5124         if (!args.iter.err && args.iter.count == cb->args[0])
5125                 args.iter.err = nft_set_catchall_dump(net, skb, set);
5126         rcu_read_unlock();
5127
5128         nla_nest_end(skb, nest);
5129         nlmsg_end(skb, nlh);
5130
5131         if (args.iter.err && args.iter.err != -EMSGSIZE)
5132                 return args.iter.err;
5133         if (args.iter.count == cb->args[0])
5134                 return 0;
5135
5136         cb->args[0] = args.iter.count;
5137         return skb->len;
5138
5139 nla_put_failure:
5140         rcu_read_unlock();
5141         return -ENOSPC;
5142 }
5143
5144 static int nf_tables_dump_set_start(struct netlink_callback *cb)
5145 {
5146         struct nft_set_dump_ctx *dump_ctx = cb->data;
5147
5148         cb->data = kmemdup(dump_ctx, sizeof(*dump_ctx), GFP_ATOMIC);
5149
5150         return cb->data ? 0 : -ENOMEM;
5151 }
5152
5153 static int nf_tables_dump_set_done(struct netlink_callback *cb)
5154 {
5155         kfree(cb->data);
5156         return 0;
5157 }
5158
5159 static int nf_tables_fill_setelem_info(struct sk_buff *skb,
5160                                        const struct nft_ctx *ctx, u32 seq,
5161                                        u32 portid, int event, u16 flags,
5162                                        const struct nft_set *set,
5163                                        const struct nft_set_elem *elem)
5164 {
5165         struct nlmsghdr *nlh;
5166         struct nlattr *nest;
5167         int err;
5168
5169         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
5170         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
5171                            NFNETLINK_V0, nft_base_seq(ctx->net));
5172         if (!nlh)
5173                 goto nla_put_failure;
5174
5175         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
5176                 goto nla_put_failure;
5177         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
5178                 goto nla_put_failure;
5179
5180         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
5181         if (nest == NULL)
5182                 goto nla_put_failure;
5183
5184         err = nf_tables_fill_setelem(skb, set, elem);
5185         if (err < 0)
5186                 goto nla_put_failure;
5187
5188         nla_nest_end(skb, nest);
5189
5190         nlmsg_end(skb, nlh);
5191         return 0;
5192
5193 nla_put_failure:
5194         nlmsg_trim(skb, nlh);
5195         return -1;
5196 }
5197
5198 static int nft_setelem_parse_flags(const struct nft_set *set,
5199                                    const struct nlattr *attr, u32 *flags)
5200 {
5201         if (attr == NULL)
5202                 return 0;
5203
5204         *flags = ntohl(nla_get_be32(attr));
5205         if (*flags & ~(NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
5206                 return -EOPNOTSUPP;
5207         if (!(set->flags & NFT_SET_INTERVAL) &&
5208             *flags & NFT_SET_ELEM_INTERVAL_END)
5209                 return -EINVAL;
5210         if ((*flags & (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL)) ==
5211             (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
5212                 return -EINVAL;
5213
5214         return 0;
5215 }
5216
5217 static int nft_setelem_parse_key(struct nft_ctx *ctx, struct nft_set *set,
5218                                  struct nft_data *key, struct nlattr *attr)
5219 {
5220         struct nft_data_desc desc = {
5221                 .type   = NFT_DATA_VALUE,
5222                 .size   = NFT_DATA_VALUE_MAXLEN,
5223                 .len    = set->klen,
5224         };
5225
5226         return nft_data_init(ctx, key, &desc, attr);
5227 }
5228
5229 static int nft_setelem_parse_data(struct nft_ctx *ctx, struct nft_set *set,
5230                                   struct nft_data_desc *desc,
5231                                   struct nft_data *data,
5232                                   struct nlattr *attr)
5233 {
5234         u32 dtype;
5235
5236         if (set->dtype == NFT_DATA_VERDICT)
5237                 dtype = NFT_DATA_VERDICT;
5238         else
5239                 dtype = NFT_DATA_VALUE;
5240
5241         desc->type = dtype;
5242         desc->size = NFT_DATA_VALUE_MAXLEN;
5243         desc->len = set->dlen;
5244         desc->flags = NFT_DATA_DESC_SETELEM;
5245
5246         return nft_data_init(ctx, data, desc, attr);
5247 }
5248
5249 static void *nft_setelem_catchall_get(const struct net *net,
5250                                       const struct nft_set *set)
5251 {
5252         struct nft_set_elem_catchall *catchall;
5253         u8 genmask = nft_genmask_cur(net);
5254         struct nft_set_ext *ext;
5255         void *priv = NULL;
5256
5257         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5258                 ext = nft_set_elem_ext(set, catchall->elem);
5259                 if (!nft_set_elem_active(ext, genmask) ||
5260                     nft_set_elem_expired(ext))
5261                         continue;
5262
5263                 priv = catchall->elem;
5264                 break;
5265         }
5266
5267         return priv;
5268 }
5269
5270 static int nft_setelem_get(struct nft_ctx *ctx, struct nft_set *set,
5271                            struct nft_set_elem *elem, u32 flags)
5272 {
5273         void *priv;
5274
5275         if (!(flags & NFT_SET_ELEM_CATCHALL)) {
5276                 priv = set->ops->get(ctx->net, set, elem, flags);
5277                 if (IS_ERR(priv))
5278                         return PTR_ERR(priv);
5279         } else {
5280                 priv = nft_setelem_catchall_get(ctx->net, set);
5281                 if (!priv)
5282                         return -ENOENT;
5283         }
5284         elem->priv = priv;
5285
5286         return 0;
5287 }
5288
5289 static int nft_get_set_elem(struct nft_ctx *ctx, struct nft_set *set,
5290                             const struct nlattr *attr)
5291 {
5292         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
5293         struct nft_set_elem elem;
5294         struct sk_buff *skb;
5295         uint32_t flags = 0;
5296         int err;
5297
5298         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
5299                                           nft_set_elem_policy, NULL);
5300         if (err < 0)
5301                 return err;
5302
5303         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
5304         if (err < 0)
5305                 return err;
5306
5307         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
5308                 return -EINVAL;
5309
5310         if (nla[NFTA_SET_ELEM_KEY]) {
5311                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
5312                                             nla[NFTA_SET_ELEM_KEY]);
5313                 if (err < 0)
5314                         return err;
5315         }
5316
5317         if (nla[NFTA_SET_ELEM_KEY_END]) {
5318                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
5319                                             nla[NFTA_SET_ELEM_KEY_END]);
5320                 if (err < 0)
5321                         return err;
5322         }
5323
5324         err = nft_setelem_get(ctx, set, &elem, flags);
5325         if (err < 0)
5326                 return err;
5327
5328         err = -ENOMEM;
5329         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_ATOMIC);
5330         if (skb == NULL)
5331                 return err;
5332
5333         err = nf_tables_fill_setelem_info(skb, ctx, ctx->seq, ctx->portid,
5334                                           NFT_MSG_NEWSETELEM, 0, set, &elem);
5335         if (err < 0)
5336                 goto err_fill_setelem;
5337
5338         return nfnetlink_unicast(skb, ctx->net, ctx->portid);
5339
5340 err_fill_setelem:
5341         kfree_skb(skb);
5342         return err;
5343 }
5344
5345 /* called with rcu_read_lock held */
5346 static int nf_tables_getsetelem(struct sk_buff *skb,
5347                                 const struct nfnl_info *info,
5348                                 const struct nlattr * const nla[])
5349 {
5350         struct netlink_ext_ack *extack = info->extack;
5351         u8 genmask = nft_genmask_cur(info->net);
5352         u8 family = info->nfmsg->nfgen_family;
5353         struct net *net = info->net;
5354         struct nft_table *table;
5355         struct nft_set *set;
5356         struct nlattr *attr;
5357         struct nft_ctx ctx;
5358         int rem, err = 0;
5359
5360         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
5361                                  genmask, NETLINK_CB(skb).portid);
5362         if (IS_ERR(table)) {
5363                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
5364                 return PTR_ERR(table);
5365         }
5366
5367         set = nft_set_lookup(table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
5368         if (IS_ERR(set))
5369                 return PTR_ERR(set);
5370
5371         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
5372
5373         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
5374                 struct netlink_dump_control c = {
5375                         .start = nf_tables_dump_set_start,
5376                         .dump = nf_tables_dump_set,
5377                         .done = nf_tables_dump_set_done,
5378                         .module = THIS_MODULE,
5379                 };
5380                 struct nft_set_dump_ctx dump_ctx = {
5381                         .set = set,
5382                         .ctx = ctx,
5383                 };
5384
5385                 c.data = &dump_ctx;
5386                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
5387         }
5388
5389         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
5390                 return -EINVAL;
5391
5392         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
5393                 err = nft_get_set_elem(&ctx, set, attr);
5394                 if (err < 0) {
5395                         NL_SET_BAD_ATTR(extack, attr);
5396                         break;
5397                 }
5398         }
5399
5400         return err;
5401 }
5402
5403 static void nf_tables_setelem_notify(const struct nft_ctx *ctx,
5404                                      const struct nft_set *set,
5405                                      const struct nft_set_elem *elem,
5406                                      int event)
5407 {
5408         struct nftables_pernet *nft_net;
5409         struct net *net = ctx->net;
5410         u32 portid = ctx->portid;
5411         struct sk_buff *skb;
5412         u16 flags = 0;
5413         int err;
5414
5415         if (!ctx->report && !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
5416                 return;
5417
5418         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
5419         if (skb == NULL)
5420                 goto err;
5421
5422         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
5423                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
5424
5425         err = nf_tables_fill_setelem_info(skb, ctx, 0, portid, event, flags,
5426                                           set, elem);
5427         if (err < 0) {
5428                 kfree_skb(skb);
5429                 goto err;
5430         }
5431
5432         nft_net = nft_pernet(net);
5433         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
5434         return;
5435 err:
5436         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
5437 }
5438
5439 static struct nft_trans *nft_trans_elem_alloc(struct nft_ctx *ctx,
5440                                               int msg_type,
5441                                               struct nft_set *set)
5442 {
5443         struct nft_trans *trans;
5444
5445         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_elem));
5446         if (trans == NULL)
5447                 return NULL;
5448
5449         nft_trans_elem_set(trans) = set;
5450         return trans;
5451 }
5452
5453 struct nft_expr *nft_set_elem_expr_alloc(const struct nft_ctx *ctx,
5454                                          const struct nft_set *set,
5455                                          const struct nlattr *attr)
5456 {
5457         struct nft_expr *expr;
5458         int err;
5459
5460         expr = nft_expr_init(ctx, attr);
5461         if (IS_ERR(expr))
5462                 return expr;
5463
5464         err = -EOPNOTSUPP;
5465         if (expr->ops->type->flags & NFT_EXPR_GC) {
5466                 if (set->flags & NFT_SET_TIMEOUT)
5467                         goto err_set_elem_expr;
5468                 if (!set->ops->gc_init)
5469                         goto err_set_elem_expr;
5470                 set->ops->gc_init(set);
5471         }
5472
5473         return expr;
5474
5475 err_set_elem_expr:
5476         nft_expr_destroy(ctx, expr);
5477         return ERR_PTR(err);
5478 }
5479
5480 static int nft_set_ext_check(const struct nft_set_ext_tmpl *tmpl, u8 id, u32 len)
5481 {
5482         len += nft_set_ext_types[id].len;
5483         if (len > tmpl->ext_len[id] ||
5484             len > U8_MAX)
5485                 return -1;
5486
5487         return 0;
5488 }
5489
5490 static int nft_set_ext_memcpy(const struct nft_set_ext_tmpl *tmpl, u8 id,
5491                               void *to, const void *from, u32 len)
5492 {
5493         if (nft_set_ext_check(tmpl, id, len) < 0)
5494                 return -1;
5495
5496         memcpy(to, from, len);
5497
5498         return 0;
5499 }
5500
5501 void *nft_set_elem_init(const struct nft_set *set,
5502                         const struct nft_set_ext_tmpl *tmpl,
5503                         const u32 *key, const u32 *key_end,
5504                         const u32 *data, u64 timeout, u64 expiration, gfp_t gfp)
5505 {
5506         struct nft_set_ext *ext;
5507         void *elem;
5508
5509         elem = kzalloc(set->ops->elemsize + tmpl->len, gfp);
5510         if (elem == NULL)
5511                 return ERR_PTR(-ENOMEM);
5512
5513         ext = nft_set_elem_ext(set, elem);
5514         nft_set_ext_init(ext, tmpl);
5515
5516         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
5517             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY,
5518                                nft_set_ext_key(ext), key, set->klen) < 0)
5519                 goto err_ext_check;
5520
5521         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
5522             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY_END,
5523                                nft_set_ext_key_end(ext), key_end, set->klen) < 0)
5524                 goto err_ext_check;
5525
5526         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
5527             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_DATA,
5528                                nft_set_ext_data(ext), data, set->dlen) < 0)
5529                 goto err_ext_check;
5530
5531         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
5532                 *nft_set_ext_expiration(ext) = get_jiffies_64() + expiration;
5533                 if (expiration == 0)
5534                         *nft_set_ext_expiration(ext) += timeout;
5535         }
5536         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT))
5537                 *nft_set_ext_timeout(ext) = timeout;
5538
5539         return elem;
5540
5541 err_ext_check:
5542         kfree(elem);
5543
5544         return ERR_PTR(-EINVAL);
5545 }
5546
5547 static void __nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
5548                                         struct nft_expr *expr)
5549 {
5550         if (expr->ops->destroy_clone) {
5551                 expr->ops->destroy_clone(ctx, expr);
5552                 module_put(expr->ops->type->owner);
5553         } else {
5554                 nf_tables_expr_destroy(ctx, expr);
5555         }
5556 }
5557
5558 static void nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
5559                                       struct nft_set_elem_expr *elem_expr)
5560 {
5561         struct nft_expr *expr;
5562         u32 size;
5563
5564         nft_setelem_expr_foreach(expr, elem_expr, size)
5565                 __nft_set_elem_expr_destroy(ctx, expr);
5566 }
5567
5568 void nft_set_elem_destroy(const struct nft_set *set, void *elem,
5569                           bool destroy_expr)
5570 {
5571         struct nft_set_ext *ext = nft_set_elem_ext(set, elem);
5572         struct nft_ctx ctx = {
5573                 .net    = read_pnet(&set->net),
5574                 .family = set->table->family,
5575         };
5576
5577         nft_data_release(nft_set_ext_key(ext), NFT_DATA_VALUE);
5578         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
5579                 nft_data_release(nft_set_ext_data(ext), set->dtype);
5580         if (destroy_expr && nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
5581                 nft_set_elem_expr_destroy(&ctx, nft_set_ext_expr(ext));
5582
5583         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
5584                 (*nft_set_ext_obj(ext))->use--;
5585         kfree(elem);
5586 }
5587 EXPORT_SYMBOL_GPL(nft_set_elem_destroy);
5588
5589 /* Only called from commit path, nft_setelem_data_deactivate() already deals
5590  * with the refcounting from the preparation phase.
5591  */
5592 static void nf_tables_set_elem_destroy(const struct nft_ctx *ctx,
5593                                        const struct nft_set *set, void *elem)
5594 {
5595         struct nft_set_ext *ext = nft_set_elem_ext(set, elem);
5596
5597         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
5598                 nft_set_elem_expr_destroy(ctx, nft_set_ext_expr(ext));
5599
5600         kfree(elem);
5601 }
5602
5603 int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set,
5604                             struct nft_expr *expr_array[])
5605 {
5606         struct nft_expr *expr;
5607         int err, i, k;
5608
5609         for (i = 0; i < set->num_exprs; i++) {
5610                 expr = kzalloc(set->exprs[i]->ops->size, GFP_KERNEL_ACCOUNT);
5611                 if (!expr)
5612                         goto err_expr;
5613
5614                 err = nft_expr_clone(expr, set->exprs[i]);
5615                 if (err < 0) {
5616                         kfree(expr);
5617                         goto err_expr;
5618                 }
5619                 expr_array[i] = expr;
5620         }
5621
5622         return 0;
5623
5624 err_expr:
5625         for (k = i - 1; k >= 0; k--)
5626                 nft_expr_destroy(ctx, expr_array[k]);
5627
5628         return -ENOMEM;
5629 }
5630
5631 static int nft_set_elem_expr_setup(struct nft_ctx *ctx,
5632                                    const struct nft_set_ext_tmpl *tmpl,
5633                                    const struct nft_set_ext *ext,
5634                                    struct nft_expr *expr_array[],
5635                                    u32 num_exprs)
5636 {
5637         struct nft_set_elem_expr *elem_expr = nft_set_ext_expr(ext);
5638         u32 len = sizeof(struct nft_set_elem_expr);
5639         struct nft_expr *expr;
5640         int i, err;
5641
5642         if (num_exprs == 0)
5643                 return 0;
5644
5645         for (i = 0; i < num_exprs; i++)
5646                 len += expr_array[i]->ops->size;
5647
5648         if (nft_set_ext_check(tmpl, NFT_SET_EXT_EXPRESSIONS, len) < 0)
5649                 return -EINVAL;
5650
5651         for (i = 0; i < num_exprs; i++) {
5652                 expr = nft_setelem_expr_at(elem_expr, elem_expr->size);
5653                 err = nft_expr_clone(expr, expr_array[i]);
5654                 if (err < 0)
5655                         goto err_elem_expr_setup;
5656
5657                 elem_expr->size += expr_array[i]->ops->size;
5658                 nft_expr_destroy(ctx, expr_array[i]);
5659                 expr_array[i] = NULL;
5660         }
5661
5662         return 0;
5663
5664 err_elem_expr_setup:
5665         for (; i < num_exprs; i++) {
5666                 nft_expr_destroy(ctx, expr_array[i]);
5667                 expr_array[i] = NULL;
5668         }
5669
5670         return -ENOMEM;
5671 }
5672
5673 struct nft_set_ext *nft_set_catchall_lookup(const struct net *net,
5674                                             const struct nft_set *set)
5675 {
5676         struct nft_set_elem_catchall *catchall;
5677         u8 genmask = nft_genmask_cur(net);
5678         struct nft_set_ext *ext;
5679
5680         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5681                 ext = nft_set_elem_ext(set, catchall->elem);
5682                 if (nft_set_elem_active(ext, genmask) &&
5683                     !nft_set_elem_expired(ext))
5684                         return ext;
5685         }
5686
5687         return NULL;
5688 }
5689 EXPORT_SYMBOL_GPL(nft_set_catchall_lookup);
5690
5691 void *nft_set_catchall_gc(const struct nft_set *set)
5692 {
5693         struct nft_set_elem_catchall *catchall, *next;
5694         struct nft_set_ext *ext;
5695         void *elem = NULL;
5696
5697         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
5698                 ext = nft_set_elem_ext(set, catchall->elem);
5699
5700                 if (!nft_set_elem_expired(ext) ||
5701                     nft_set_elem_mark_busy(ext))
5702                         continue;
5703
5704                 elem = catchall->elem;
5705                 list_del_rcu(&catchall->list);
5706                 kfree_rcu(catchall, rcu);
5707                 break;
5708         }
5709
5710         return elem;
5711 }
5712 EXPORT_SYMBOL_GPL(nft_set_catchall_gc);
5713
5714 static int nft_setelem_catchall_insert(const struct net *net,
5715                                        struct nft_set *set,
5716                                        const struct nft_set_elem *elem,
5717                                        struct nft_set_ext **pext)
5718 {
5719         struct nft_set_elem_catchall *catchall;
5720         u8 genmask = nft_genmask_next(net);
5721         struct nft_set_ext *ext;
5722
5723         list_for_each_entry(catchall, &set->catchall_list, list) {
5724                 ext = nft_set_elem_ext(set, catchall->elem);
5725                 if (nft_set_elem_active(ext, genmask)) {
5726                         *pext = ext;
5727                         return -EEXIST;
5728                 }
5729         }
5730
5731         catchall = kmalloc(sizeof(*catchall), GFP_KERNEL);
5732         if (!catchall)
5733                 return -ENOMEM;
5734
5735         catchall->elem = elem->priv;
5736         list_add_tail_rcu(&catchall->list, &set->catchall_list);
5737
5738         return 0;
5739 }
5740
5741 static int nft_setelem_insert(const struct net *net,
5742                               struct nft_set *set,
5743                               const struct nft_set_elem *elem,
5744                               struct nft_set_ext **ext, unsigned int flags)
5745 {
5746         int ret;
5747
5748         if (flags & NFT_SET_ELEM_CATCHALL)
5749                 ret = nft_setelem_catchall_insert(net, set, elem, ext);
5750         else
5751                 ret = set->ops->insert(net, set, elem, ext);
5752
5753         return ret;
5754 }
5755
5756 static bool nft_setelem_is_catchall(const struct nft_set *set,
5757                                     const struct nft_set_elem *elem)
5758 {
5759         struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
5760
5761         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
5762             *nft_set_ext_flags(ext) & NFT_SET_ELEM_CATCHALL)
5763                 return true;
5764
5765         return false;
5766 }
5767
5768 static void nft_setelem_activate(struct net *net, struct nft_set *set,
5769                                  struct nft_set_elem *elem)
5770 {
5771         struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
5772
5773         if (nft_setelem_is_catchall(set, elem)) {
5774                 nft_set_elem_change_active(net, set, ext);
5775                 nft_set_elem_clear_busy(ext);
5776         } else {
5777                 set->ops->activate(net, set, elem);
5778         }
5779 }
5780
5781 static int nft_setelem_catchall_deactivate(const struct net *net,
5782                                            struct nft_set *set,
5783                                            struct nft_set_elem *elem)
5784 {
5785         struct nft_set_elem_catchall *catchall;
5786         struct nft_set_ext *ext;
5787
5788         list_for_each_entry(catchall, &set->catchall_list, list) {
5789                 ext = nft_set_elem_ext(set, catchall->elem);
5790                 if (!nft_is_active(net, ext) ||
5791                     nft_set_elem_mark_busy(ext))
5792                         continue;
5793
5794                 kfree(elem->priv);
5795                 elem->priv = catchall->elem;
5796                 nft_set_elem_change_active(net, set, ext);
5797                 return 0;
5798         }
5799
5800         return -ENOENT;
5801 }
5802
5803 static int __nft_setelem_deactivate(const struct net *net,
5804                                     struct nft_set *set,
5805                                     struct nft_set_elem *elem)
5806 {
5807         void *priv;
5808
5809         priv = set->ops->deactivate(net, set, elem);
5810         if (!priv)
5811                 return -ENOENT;
5812
5813         kfree(elem->priv);
5814         elem->priv = priv;
5815         set->ndeact++;
5816
5817         return 0;
5818 }
5819
5820 static int nft_setelem_deactivate(const struct net *net,
5821                                   struct nft_set *set,
5822                                   struct nft_set_elem *elem, u32 flags)
5823 {
5824         int ret;
5825
5826         if (flags & NFT_SET_ELEM_CATCHALL)
5827                 ret = nft_setelem_catchall_deactivate(net, set, elem);
5828         else
5829                 ret = __nft_setelem_deactivate(net, set, elem);
5830
5831         return ret;
5832 }
5833
5834 static void nft_setelem_catchall_remove(const struct net *net,
5835                                         const struct nft_set *set,
5836                                         const struct nft_set_elem *elem)
5837 {
5838         struct nft_set_elem_catchall *catchall, *next;
5839
5840         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
5841                 if (catchall->elem == elem->priv) {
5842                         list_del_rcu(&catchall->list);
5843                         kfree_rcu(catchall, rcu);
5844                         break;
5845                 }
5846         }
5847 }
5848
5849 static void nft_setelem_remove(const struct net *net,
5850                                const struct nft_set *set,
5851                                const struct nft_set_elem *elem)
5852 {
5853         if (nft_setelem_is_catchall(set, elem))
5854                 nft_setelem_catchall_remove(net, set, elem);
5855         else
5856                 set->ops->remove(net, set, elem);
5857 }
5858
5859 static bool nft_setelem_valid_key_end(const struct nft_set *set,
5860                                       struct nlattr **nla, u32 flags)
5861 {
5862         if ((set->flags & (NFT_SET_CONCAT | NFT_SET_INTERVAL)) ==
5863                           (NFT_SET_CONCAT | NFT_SET_INTERVAL)) {
5864                 if (flags & NFT_SET_ELEM_INTERVAL_END)
5865                         return false;
5866                 if (!nla[NFTA_SET_ELEM_KEY_END] &&
5867                     !(flags & NFT_SET_ELEM_CATCHALL))
5868                         return false;
5869         } else {
5870                 if (nla[NFTA_SET_ELEM_KEY_END])
5871                         return false;
5872         }
5873
5874         return true;
5875 }
5876
5877 static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set,
5878                             const struct nlattr *attr, u32 nlmsg_flags)
5879 {
5880         struct nft_expr *expr_array[NFT_SET_EXPR_MAX] = {};
5881         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
5882         u8 genmask = nft_genmask_next(ctx->net);
5883         u32 flags = 0, size = 0, num_exprs = 0;
5884         struct nft_set_ext_tmpl tmpl;
5885         struct nft_set_ext *ext, *ext2;
5886         struct nft_set_elem elem;
5887         struct nft_set_binding *binding;
5888         struct nft_object *obj = NULL;
5889         struct nft_userdata *udata;
5890         struct nft_data_desc desc;
5891         enum nft_registers dreg;
5892         struct nft_trans *trans;
5893         u64 timeout;
5894         u64 expiration;
5895         int err, i;
5896         u8 ulen;
5897
5898         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
5899                                           nft_set_elem_policy, NULL);
5900         if (err < 0)
5901                 return err;
5902
5903         nft_set_ext_prepare(&tmpl);
5904
5905         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
5906         if (err < 0)
5907                 return err;
5908
5909         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
5910                 return -EINVAL;
5911
5912         if (flags != 0) {
5913                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
5914                 if (err < 0)
5915                         return err;
5916         }
5917
5918         if (set->flags & NFT_SET_MAP) {
5919                 if (nla[NFTA_SET_ELEM_DATA] == NULL &&
5920                     !(flags & NFT_SET_ELEM_INTERVAL_END))
5921                         return -EINVAL;
5922         } else {
5923                 if (nla[NFTA_SET_ELEM_DATA] != NULL)
5924                         return -EINVAL;
5925         }
5926
5927         if (set->flags & NFT_SET_OBJECT) {
5928                 if (!nla[NFTA_SET_ELEM_OBJREF] &&
5929                     !(flags & NFT_SET_ELEM_INTERVAL_END))
5930                         return -EINVAL;
5931         } else {
5932                 if (nla[NFTA_SET_ELEM_OBJREF])
5933                         return -EINVAL;
5934         }
5935
5936         if (!nft_setelem_valid_key_end(set, nla, flags))
5937                 return -EINVAL;
5938
5939         if ((flags & NFT_SET_ELEM_INTERVAL_END) &&
5940              (nla[NFTA_SET_ELEM_DATA] ||
5941               nla[NFTA_SET_ELEM_OBJREF] ||
5942               nla[NFTA_SET_ELEM_TIMEOUT] ||
5943               nla[NFTA_SET_ELEM_EXPIRATION] ||
5944               nla[NFTA_SET_ELEM_USERDATA] ||
5945               nla[NFTA_SET_ELEM_EXPR] ||
5946               nla[NFTA_SET_ELEM_KEY_END] ||
5947               nla[NFTA_SET_ELEM_EXPRESSIONS]))
5948                 return -EINVAL;
5949
5950         timeout = 0;
5951         if (nla[NFTA_SET_ELEM_TIMEOUT] != NULL) {
5952                 if (!(set->flags & NFT_SET_TIMEOUT))
5953                         return -EINVAL;
5954                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_TIMEOUT],
5955                                             &timeout);
5956                 if (err)
5957                         return err;
5958         } else if (set->flags & NFT_SET_TIMEOUT) {
5959                 timeout = set->timeout;
5960         }
5961
5962         expiration = 0;
5963         if (nla[NFTA_SET_ELEM_EXPIRATION] != NULL) {
5964                 if (!(set->flags & NFT_SET_TIMEOUT))
5965                         return -EINVAL;
5966                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_EXPIRATION],
5967                                             &expiration);
5968                 if (err)
5969                         return err;
5970         }
5971
5972         if (nla[NFTA_SET_ELEM_EXPR]) {
5973                 struct nft_expr *expr;
5974
5975                 if (set->num_exprs && set->num_exprs != 1)
5976                         return -EOPNOTSUPP;
5977
5978                 expr = nft_set_elem_expr_alloc(ctx, set,
5979                                                nla[NFTA_SET_ELEM_EXPR]);
5980                 if (IS_ERR(expr))
5981                         return PTR_ERR(expr);
5982
5983                 expr_array[0] = expr;
5984                 num_exprs = 1;
5985
5986                 if (set->num_exprs && set->exprs[0]->ops != expr->ops) {
5987                         err = -EOPNOTSUPP;
5988                         goto err_set_elem_expr;
5989                 }
5990         } else if (nla[NFTA_SET_ELEM_EXPRESSIONS]) {
5991                 struct nft_expr *expr;
5992                 struct nlattr *tmp;
5993                 int left;
5994
5995                 i = 0;
5996                 nla_for_each_nested(tmp, nla[NFTA_SET_ELEM_EXPRESSIONS], left) {
5997                         if (i == NFT_SET_EXPR_MAX ||
5998                             (set->num_exprs && set->num_exprs == i)) {
5999                                 err = -E2BIG;
6000                                 goto err_set_elem_expr;
6001                         }
6002                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
6003                                 err = -EINVAL;
6004                                 goto err_set_elem_expr;
6005                         }
6006                         expr = nft_set_elem_expr_alloc(ctx, set, tmp);
6007                         if (IS_ERR(expr)) {
6008                                 err = PTR_ERR(expr);
6009                                 goto err_set_elem_expr;
6010                         }
6011                         expr_array[i] = expr;
6012                         num_exprs++;
6013
6014                         if (set->num_exprs && expr->ops != set->exprs[i]->ops) {
6015                                 err = -EOPNOTSUPP;
6016                                 goto err_set_elem_expr;
6017                         }
6018                         i++;
6019                 }
6020                 if (set->num_exprs && set->num_exprs != i) {
6021                         err = -EOPNOTSUPP;
6022                         goto err_set_elem_expr;
6023                 }
6024         } else if (set->num_exprs > 0) {
6025                 err = nft_set_elem_expr_clone(ctx, set, expr_array);
6026                 if (err < 0)
6027                         goto err_set_elem_expr_clone;
6028
6029                 num_exprs = set->num_exprs;
6030         }
6031
6032         if (nla[NFTA_SET_ELEM_KEY]) {
6033                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
6034                                             nla[NFTA_SET_ELEM_KEY]);
6035                 if (err < 0)
6036                         goto err_set_elem_expr;
6037
6038                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
6039                 if (err < 0)
6040                         goto err_parse_key;
6041         }
6042
6043         if (nla[NFTA_SET_ELEM_KEY_END]) {
6044                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
6045                                             nla[NFTA_SET_ELEM_KEY_END]);
6046                 if (err < 0)
6047                         goto err_parse_key;
6048
6049                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
6050                 if (err < 0)
6051                         goto err_parse_key_end;
6052         }
6053
6054         if (timeout > 0) {
6055                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_EXPIRATION);
6056                 if (err < 0)
6057                         goto err_parse_key_end;
6058
6059                 if (timeout != set->timeout) {
6060                         err = nft_set_ext_add(&tmpl, NFT_SET_EXT_TIMEOUT);
6061                         if (err < 0)
6062                                 goto err_parse_key_end;
6063                 }
6064         }
6065
6066         if (num_exprs) {
6067                 for (i = 0; i < num_exprs; i++)
6068                         size += expr_array[i]->ops->size;
6069
6070                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_EXPRESSIONS,
6071                                              sizeof(struct nft_set_elem_expr) + size);
6072                 if (err < 0)
6073                         goto err_parse_key_end;
6074         }
6075
6076         if (nla[NFTA_SET_ELEM_OBJREF] != NULL) {
6077                 obj = nft_obj_lookup(ctx->net, ctx->table,
6078                                      nla[NFTA_SET_ELEM_OBJREF],
6079                                      set->objtype, genmask);
6080                 if (IS_ERR(obj)) {
6081                         err = PTR_ERR(obj);
6082                         goto err_parse_key_end;
6083                 }
6084                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_OBJREF);
6085                 if (err < 0)
6086                         goto err_parse_key_end;
6087         }
6088
6089         if (nla[NFTA_SET_ELEM_DATA] != NULL) {
6090                 err = nft_setelem_parse_data(ctx, set, &desc, &elem.data.val,
6091                                              nla[NFTA_SET_ELEM_DATA]);
6092                 if (err < 0)
6093                         goto err_parse_key_end;
6094
6095                 dreg = nft_type_to_reg(set->dtype);
6096                 list_for_each_entry(binding, &set->bindings, list) {
6097                         struct nft_ctx bind_ctx = {
6098                                 .net    = ctx->net,
6099                                 .family = ctx->family,
6100                                 .table  = ctx->table,
6101                                 .chain  = (struct nft_chain *)binding->chain,
6102                         };
6103
6104                         if (!(binding->flags & NFT_SET_MAP))
6105                                 continue;
6106
6107                         err = nft_validate_register_store(&bind_ctx, dreg,
6108                                                           &elem.data.val,
6109                                                           desc.type, desc.len);
6110                         if (err < 0)
6111                                 goto err_parse_data;
6112
6113                         if (desc.type == NFT_DATA_VERDICT &&
6114                             (elem.data.val.verdict.code == NFT_GOTO ||
6115                              elem.data.val.verdict.code == NFT_JUMP))
6116                                 nft_validate_state_update(ctx->net,
6117                                                           NFT_VALIDATE_NEED);
6118                 }
6119
6120                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_DATA, desc.len);
6121                 if (err < 0)
6122                         goto err_parse_data;
6123         }
6124
6125         /* The full maximum length of userdata can exceed the maximum
6126          * offset value (U8_MAX) for following extensions, therefor it
6127          * must be the last extension added.
6128          */
6129         ulen = 0;
6130         if (nla[NFTA_SET_ELEM_USERDATA] != NULL) {
6131                 ulen = nla_len(nla[NFTA_SET_ELEM_USERDATA]);
6132                 if (ulen > 0) {
6133                         err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_USERDATA,
6134                                                      ulen);
6135                         if (err < 0)
6136                                 goto err_parse_data;
6137                 }
6138         }
6139
6140         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
6141                                       elem.key_end.val.data, elem.data.val.data,
6142                                       timeout, expiration, GFP_KERNEL_ACCOUNT);
6143         if (IS_ERR(elem.priv)) {
6144                 err = PTR_ERR(elem.priv);
6145                 goto err_parse_data;
6146         }
6147
6148         ext = nft_set_elem_ext(set, elem.priv);
6149         if (flags)
6150                 *nft_set_ext_flags(ext) = flags;
6151
6152         if (ulen > 0) {
6153                 if (nft_set_ext_check(&tmpl, NFT_SET_EXT_USERDATA, ulen) < 0) {
6154                         err = -EINVAL;
6155                         goto err_elem_userdata;
6156                 }
6157                 udata = nft_set_ext_userdata(ext);
6158                 udata->len = ulen - 1;
6159                 nla_memcpy(&udata->data, nla[NFTA_SET_ELEM_USERDATA], ulen);
6160         }
6161         if (obj) {
6162                 *nft_set_ext_obj(ext) = obj;
6163                 obj->use++;
6164         }
6165         err = nft_set_elem_expr_setup(ctx, &tmpl, ext, expr_array, num_exprs);
6166         if (err < 0)
6167                 goto err_elem_free;
6168
6169         trans = nft_trans_elem_alloc(ctx, NFT_MSG_NEWSETELEM, set);
6170         if (trans == NULL) {
6171                 err = -ENOMEM;
6172                 goto err_elem_free;
6173         }
6174
6175         ext->genmask = nft_genmask_cur(ctx->net) | NFT_SET_ELEM_BUSY_MASK;
6176
6177         err = nft_setelem_insert(ctx->net, set, &elem, &ext2, flags);
6178         if (err) {
6179                 if (err == -EEXIST) {
6180                         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) ^
6181                             nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) ||
6182                             nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) ^
6183                             nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF))
6184                                 goto err_element_clash;
6185                         if ((nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
6186                              nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) &&
6187                              memcmp(nft_set_ext_data(ext),
6188                                     nft_set_ext_data(ext2), set->dlen) != 0) ||
6189                             (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
6190                              nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF) &&
6191                              *nft_set_ext_obj(ext) != *nft_set_ext_obj(ext2)))
6192                                 goto err_element_clash;
6193                         else if (!(nlmsg_flags & NLM_F_EXCL))
6194                                 err = 0;
6195                 } else if (err == -ENOTEMPTY) {
6196                         /* ENOTEMPTY reports overlapping between this element
6197                          * and an existing one.
6198                          */
6199                         err = -EEXIST;
6200                 }
6201                 goto err_element_clash;
6202         }
6203
6204         if (!(flags & NFT_SET_ELEM_CATCHALL) && set->size &&
6205             !atomic_add_unless(&set->nelems, 1, set->size + set->ndeact)) {
6206                 err = -ENFILE;
6207                 goto err_set_full;
6208         }
6209
6210         nft_trans_elem(trans) = elem;
6211         nft_trans_commit_list_add_tail(ctx->net, trans);
6212         return 0;
6213
6214 err_set_full:
6215         nft_setelem_remove(ctx->net, set, &elem);
6216 err_element_clash:
6217         kfree(trans);
6218 err_elem_free:
6219         if (obj)
6220                 obj->use--;
6221 err_elem_userdata:
6222         nf_tables_set_elem_destroy(ctx, set, elem.priv);
6223 err_parse_data:
6224         if (nla[NFTA_SET_ELEM_DATA] != NULL)
6225                 nft_data_release(&elem.data.val, desc.type);
6226 err_parse_key_end:
6227         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
6228 err_parse_key:
6229         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
6230 err_set_elem_expr:
6231         for (i = 0; i < num_exprs && expr_array[i]; i++)
6232                 nft_expr_destroy(ctx, expr_array[i]);
6233 err_set_elem_expr_clone:
6234         return err;
6235 }
6236
6237 static int nf_tables_newsetelem(struct sk_buff *skb,
6238                                 const struct nfnl_info *info,
6239                                 const struct nlattr * const nla[])
6240 {
6241         struct nftables_pernet *nft_net = nft_pernet(info->net);
6242         struct netlink_ext_ack *extack = info->extack;
6243         u8 genmask = nft_genmask_next(info->net);
6244         u8 family = info->nfmsg->nfgen_family;
6245         struct net *net = info->net;
6246         const struct nlattr *attr;
6247         struct nft_table *table;
6248         struct nft_set *set;
6249         struct nft_ctx ctx;
6250         int rem, err;
6251
6252         if (nla[NFTA_SET_ELEM_LIST_ELEMENTS] == NULL)
6253                 return -EINVAL;
6254
6255         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
6256                                  genmask, NETLINK_CB(skb).portid);
6257         if (IS_ERR(table)) {
6258                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
6259                 return PTR_ERR(table);
6260         }
6261
6262         set = nft_set_lookup_global(net, table, nla[NFTA_SET_ELEM_LIST_SET],
6263                                     nla[NFTA_SET_ELEM_LIST_SET_ID], genmask);
6264         if (IS_ERR(set))
6265                 return PTR_ERR(set);
6266
6267         if (!list_empty(&set->bindings) && set->flags & NFT_SET_CONSTANT)
6268                 return -EBUSY;
6269
6270         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
6271
6272         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
6273                 err = nft_add_set_elem(&ctx, set, attr, info->nlh->nlmsg_flags);
6274                 if (err < 0) {
6275                         NL_SET_BAD_ATTR(extack, attr);
6276                         return err;
6277                 }
6278         }
6279
6280         if (nft_net->validate_state == NFT_VALIDATE_DO)
6281                 return nft_table_validate(net, table);
6282
6283         return 0;
6284 }
6285
6286 /**
6287  *      nft_data_hold - hold a nft_data item
6288  *
6289  *      @data: struct nft_data to release
6290  *      @type: type of data
6291  *
6292  *      Hold a nft_data item. NFT_DATA_VALUE types can be silently discarded,
6293  *      NFT_DATA_VERDICT bumps the reference to chains in case of NFT_JUMP and
6294  *      NFT_GOTO verdicts. This function must be called on active data objects
6295  *      from the second phase of the commit protocol.
6296  */
6297 void nft_data_hold(const struct nft_data *data, enum nft_data_types type)
6298 {
6299         struct nft_chain *chain;
6300         struct nft_rule *rule;
6301
6302         if (type == NFT_DATA_VERDICT) {
6303                 switch (data->verdict.code) {
6304                 case NFT_JUMP:
6305                 case NFT_GOTO:
6306                         chain = data->verdict.chain;
6307                         chain->use++;
6308
6309                         if (!nft_chain_is_bound(chain))
6310                                 break;
6311
6312                         chain->table->use++;
6313                         list_for_each_entry(rule, &chain->rules, list)
6314                                 chain->use++;
6315
6316                         nft_chain_add(chain->table, chain);
6317                         break;
6318                 }
6319         }
6320 }
6321
6322 static void nft_setelem_data_activate(const struct net *net,
6323                                       const struct nft_set *set,
6324                                       struct nft_set_elem *elem)
6325 {
6326         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6327
6328         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
6329                 nft_data_hold(nft_set_ext_data(ext), set->dtype);
6330         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
6331                 (*nft_set_ext_obj(ext))->use++;
6332 }
6333
6334 static void nft_setelem_data_deactivate(const struct net *net,
6335                                         const struct nft_set *set,
6336                                         struct nft_set_elem *elem)
6337 {
6338         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6339
6340         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
6341                 nft_data_release(nft_set_ext_data(ext), set->dtype);
6342         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
6343                 (*nft_set_ext_obj(ext))->use--;
6344 }
6345
6346 static int nft_del_setelem(struct nft_ctx *ctx, struct nft_set *set,
6347                            const struct nlattr *attr)
6348 {
6349         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
6350         struct nft_set_ext_tmpl tmpl;
6351         struct nft_set_elem elem;
6352         struct nft_set_ext *ext;
6353         struct nft_trans *trans;
6354         u32 flags = 0;
6355         int err;
6356
6357         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
6358                                           nft_set_elem_policy, NULL);
6359         if (err < 0)
6360                 return err;
6361
6362         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
6363         if (err < 0)
6364                 return err;
6365
6366         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
6367                 return -EINVAL;
6368
6369         if (!nft_setelem_valid_key_end(set, nla, flags))
6370                 return -EINVAL;
6371
6372         nft_set_ext_prepare(&tmpl);
6373
6374         if (flags != 0) {
6375                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
6376                 if (err < 0)
6377                         return err;
6378         }
6379
6380         if (nla[NFTA_SET_ELEM_KEY]) {
6381                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
6382                                             nla[NFTA_SET_ELEM_KEY]);
6383                 if (err < 0)
6384                         return err;
6385
6386                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
6387                 if (err < 0)
6388                         goto fail_elem;
6389         }
6390
6391         if (nla[NFTA_SET_ELEM_KEY_END]) {
6392                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
6393                                             nla[NFTA_SET_ELEM_KEY_END]);
6394                 if (err < 0)
6395                         goto fail_elem;
6396
6397                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
6398                 if (err < 0)
6399                         goto fail_elem_key_end;
6400         }
6401
6402         err = -ENOMEM;
6403         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
6404                                       elem.key_end.val.data, NULL, 0, 0,
6405                                       GFP_KERNEL_ACCOUNT);
6406         if (IS_ERR(elem.priv)) {
6407                 err = PTR_ERR(elem.priv);
6408                 goto fail_elem_key_end;
6409         }
6410
6411         ext = nft_set_elem_ext(set, elem.priv);
6412         if (flags)
6413                 *nft_set_ext_flags(ext) = flags;
6414
6415         trans = nft_trans_elem_alloc(ctx, NFT_MSG_DELSETELEM, set);
6416         if (trans == NULL)
6417                 goto fail_trans;
6418
6419         err = nft_setelem_deactivate(ctx->net, set, &elem, flags);
6420         if (err < 0)
6421                 goto fail_ops;
6422
6423         nft_setelem_data_deactivate(ctx->net, set, &elem);
6424
6425         nft_trans_elem(trans) = elem;
6426         nft_trans_commit_list_add_tail(ctx->net, trans);
6427         return 0;
6428
6429 fail_ops:
6430         kfree(trans);
6431 fail_trans:
6432         kfree(elem.priv);
6433 fail_elem_key_end:
6434         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
6435 fail_elem:
6436         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
6437         return err;
6438 }
6439
6440 static int nft_setelem_flush(const struct nft_ctx *ctx,
6441                              struct nft_set *set,
6442                              const struct nft_set_iter *iter,
6443                              struct nft_set_elem *elem)
6444 {
6445         struct nft_trans *trans;
6446         int err;
6447
6448         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
6449                                     sizeof(struct nft_trans_elem), GFP_ATOMIC);
6450         if (!trans)
6451                 return -ENOMEM;
6452
6453         if (!set->ops->flush(ctx->net, set, elem->priv)) {
6454                 err = -ENOENT;
6455                 goto err1;
6456         }
6457         set->ndeact++;
6458
6459         nft_setelem_data_deactivate(ctx->net, set, elem);
6460         nft_trans_elem_set(trans) = set;
6461         nft_trans_elem(trans) = *elem;
6462         nft_trans_commit_list_add_tail(ctx->net, trans);
6463
6464         return 0;
6465 err1:
6466         kfree(trans);
6467         return err;
6468 }
6469
6470 static int __nft_set_catchall_flush(const struct nft_ctx *ctx,
6471                                     struct nft_set *set,
6472                                     struct nft_set_elem *elem)
6473 {
6474         struct nft_trans *trans;
6475
6476         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
6477                                     sizeof(struct nft_trans_elem), GFP_KERNEL);
6478         if (!trans)
6479                 return -ENOMEM;
6480
6481         nft_setelem_data_deactivate(ctx->net, set, elem);
6482         nft_trans_elem_set(trans) = set;
6483         nft_trans_elem(trans) = *elem;
6484         nft_trans_commit_list_add_tail(ctx->net, trans);
6485
6486         return 0;
6487 }
6488
6489 static int nft_set_catchall_flush(const struct nft_ctx *ctx,
6490                                   struct nft_set *set)
6491 {
6492         u8 genmask = nft_genmask_next(ctx->net);
6493         struct nft_set_elem_catchall *catchall;
6494         struct nft_set_elem elem;
6495         struct nft_set_ext *ext;
6496         int ret = 0;
6497
6498         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
6499                 ext = nft_set_elem_ext(set, catchall->elem);
6500                 if (!nft_set_elem_active(ext, genmask) ||
6501                     nft_set_elem_mark_busy(ext))
6502                         continue;
6503
6504                 elem.priv = catchall->elem;
6505                 ret = __nft_set_catchall_flush(ctx, set, &elem);
6506                 if (ret < 0)
6507                         break;
6508         }
6509
6510         return ret;
6511 }
6512
6513 static int nft_set_flush(struct nft_ctx *ctx, struct nft_set *set, u8 genmask)
6514 {
6515         struct nft_set_iter iter = {
6516                 .genmask        = genmask,
6517                 .fn             = nft_setelem_flush,
6518         };
6519
6520         set->ops->walk(ctx, set, &iter);
6521         if (!iter.err)
6522                 iter.err = nft_set_catchall_flush(ctx, set);
6523
6524         return iter.err;
6525 }
6526
6527 static int nf_tables_delsetelem(struct sk_buff *skb,
6528                                 const struct nfnl_info *info,
6529                                 const struct nlattr * const nla[])
6530 {
6531         struct netlink_ext_ack *extack = info->extack;
6532         u8 genmask = nft_genmask_next(info->net);
6533         u8 family = info->nfmsg->nfgen_family;
6534         struct net *net = info->net;
6535         const struct nlattr *attr;
6536         struct nft_table *table;
6537         struct nft_set *set;
6538         struct nft_ctx ctx;
6539         int rem, err = 0;
6540
6541         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
6542                                  genmask, NETLINK_CB(skb).portid);
6543         if (IS_ERR(table)) {
6544                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
6545                 return PTR_ERR(table);
6546         }
6547
6548         set = nft_set_lookup(table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
6549         if (IS_ERR(set))
6550                 return PTR_ERR(set);
6551         if (!list_empty(&set->bindings) && set->flags & NFT_SET_CONSTANT)
6552                 return -EBUSY;
6553
6554         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
6555
6556         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
6557                 return nft_set_flush(&ctx, set, genmask);
6558
6559         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
6560                 err = nft_del_setelem(&ctx, set, attr);
6561                 if (err < 0) {
6562                         NL_SET_BAD_ATTR(extack, attr);
6563                         break;
6564                 }
6565         }
6566         return err;
6567 }
6568
6569 void nft_set_gc_batch_release(struct rcu_head *rcu)
6570 {
6571         struct nft_set_gc_batch *gcb;
6572         unsigned int i;
6573
6574         gcb = container_of(rcu, struct nft_set_gc_batch, head.rcu);
6575         for (i = 0; i < gcb->head.cnt; i++)
6576                 nft_set_elem_destroy(gcb->head.set, gcb->elems[i], true);
6577         kfree(gcb);
6578 }
6579
6580 struct nft_set_gc_batch *nft_set_gc_batch_alloc(const struct nft_set *set,
6581                                                 gfp_t gfp)
6582 {
6583         struct nft_set_gc_batch *gcb;
6584
6585         gcb = kzalloc(sizeof(*gcb), gfp);
6586         if (gcb == NULL)
6587                 return gcb;
6588         gcb->head.set = set;
6589         return gcb;
6590 }
6591
6592 /*
6593  * Stateful objects
6594  */
6595
6596 /**
6597  *      nft_register_obj- register nf_tables stateful object type
6598  *      @obj_type: object type
6599  *
6600  *      Registers the object type for use with nf_tables. Returns zero on
6601  *      success or a negative errno code otherwise.
6602  */
6603 int nft_register_obj(struct nft_object_type *obj_type)
6604 {
6605         if (obj_type->type == NFT_OBJECT_UNSPEC)
6606                 return -EINVAL;
6607
6608         nfnl_lock(NFNL_SUBSYS_NFTABLES);
6609         list_add_rcu(&obj_type->list, &nf_tables_objects);
6610         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
6611         return 0;
6612 }
6613 EXPORT_SYMBOL_GPL(nft_register_obj);
6614
6615 /**
6616  *      nft_unregister_obj - unregister nf_tables object type
6617  *      @obj_type: object type
6618  *
6619  *      Unregisters the object type for use with nf_tables.
6620  */
6621 void nft_unregister_obj(struct nft_object_type *obj_type)
6622 {
6623         nfnl_lock(NFNL_SUBSYS_NFTABLES);
6624         list_del_rcu(&obj_type->list);
6625         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
6626 }
6627 EXPORT_SYMBOL_GPL(nft_unregister_obj);
6628
6629 struct nft_object *nft_obj_lookup(const struct net *net,
6630                                   const struct nft_table *table,
6631                                   const struct nlattr *nla, u32 objtype,
6632                                   u8 genmask)
6633 {
6634         struct nft_object_hash_key k = { .table = table };
6635         char search[NFT_OBJ_MAXNAMELEN];
6636         struct rhlist_head *tmp, *list;
6637         struct nft_object *obj;
6638
6639         nla_strscpy(search, nla, sizeof(search));
6640         k.name = search;
6641
6642         WARN_ON_ONCE(!rcu_read_lock_held() &&
6643                      !lockdep_commit_lock_is_held(net));
6644
6645         rcu_read_lock();
6646         list = rhltable_lookup(&nft_objname_ht, &k, nft_objname_ht_params);
6647         if (!list)
6648                 goto out;
6649
6650         rhl_for_each_entry_rcu(obj, tmp, list, rhlhead) {
6651                 if (objtype == obj->ops->type->type &&
6652                     nft_active_genmask(obj, genmask)) {
6653                         rcu_read_unlock();
6654                         return obj;
6655                 }
6656         }
6657 out:
6658         rcu_read_unlock();
6659         return ERR_PTR(-ENOENT);
6660 }
6661 EXPORT_SYMBOL_GPL(nft_obj_lookup);
6662
6663 static struct nft_object *nft_obj_lookup_byhandle(const struct nft_table *table,
6664                                                   const struct nlattr *nla,
6665                                                   u32 objtype, u8 genmask)
6666 {
6667         struct nft_object *obj;
6668
6669         list_for_each_entry(obj, &table->objects, list) {
6670                 if (be64_to_cpu(nla_get_be64(nla)) == obj->handle &&
6671                     objtype == obj->ops->type->type &&
6672                     nft_active_genmask(obj, genmask))
6673                         return obj;
6674         }
6675         return ERR_PTR(-ENOENT);
6676 }
6677
6678 static const struct nla_policy nft_obj_policy[NFTA_OBJ_MAX + 1] = {
6679         [NFTA_OBJ_TABLE]        = { .type = NLA_STRING,
6680                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
6681         [NFTA_OBJ_NAME]         = { .type = NLA_STRING,
6682                                     .len = NFT_OBJ_MAXNAMELEN - 1 },
6683         [NFTA_OBJ_TYPE]         = { .type = NLA_U32 },
6684         [NFTA_OBJ_DATA]         = { .type = NLA_NESTED },
6685         [NFTA_OBJ_HANDLE]       = { .type = NLA_U64},
6686         [NFTA_OBJ_USERDATA]     = { .type = NLA_BINARY,
6687                                     .len = NFT_USERDATA_MAXLEN },
6688 };
6689
6690 static struct nft_object *nft_obj_init(const struct nft_ctx *ctx,
6691                                        const struct nft_object_type *type,
6692                                        const struct nlattr *attr)
6693 {
6694         struct nlattr **tb;
6695         const struct nft_object_ops *ops;
6696         struct nft_object *obj;
6697         int err = -ENOMEM;
6698
6699         tb = kmalloc_array(type->maxattr + 1, sizeof(*tb), GFP_KERNEL);
6700         if (!tb)
6701                 goto err1;
6702
6703         if (attr) {
6704                 err = nla_parse_nested_deprecated(tb, type->maxattr, attr,
6705                                                   type->policy, NULL);
6706                 if (err < 0)
6707                         goto err2;
6708         } else {
6709                 memset(tb, 0, sizeof(tb[0]) * (type->maxattr + 1));
6710         }
6711
6712         if (type->select_ops) {
6713                 ops = type->select_ops(ctx, (const struct nlattr * const *)tb);
6714                 if (IS_ERR(ops)) {
6715                         err = PTR_ERR(ops);
6716                         goto err2;
6717                 }
6718         } else {
6719                 ops = type->ops;
6720         }
6721
6722         err = -ENOMEM;
6723         obj = kzalloc(sizeof(*obj) + ops->size, GFP_KERNEL_ACCOUNT);
6724         if (!obj)
6725                 goto err2;
6726
6727         err = ops->init(ctx, (const struct nlattr * const *)tb, obj);
6728         if (err < 0)
6729                 goto err3;
6730
6731         obj->ops = ops;
6732
6733         kfree(tb);
6734         return obj;
6735 err3:
6736         kfree(obj);
6737 err2:
6738         kfree(tb);
6739 err1:
6740         return ERR_PTR(err);
6741 }
6742
6743 static int nft_object_dump(struct sk_buff *skb, unsigned int attr,
6744                            struct nft_object *obj, bool reset)
6745 {
6746         struct nlattr *nest;
6747
6748         nest = nla_nest_start_noflag(skb, attr);
6749         if (!nest)
6750                 goto nla_put_failure;
6751         if (obj->ops->dump(skb, obj, reset) < 0)
6752                 goto nla_put_failure;
6753         nla_nest_end(skb, nest);
6754         return 0;
6755
6756 nla_put_failure:
6757         return -1;
6758 }
6759
6760 static const struct nft_object_type *__nft_obj_type_get(u32 objtype)
6761 {
6762         const struct nft_object_type *type;
6763
6764         list_for_each_entry(type, &nf_tables_objects, list) {
6765                 if (objtype == type->type)
6766                         return type;
6767         }
6768         return NULL;
6769 }
6770
6771 static const struct nft_object_type *
6772 nft_obj_type_get(struct net *net, u32 objtype)
6773 {
6774         const struct nft_object_type *type;
6775
6776         type = __nft_obj_type_get(objtype);
6777         if (type != NULL && try_module_get(type->owner))
6778                 return type;
6779
6780         lockdep_nfnl_nft_mutex_not_held();
6781 #ifdef CONFIG_MODULES
6782         if (type == NULL) {
6783                 if (nft_request_module(net, "nft-obj-%u", objtype) == -EAGAIN)
6784                         return ERR_PTR(-EAGAIN);
6785         }
6786 #endif
6787         return ERR_PTR(-ENOENT);
6788 }
6789
6790 static int nf_tables_updobj(const struct nft_ctx *ctx,
6791                             const struct nft_object_type *type,
6792                             const struct nlattr *attr,
6793                             struct nft_object *obj)
6794 {
6795         struct nft_object *newobj;
6796         struct nft_trans *trans;
6797         int err = -ENOMEM;
6798
6799         if (!try_module_get(type->owner))
6800                 return -ENOENT;
6801
6802         trans = nft_trans_alloc(ctx, NFT_MSG_NEWOBJ,
6803                                 sizeof(struct nft_trans_obj));
6804         if (!trans)
6805                 goto err_trans;
6806
6807         newobj = nft_obj_init(ctx, type, attr);
6808         if (IS_ERR(newobj)) {
6809                 err = PTR_ERR(newobj);
6810                 goto err_free_trans;
6811         }
6812
6813         nft_trans_obj(trans) = obj;
6814         nft_trans_obj_update(trans) = true;
6815         nft_trans_obj_newobj(trans) = newobj;
6816         nft_trans_commit_list_add_tail(ctx->net, trans);
6817
6818         return 0;
6819
6820 err_free_trans:
6821         kfree(trans);
6822 err_trans:
6823         module_put(type->owner);
6824         return err;
6825 }
6826
6827 static int nf_tables_newobj(struct sk_buff *skb, const struct nfnl_info *info,
6828                             const struct nlattr * const nla[])
6829 {
6830         struct netlink_ext_ack *extack = info->extack;
6831         u8 genmask = nft_genmask_next(info->net);
6832         u8 family = info->nfmsg->nfgen_family;
6833         const struct nft_object_type *type;
6834         struct net *net = info->net;
6835         struct nft_table *table;
6836         struct nft_object *obj;
6837         struct nft_ctx ctx;
6838         u32 objtype;
6839         int err;
6840
6841         if (!nla[NFTA_OBJ_TYPE] ||
6842             !nla[NFTA_OBJ_NAME] ||
6843             !nla[NFTA_OBJ_DATA])
6844                 return -EINVAL;
6845
6846         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
6847                                  NETLINK_CB(skb).portid);
6848         if (IS_ERR(table)) {
6849                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
6850                 return PTR_ERR(table);
6851         }
6852
6853         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
6854         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
6855         if (IS_ERR(obj)) {
6856                 err = PTR_ERR(obj);
6857                 if (err != -ENOENT) {
6858                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
6859                         return err;
6860                 }
6861         } else {
6862                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
6863                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
6864                         return -EEXIST;
6865                 }
6866                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
6867                         return -EOPNOTSUPP;
6868
6869                 type = __nft_obj_type_get(objtype);
6870                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
6871
6872                 return nf_tables_updobj(&ctx, type, nla[NFTA_OBJ_DATA], obj);
6873         }
6874
6875         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
6876
6877         type = nft_obj_type_get(net, objtype);
6878         if (IS_ERR(type))
6879                 return PTR_ERR(type);
6880
6881         obj = nft_obj_init(&ctx, type, nla[NFTA_OBJ_DATA]);
6882         if (IS_ERR(obj)) {
6883                 err = PTR_ERR(obj);
6884                 goto err_init;
6885         }
6886         obj->key.table = table;
6887         obj->handle = nf_tables_alloc_handle(table);
6888
6889         obj->key.name = nla_strdup(nla[NFTA_OBJ_NAME], GFP_KERNEL_ACCOUNT);
6890         if (!obj->key.name) {
6891                 err = -ENOMEM;
6892                 goto err_strdup;
6893         }
6894
6895         if (nla[NFTA_OBJ_USERDATA]) {
6896                 obj->udata = nla_memdup(nla[NFTA_OBJ_USERDATA], GFP_KERNEL);
6897                 if (obj->udata == NULL)
6898                         goto err_userdata;
6899
6900                 obj->udlen = nla_len(nla[NFTA_OBJ_USERDATA]);
6901         }
6902
6903         err = nft_trans_obj_add(&ctx, NFT_MSG_NEWOBJ, obj);
6904         if (err < 0)
6905                 goto err_trans;
6906
6907         err = rhltable_insert(&nft_objname_ht, &obj->rhlhead,
6908                               nft_objname_ht_params);
6909         if (err < 0)
6910                 goto err_obj_ht;
6911
6912         list_add_tail_rcu(&obj->list, &table->objects);
6913         table->use++;
6914         return 0;
6915 err_obj_ht:
6916         /* queued in transaction log */
6917         INIT_LIST_HEAD(&obj->list);
6918         return err;
6919 err_trans:
6920         kfree(obj->udata);
6921 err_userdata:
6922         kfree(obj->key.name);
6923 err_strdup:
6924         if (obj->ops->destroy)
6925                 obj->ops->destroy(&ctx, obj);
6926         kfree(obj);
6927 err_init:
6928         module_put(type->owner);
6929         return err;
6930 }
6931
6932 static int nf_tables_fill_obj_info(struct sk_buff *skb, struct net *net,
6933                                    u32 portid, u32 seq, int event, u32 flags,
6934                                    int family, const struct nft_table *table,
6935                                    struct nft_object *obj, bool reset)
6936 {
6937         struct nlmsghdr *nlh;
6938
6939         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
6940         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
6941                            NFNETLINK_V0, nft_base_seq(net));
6942         if (!nlh)
6943                 goto nla_put_failure;
6944
6945         if (nla_put_string(skb, NFTA_OBJ_TABLE, table->name) ||
6946             nla_put_string(skb, NFTA_OBJ_NAME, obj->key.name) ||
6947             nla_put_be32(skb, NFTA_OBJ_TYPE, htonl(obj->ops->type->type)) ||
6948             nla_put_be32(skb, NFTA_OBJ_USE, htonl(obj->use)) ||
6949             nft_object_dump(skb, NFTA_OBJ_DATA, obj, reset) ||
6950             nla_put_be64(skb, NFTA_OBJ_HANDLE, cpu_to_be64(obj->handle),
6951                          NFTA_OBJ_PAD))
6952                 goto nla_put_failure;
6953
6954         if (obj->udata &&
6955             nla_put(skb, NFTA_OBJ_USERDATA, obj->udlen, obj->udata))
6956                 goto nla_put_failure;
6957
6958         nlmsg_end(skb, nlh);
6959         return 0;
6960
6961 nla_put_failure:
6962         nlmsg_trim(skb, nlh);
6963         return -1;
6964 }
6965
6966 struct nft_obj_filter {
6967         char            *table;
6968         u32             type;
6969 };
6970
6971 static int nf_tables_dump_obj(struct sk_buff *skb, struct netlink_callback *cb)
6972 {
6973         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
6974         const struct nft_table *table;
6975         unsigned int idx = 0, s_idx = cb->args[0];
6976         struct nft_obj_filter *filter = cb->data;
6977         struct net *net = sock_net(skb->sk);
6978         int family = nfmsg->nfgen_family;
6979         struct nftables_pernet *nft_net;
6980         struct nft_object *obj;
6981         bool reset = false;
6982
6983         if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
6984                 reset = true;
6985
6986         rcu_read_lock();
6987         nft_net = nft_pernet(net);
6988         cb->seq = READ_ONCE(nft_net->base_seq);
6989
6990         list_for_each_entry_rcu(table, &nft_net->tables, list) {
6991                 if (family != NFPROTO_UNSPEC && family != table->family)
6992                         continue;
6993
6994                 list_for_each_entry_rcu(obj, &table->objects, list) {
6995                         if (!nft_is_active(net, obj))
6996                                 goto cont;
6997                         if (idx < s_idx)
6998                                 goto cont;
6999                         if (idx > s_idx)
7000                                 memset(&cb->args[1], 0,
7001                                        sizeof(cb->args) - sizeof(cb->args[0]));
7002                         if (filter && filter->table &&
7003                             strcmp(filter->table, table->name))
7004                                 goto cont;
7005                         if (filter &&
7006                             filter->type != NFT_OBJECT_UNSPEC &&
7007                             obj->ops->type->type != filter->type)
7008                                 goto cont;
7009                         if (reset) {
7010                                 char *buf = kasprintf(GFP_ATOMIC,
7011                                                       "%s:%u",
7012                                                       table->name,
7013                                                       nft_net->base_seq);
7014
7015                                 audit_log_nfcfg(buf,
7016                                                 family,
7017                                                 obj->handle,
7018                                                 AUDIT_NFT_OP_OBJ_RESET,
7019                                                 GFP_ATOMIC);
7020                                 kfree(buf);
7021                         }
7022
7023                         if (nf_tables_fill_obj_info(skb, net, NETLINK_CB(cb->skb).portid,
7024                                                     cb->nlh->nlmsg_seq,
7025                                                     NFT_MSG_NEWOBJ,
7026                                                     NLM_F_MULTI | NLM_F_APPEND,
7027                                                     table->family, table,
7028                                                     obj, reset) < 0)
7029                                 goto done;
7030
7031                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
7032 cont:
7033                         idx++;
7034                 }
7035         }
7036 done:
7037         rcu_read_unlock();
7038
7039         cb->args[0] = idx;
7040         return skb->len;
7041 }
7042
7043 static int nf_tables_dump_obj_start(struct netlink_callback *cb)
7044 {
7045         const struct nlattr * const *nla = cb->data;
7046         struct nft_obj_filter *filter = NULL;
7047
7048         if (nla[NFTA_OBJ_TABLE] || nla[NFTA_OBJ_TYPE]) {
7049                 filter = kzalloc(sizeof(*filter), GFP_ATOMIC);
7050                 if (!filter)
7051                         return -ENOMEM;
7052
7053                 if (nla[NFTA_OBJ_TABLE]) {
7054                         filter->table = nla_strdup(nla[NFTA_OBJ_TABLE], GFP_ATOMIC);
7055                         if (!filter->table) {
7056                                 kfree(filter);
7057                                 return -ENOMEM;
7058                         }
7059                 }
7060
7061                 if (nla[NFTA_OBJ_TYPE])
7062                         filter->type = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7063         }
7064
7065         cb->data = filter;
7066         return 0;
7067 }
7068
7069 static int nf_tables_dump_obj_done(struct netlink_callback *cb)
7070 {
7071         struct nft_obj_filter *filter = cb->data;
7072
7073         if (filter) {
7074                 kfree(filter->table);
7075                 kfree(filter);
7076         }
7077
7078         return 0;
7079 }
7080
7081 /* called with rcu_read_lock held */
7082 static int nf_tables_getobj(struct sk_buff *skb, const struct nfnl_info *info,
7083                             const struct nlattr * const nla[])
7084 {
7085         struct netlink_ext_ack *extack = info->extack;
7086         u8 genmask = nft_genmask_cur(info->net);
7087         u8 family = info->nfmsg->nfgen_family;
7088         const struct nft_table *table;
7089         struct net *net = info->net;
7090         struct nft_object *obj;
7091         struct sk_buff *skb2;
7092         bool reset = false;
7093         u32 objtype;
7094         int err;
7095
7096         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
7097                 struct netlink_dump_control c = {
7098                         .start = nf_tables_dump_obj_start,
7099                         .dump = nf_tables_dump_obj,
7100                         .done = nf_tables_dump_obj_done,
7101                         .module = THIS_MODULE,
7102                         .data = (void *)nla,
7103                 };
7104
7105                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
7106         }
7107
7108         if (!nla[NFTA_OBJ_NAME] ||
7109             !nla[NFTA_OBJ_TYPE])
7110                 return -EINVAL;
7111
7112         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask, 0);
7113         if (IS_ERR(table)) {
7114                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7115                 return PTR_ERR(table);
7116         }
7117
7118         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7119         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
7120         if (IS_ERR(obj)) {
7121                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7122                 return PTR_ERR(obj);
7123         }
7124
7125         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
7126         if (!skb2)
7127                 return -ENOMEM;
7128
7129         if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
7130                 reset = true;
7131
7132         if (reset) {
7133                 const struct nftables_pernet *nft_net;
7134                 char *buf;
7135
7136                 nft_net = nft_pernet(net);
7137                 buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, nft_net->base_seq);
7138
7139                 audit_log_nfcfg(buf,
7140                                 family,
7141                                 obj->handle,
7142                                 AUDIT_NFT_OP_OBJ_RESET,
7143                                 GFP_ATOMIC);
7144                 kfree(buf);
7145         }
7146
7147         err = nf_tables_fill_obj_info(skb2, net, NETLINK_CB(skb).portid,
7148                                       info->nlh->nlmsg_seq, NFT_MSG_NEWOBJ, 0,
7149                                       family, table, obj, reset);
7150         if (err < 0)
7151                 goto err_fill_obj_info;
7152
7153         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
7154
7155 err_fill_obj_info:
7156         kfree_skb(skb2);
7157         return err;
7158 }
7159
7160 static void nft_obj_destroy(const struct nft_ctx *ctx, struct nft_object *obj)
7161 {
7162         if (obj->ops->destroy)
7163                 obj->ops->destroy(ctx, obj);
7164
7165         module_put(obj->ops->type->owner);
7166         kfree(obj->key.name);
7167         kfree(obj->udata);
7168         kfree(obj);
7169 }
7170
7171 static int nf_tables_delobj(struct sk_buff *skb, const struct nfnl_info *info,
7172                             const struct nlattr * const nla[])
7173 {
7174         struct netlink_ext_ack *extack = info->extack;
7175         u8 genmask = nft_genmask_next(info->net);
7176         u8 family = info->nfmsg->nfgen_family;
7177         struct net *net = info->net;
7178         const struct nlattr *attr;
7179         struct nft_table *table;
7180         struct nft_object *obj;
7181         struct nft_ctx ctx;
7182         u32 objtype;
7183
7184         if (!nla[NFTA_OBJ_TYPE] ||
7185             (!nla[NFTA_OBJ_NAME] && !nla[NFTA_OBJ_HANDLE]))
7186                 return -EINVAL;
7187
7188         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
7189                                  NETLINK_CB(skb).portid);
7190         if (IS_ERR(table)) {
7191                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7192                 return PTR_ERR(table);
7193         }
7194
7195         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7196         if (nla[NFTA_OBJ_HANDLE]) {
7197                 attr = nla[NFTA_OBJ_HANDLE];
7198                 obj = nft_obj_lookup_byhandle(table, attr, objtype, genmask);
7199         } else {
7200                 attr = nla[NFTA_OBJ_NAME];
7201                 obj = nft_obj_lookup(net, table, attr, objtype, genmask);
7202         }
7203
7204         if (IS_ERR(obj)) {
7205                 NL_SET_BAD_ATTR(extack, attr);
7206                 return PTR_ERR(obj);
7207         }
7208         if (obj->use > 0) {
7209                 NL_SET_BAD_ATTR(extack, attr);
7210                 return -EBUSY;
7211         }
7212
7213         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7214
7215         return nft_delobj(&ctx, obj);
7216 }
7217
7218 void nft_obj_notify(struct net *net, const struct nft_table *table,
7219                     struct nft_object *obj, u32 portid, u32 seq, int event,
7220                     u16 flags, int family, int report, gfp_t gfp)
7221 {
7222         struct nftables_pernet *nft_net = nft_pernet(net);
7223         struct sk_buff *skb;
7224         int err;
7225         char *buf = kasprintf(gfp, "%s:%u",
7226                               table->name, nft_net->base_seq);
7227
7228         audit_log_nfcfg(buf,
7229                         family,
7230                         obj->handle,
7231                         event == NFT_MSG_NEWOBJ ?
7232                                  AUDIT_NFT_OP_OBJ_REGISTER :
7233                                  AUDIT_NFT_OP_OBJ_UNREGISTER,
7234                         gfp);
7235         kfree(buf);
7236
7237         if (!report &&
7238             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
7239                 return;
7240
7241         skb = nlmsg_new(NLMSG_GOODSIZE, gfp);
7242         if (skb == NULL)
7243                 goto err;
7244
7245         err = nf_tables_fill_obj_info(skb, net, portid, seq, event,
7246                                       flags & (NLM_F_CREATE | NLM_F_EXCL),
7247                                       family, table, obj, false);
7248         if (err < 0) {
7249                 kfree_skb(skb);
7250                 goto err;
7251         }
7252
7253         nft_notify_enqueue(skb, report, &nft_net->notify_list);
7254         return;
7255 err:
7256         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
7257 }
7258 EXPORT_SYMBOL_GPL(nft_obj_notify);
7259
7260 static void nf_tables_obj_notify(const struct nft_ctx *ctx,
7261                                  struct nft_object *obj, int event)
7262 {
7263         nft_obj_notify(ctx->net, ctx->table, obj, ctx->portid, ctx->seq, event,
7264                        ctx->flags, ctx->family, ctx->report, GFP_KERNEL);
7265 }
7266
7267 /*
7268  * Flow tables
7269  */
7270 void nft_register_flowtable_type(struct nf_flowtable_type *type)
7271 {
7272         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7273         list_add_tail_rcu(&type->list, &nf_tables_flowtables);
7274         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7275 }
7276 EXPORT_SYMBOL_GPL(nft_register_flowtable_type);
7277
7278 void nft_unregister_flowtable_type(struct nf_flowtable_type *type)
7279 {
7280         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7281         list_del_rcu(&type->list);
7282         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7283 }
7284 EXPORT_SYMBOL_GPL(nft_unregister_flowtable_type);
7285
7286 static const struct nla_policy nft_flowtable_policy[NFTA_FLOWTABLE_MAX + 1] = {
7287         [NFTA_FLOWTABLE_TABLE]          = { .type = NLA_STRING,
7288                                             .len = NFT_NAME_MAXLEN - 1 },
7289         [NFTA_FLOWTABLE_NAME]           = { .type = NLA_STRING,
7290                                             .len = NFT_NAME_MAXLEN - 1 },
7291         [NFTA_FLOWTABLE_HOOK]           = { .type = NLA_NESTED },
7292         [NFTA_FLOWTABLE_HANDLE]         = { .type = NLA_U64 },
7293         [NFTA_FLOWTABLE_FLAGS]          = { .type = NLA_U32 },
7294 };
7295
7296 struct nft_flowtable *nft_flowtable_lookup(const struct nft_table *table,
7297                                            const struct nlattr *nla, u8 genmask)
7298 {
7299         struct nft_flowtable *flowtable;
7300
7301         list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
7302                 if (!nla_strcmp(nla, flowtable->name) &&
7303                     nft_active_genmask(flowtable, genmask))
7304                         return flowtable;
7305         }
7306         return ERR_PTR(-ENOENT);
7307 }
7308 EXPORT_SYMBOL_GPL(nft_flowtable_lookup);
7309
7310 void nf_tables_deactivate_flowtable(const struct nft_ctx *ctx,
7311                                     struct nft_flowtable *flowtable,
7312                                     enum nft_trans_phase phase)
7313 {
7314         switch (phase) {
7315         case NFT_TRANS_PREPARE:
7316         case NFT_TRANS_ABORT:
7317         case NFT_TRANS_RELEASE:
7318                 flowtable->use--;
7319                 fallthrough;
7320         default:
7321                 return;
7322         }
7323 }
7324 EXPORT_SYMBOL_GPL(nf_tables_deactivate_flowtable);
7325
7326 static struct nft_flowtable *
7327 nft_flowtable_lookup_byhandle(const struct nft_table *table,
7328                               const struct nlattr *nla, u8 genmask)
7329 {
7330        struct nft_flowtable *flowtable;
7331
7332        list_for_each_entry(flowtable, &table->flowtables, list) {
7333                if (be64_to_cpu(nla_get_be64(nla)) == flowtable->handle &&
7334                    nft_active_genmask(flowtable, genmask))
7335                        return flowtable;
7336        }
7337        return ERR_PTR(-ENOENT);
7338 }
7339
7340 struct nft_flowtable_hook {
7341         u32                     num;
7342         int                     priority;
7343         struct list_head        list;
7344 };
7345
7346 static const struct nla_policy nft_flowtable_hook_policy[NFTA_FLOWTABLE_HOOK_MAX + 1] = {
7347         [NFTA_FLOWTABLE_HOOK_NUM]       = { .type = NLA_U32 },
7348         [NFTA_FLOWTABLE_HOOK_PRIORITY]  = { .type = NLA_U32 },
7349         [NFTA_FLOWTABLE_HOOK_DEVS]      = { .type = NLA_NESTED },
7350 };
7351
7352 static int nft_flowtable_parse_hook(const struct nft_ctx *ctx,
7353                                     const struct nlattr *attr,
7354                                     struct nft_flowtable_hook *flowtable_hook,
7355                                     struct nft_flowtable *flowtable, bool add)
7356 {
7357         struct nlattr *tb[NFTA_FLOWTABLE_HOOK_MAX + 1];
7358         struct nft_hook *hook;
7359         int hooknum, priority;
7360         int err;
7361
7362         INIT_LIST_HEAD(&flowtable_hook->list);
7363
7364         err = nla_parse_nested_deprecated(tb, NFTA_FLOWTABLE_HOOK_MAX, attr,
7365                                           nft_flowtable_hook_policy, NULL);
7366         if (err < 0)
7367                 return err;
7368
7369         if (add) {
7370                 if (!tb[NFTA_FLOWTABLE_HOOK_NUM] ||
7371                     !tb[NFTA_FLOWTABLE_HOOK_PRIORITY])
7372                         return -EINVAL;
7373
7374                 hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
7375                 if (hooknum != NF_NETDEV_INGRESS)
7376                         return -EOPNOTSUPP;
7377
7378                 priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
7379
7380                 flowtable_hook->priority        = priority;
7381                 flowtable_hook->num             = hooknum;
7382         } else {
7383                 if (tb[NFTA_FLOWTABLE_HOOK_NUM]) {
7384                         hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
7385                         if (hooknum != flowtable->hooknum)
7386                                 return -EOPNOTSUPP;
7387                 }
7388
7389                 if (tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
7390                         priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
7391                         if (priority != flowtable->data.priority)
7392                                 return -EOPNOTSUPP;
7393                 }
7394
7395                 flowtable_hook->priority        = flowtable->data.priority;
7396                 flowtable_hook->num             = flowtable->hooknum;
7397         }
7398
7399         if (tb[NFTA_FLOWTABLE_HOOK_DEVS]) {
7400                 err = nf_tables_parse_netdev_hooks(ctx->net,
7401                                                    tb[NFTA_FLOWTABLE_HOOK_DEVS],
7402                                                    &flowtable_hook->list);
7403                 if (err < 0)
7404                         return err;
7405         }
7406
7407         list_for_each_entry(hook, &flowtable_hook->list, list) {
7408                 hook->ops.pf            = NFPROTO_NETDEV;
7409                 hook->ops.hooknum       = flowtable_hook->num;
7410                 hook->ops.priority      = flowtable_hook->priority;
7411                 hook->ops.priv          = &flowtable->data;
7412                 hook->ops.hook          = flowtable->data.type->hook;
7413         }
7414
7415         return err;
7416 }
7417
7418 static const struct nf_flowtable_type *__nft_flowtable_type_get(u8 family)
7419 {
7420         const struct nf_flowtable_type *type;
7421
7422         list_for_each_entry(type, &nf_tables_flowtables, list) {
7423                 if (family == type->family)
7424                         return type;
7425         }
7426         return NULL;
7427 }
7428
7429 static const struct nf_flowtable_type *
7430 nft_flowtable_type_get(struct net *net, u8 family)
7431 {
7432         const struct nf_flowtable_type *type;
7433
7434         type = __nft_flowtable_type_get(family);
7435         if (type != NULL && try_module_get(type->owner))
7436                 return type;
7437
7438         lockdep_nfnl_nft_mutex_not_held();
7439 #ifdef CONFIG_MODULES
7440         if (type == NULL) {
7441                 if (nft_request_module(net, "nf-flowtable-%u", family) == -EAGAIN)
7442                         return ERR_PTR(-EAGAIN);
7443         }
7444 #endif
7445         return ERR_PTR(-ENOENT);
7446 }
7447
7448 /* Only called from error and netdev event paths. */
7449 static void nft_unregister_flowtable_hook(struct net *net,
7450                                           struct nft_flowtable *flowtable,
7451                                           struct nft_hook *hook)
7452 {
7453         nf_unregister_net_hook(net, &hook->ops);
7454         flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
7455                                     FLOW_BLOCK_UNBIND);
7456 }
7457
7458 static void __nft_unregister_flowtable_net_hooks(struct net *net,
7459                                                  struct list_head *hook_list,
7460                                                  bool release_netdev)
7461 {
7462         struct nft_hook *hook, *next;
7463
7464         list_for_each_entry_safe(hook, next, hook_list, list) {
7465                 nf_unregister_net_hook(net, &hook->ops);
7466                 if (release_netdev) {
7467                         list_del(&hook->list);
7468                         kfree_rcu(hook, rcu);
7469                 }
7470         }
7471 }
7472
7473 static void nft_unregister_flowtable_net_hooks(struct net *net,
7474                                                struct list_head *hook_list)
7475 {
7476         __nft_unregister_flowtable_net_hooks(net, hook_list, false);
7477 }
7478
7479 static int nft_register_flowtable_net_hooks(struct net *net,
7480                                             struct nft_table *table,
7481                                             struct list_head *hook_list,
7482                                             struct nft_flowtable *flowtable)
7483 {
7484         struct nft_hook *hook, *hook2, *next;
7485         struct nft_flowtable *ft;
7486         int err, i = 0;
7487
7488         list_for_each_entry(hook, hook_list, list) {
7489                 list_for_each_entry(ft, &table->flowtables, list) {
7490                         if (!nft_is_active_next(net, ft))
7491                                 continue;
7492
7493                         list_for_each_entry(hook2, &ft->hook_list, list) {
7494                                 if (hook->ops.dev == hook2->ops.dev &&
7495                                     hook->ops.pf == hook2->ops.pf) {
7496                                         err = -EEXIST;
7497                                         goto err_unregister_net_hooks;
7498                                 }
7499                         }
7500                 }
7501
7502                 err = flowtable->data.type->setup(&flowtable->data,
7503                                                   hook->ops.dev,
7504                                                   FLOW_BLOCK_BIND);
7505                 if (err < 0)
7506                         goto err_unregister_net_hooks;
7507
7508                 err = nf_register_net_hook(net, &hook->ops);
7509                 if (err < 0) {
7510                         flowtable->data.type->setup(&flowtable->data,
7511                                                     hook->ops.dev,
7512                                                     FLOW_BLOCK_UNBIND);
7513                         goto err_unregister_net_hooks;
7514                 }
7515
7516                 i++;
7517         }
7518
7519         return 0;
7520
7521 err_unregister_net_hooks:
7522         list_for_each_entry_safe(hook, next, hook_list, list) {
7523                 if (i-- <= 0)
7524                         break;
7525
7526                 nft_unregister_flowtable_hook(net, flowtable, hook);
7527                 list_del_rcu(&hook->list);
7528                 kfree_rcu(hook, rcu);
7529         }
7530
7531         return err;
7532 }
7533
7534 static void nft_flowtable_hooks_destroy(struct list_head *hook_list)
7535 {
7536         struct nft_hook *hook, *next;
7537
7538         list_for_each_entry_safe(hook, next, hook_list, list) {
7539                 list_del_rcu(&hook->list);
7540                 kfree_rcu(hook, rcu);
7541         }
7542 }
7543
7544 static int nft_flowtable_update(struct nft_ctx *ctx, const struct nlmsghdr *nlh,
7545                                 struct nft_flowtable *flowtable)
7546 {
7547         const struct nlattr * const *nla = ctx->nla;
7548         struct nft_flowtable_hook flowtable_hook;
7549         struct nft_hook *hook, *next;
7550         struct nft_trans *trans;
7551         bool unregister = false;
7552         u32 flags;
7553         int err;
7554
7555         err = nft_flowtable_parse_hook(ctx, nla[NFTA_FLOWTABLE_HOOK],
7556                                        &flowtable_hook, flowtable, false);
7557         if (err < 0)
7558                 return err;
7559
7560         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
7561                 if (nft_hook_list_find(&flowtable->hook_list, hook)) {
7562                         list_del(&hook->list);
7563                         kfree(hook);
7564                 }
7565         }
7566
7567         if (nla[NFTA_FLOWTABLE_FLAGS]) {
7568                 flags = ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
7569                 if (flags & ~NFT_FLOWTABLE_MASK) {
7570                         err = -EOPNOTSUPP;
7571                         goto err_flowtable_update_hook;
7572                 }
7573                 if ((flowtable->data.flags & NFT_FLOWTABLE_HW_OFFLOAD) ^
7574                     (flags & NFT_FLOWTABLE_HW_OFFLOAD)) {
7575                         err = -EOPNOTSUPP;
7576                         goto err_flowtable_update_hook;
7577                 }
7578         } else {
7579                 flags = flowtable->data.flags;
7580         }
7581
7582         err = nft_register_flowtable_net_hooks(ctx->net, ctx->table,
7583                                                &flowtable_hook.list, flowtable);
7584         if (err < 0)
7585                 goto err_flowtable_update_hook;
7586
7587         trans = nft_trans_alloc(ctx, NFT_MSG_NEWFLOWTABLE,
7588                                 sizeof(struct nft_trans_flowtable));
7589         if (!trans) {
7590                 unregister = true;
7591                 err = -ENOMEM;
7592                 goto err_flowtable_update_hook;
7593         }
7594
7595         nft_trans_flowtable_flags(trans) = flags;
7596         nft_trans_flowtable(trans) = flowtable;
7597         nft_trans_flowtable_update(trans) = true;
7598         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
7599         list_splice(&flowtable_hook.list, &nft_trans_flowtable_hooks(trans));
7600
7601         nft_trans_commit_list_add_tail(ctx->net, trans);
7602
7603         return 0;
7604
7605 err_flowtable_update_hook:
7606         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
7607                 if (unregister)
7608                         nft_unregister_flowtable_hook(ctx->net, flowtable, hook);
7609                 list_del_rcu(&hook->list);
7610                 kfree_rcu(hook, rcu);
7611         }
7612
7613         return err;
7614
7615 }
7616
7617 static int nf_tables_newflowtable(struct sk_buff *skb,
7618                                   const struct nfnl_info *info,
7619                                   const struct nlattr * const nla[])
7620 {
7621         struct netlink_ext_ack *extack = info->extack;
7622         struct nft_flowtable_hook flowtable_hook;
7623         u8 genmask = nft_genmask_next(info->net);
7624         u8 family = info->nfmsg->nfgen_family;
7625         const struct nf_flowtable_type *type;
7626         struct nft_flowtable *flowtable;
7627         struct nft_hook *hook, *next;
7628         struct net *net = info->net;
7629         struct nft_table *table;
7630         struct nft_ctx ctx;
7631         int err;
7632
7633         if (!nla[NFTA_FLOWTABLE_TABLE] ||
7634             !nla[NFTA_FLOWTABLE_NAME] ||
7635             !nla[NFTA_FLOWTABLE_HOOK])
7636                 return -EINVAL;
7637
7638         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
7639                                  genmask, NETLINK_CB(skb).portid);
7640         if (IS_ERR(table)) {
7641                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
7642                 return PTR_ERR(table);
7643         }
7644
7645         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
7646                                          genmask);
7647         if (IS_ERR(flowtable)) {
7648                 err = PTR_ERR(flowtable);
7649                 if (err != -ENOENT) {
7650                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
7651                         return err;
7652                 }
7653         } else {
7654                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
7655                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
7656                         return -EEXIST;
7657                 }
7658
7659                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7660
7661                 return nft_flowtable_update(&ctx, info->nlh, flowtable);
7662         }
7663
7664         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7665
7666         flowtable = kzalloc(sizeof(*flowtable), GFP_KERNEL_ACCOUNT);
7667         if (!flowtable)
7668                 return -ENOMEM;
7669
7670         flowtable->table = table;
7671         flowtable->handle = nf_tables_alloc_handle(table);
7672         INIT_LIST_HEAD(&flowtable->hook_list);
7673
7674         flowtable->name = nla_strdup(nla[NFTA_FLOWTABLE_NAME], GFP_KERNEL_ACCOUNT);
7675         if (!flowtable->name) {
7676                 err = -ENOMEM;
7677                 goto err1;
7678         }
7679
7680         type = nft_flowtable_type_get(net, family);
7681         if (IS_ERR(type)) {
7682                 err = PTR_ERR(type);
7683                 goto err2;
7684         }
7685
7686         if (nla[NFTA_FLOWTABLE_FLAGS]) {
7687                 flowtable->data.flags =
7688                         ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
7689                 if (flowtable->data.flags & ~NFT_FLOWTABLE_MASK) {
7690                         err = -EOPNOTSUPP;
7691                         goto err3;
7692                 }
7693         }
7694
7695         write_pnet(&flowtable->data.net, net);
7696         flowtable->data.type = type;
7697         err = type->init(&flowtable->data);
7698         if (err < 0)
7699                 goto err3;
7700
7701         err = nft_flowtable_parse_hook(&ctx, nla[NFTA_FLOWTABLE_HOOK],
7702                                        &flowtable_hook, flowtable, true);
7703         if (err < 0)
7704                 goto err4;
7705
7706         list_splice(&flowtable_hook.list, &flowtable->hook_list);
7707         flowtable->data.priority = flowtable_hook.priority;
7708         flowtable->hooknum = flowtable_hook.num;
7709
7710         err = nft_register_flowtable_net_hooks(ctx.net, table,
7711                                                &flowtable->hook_list,
7712                                                flowtable);
7713         if (err < 0) {
7714                 nft_flowtable_hooks_destroy(&flowtable->hook_list);
7715                 goto err4;
7716         }
7717
7718         err = nft_trans_flowtable_add(&ctx, NFT_MSG_NEWFLOWTABLE, flowtable);
7719         if (err < 0)
7720                 goto err5;
7721
7722         list_add_tail_rcu(&flowtable->list, &table->flowtables);
7723         table->use++;
7724
7725         return 0;
7726 err5:
7727         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
7728                 nft_unregister_flowtable_hook(net, flowtable, hook);
7729                 list_del_rcu(&hook->list);
7730                 kfree_rcu(hook, rcu);
7731         }
7732 err4:
7733         flowtable->data.type->free(&flowtable->data);
7734 err3:
7735         module_put(type->owner);
7736 err2:
7737         kfree(flowtable->name);
7738 err1:
7739         kfree(flowtable);
7740         return err;
7741 }
7742
7743 static void nft_flowtable_hook_release(struct nft_flowtable_hook *flowtable_hook)
7744 {
7745         struct nft_hook *this, *next;
7746
7747         list_for_each_entry_safe(this, next, &flowtable_hook->list, list) {
7748                 list_del(&this->list);
7749                 kfree(this);
7750         }
7751 }
7752
7753 static int nft_delflowtable_hook(struct nft_ctx *ctx,
7754                                  struct nft_flowtable *flowtable)
7755 {
7756         const struct nlattr * const *nla = ctx->nla;
7757         struct nft_flowtable_hook flowtable_hook;
7758         LIST_HEAD(flowtable_del_list);
7759         struct nft_hook *this, *hook;
7760         struct nft_trans *trans;
7761         int err;
7762
7763         err = nft_flowtable_parse_hook(ctx, nla[NFTA_FLOWTABLE_HOOK],
7764                                        &flowtable_hook, flowtable, false);
7765         if (err < 0)
7766                 return err;
7767
7768         list_for_each_entry(this, &flowtable_hook.list, list) {
7769                 hook = nft_hook_list_find(&flowtable->hook_list, this);
7770                 if (!hook) {
7771                         err = -ENOENT;
7772                         goto err_flowtable_del_hook;
7773                 }
7774                 list_move(&hook->list, &flowtable_del_list);
7775         }
7776
7777         trans = nft_trans_alloc(ctx, NFT_MSG_DELFLOWTABLE,
7778                                 sizeof(struct nft_trans_flowtable));
7779         if (!trans) {
7780                 err = -ENOMEM;
7781                 goto err_flowtable_del_hook;
7782         }
7783
7784         nft_trans_flowtable(trans) = flowtable;
7785         nft_trans_flowtable_update(trans) = true;
7786         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
7787         list_splice(&flowtable_del_list, &nft_trans_flowtable_hooks(trans));
7788         nft_flowtable_hook_release(&flowtable_hook);
7789
7790         nft_trans_commit_list_add_tail(ctx->net, trans);
7791
7792         return 0;
7793
7794 err_flowtable_del_hook:
7795         list_splice(&flowtable_del_list, &flowtable->hook_list);
7796         nft_flowtable_hook_release(&flowtable_hook);
7797
7798         return err;
7799 }
7800
7801 static int nf_tables_delflowtable(struct sk_buff *skb,
7802                                   const struct nfnl_info *info,
7803                                   const struct nlattr * const nla[])
7804 {
7805         struct netlink_ext_ack *extack = info->extack;
7806         u8 genmask = nft_genmask_next(info->net);
7807         u8 family = info->nfmsg->nfgen_family;
7808         struct nft_flowtable *flowtable;
7809         struct net *net = info->net;
7810         const struct nlattr *attr;
7811         struct nft_table *table;
7812         struct nft_ctx ctx;
7813
7814         if (!nla[NFTA_FLOWTABLE_TABLE] ||
7815             (!nla[NFTA_FLOWTABLE_NAME] &&
7816              !nla[NFTA_FLOWTABLE_HANDLE]))
7817                 return -EINVAL;
7818
7819         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
7820                                  genmask, NETLINK_CB(skb).portid);
7821         if (IS_ERR(table)) {
7822                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
7823                 return PTR_ERR(table);
7824         }
7825
7826         if (nla[NFTA_FLOWTABLE_HANDLE]) {
7827                 attr = nla[NFTA_FLOWTABLE_HANDLE];
7828                 flowtable = nft_flowtable_lookup_byhandle(table, attr, genmask);
7829         } else {
7830                 attr = nla[NFTA_FLOWTABLE_NAME];
7831                 flowtable = nft_flowtable_lookup(table, attr, genmask);
7832         }
7833
7834         if (IS_ERR(flowtable)) {
7835                 NL_SET_BAD_ATTR(extack, attr);
7836                 return PTR_ERR(flowtable);
7837         }
7838
7839         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7840
7841         if (nla[NFTA_FLOWTABLE_HOOK])
7842                 return nft_delflowtable_hook(&ctx, flowtable);
7843
7844         if (flowtable->use > 0) {
7845                 NL_SET_BAD_ATTR(extack, attr);
7846                 return -EBUSY;
7847         }
7848
7849         return nft_delflowtable(&ctx, flowtable);
7850 }
7851
7852 static int nf_tables_fill_flowtable_info(struct sk_buff *skb, struct net *net,
7853                                          u32 portid, u32 seq, int event,
7854                                          u32 flags, int family,
7855                                          struct nft_flowtable *flowtable,
7856                                          struct list_head *hook_list)
7857 {
7858         struct nlattr *nest, *nest_devs;
7859         struct nft_hook *hook;
7860         struct nlmsghdr *nlh;
7861
7862         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
7863         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
7864                            NFNETLINK_V0, nft_base_seq(net));
7865         if (!nlh)
7866                 goto nla_put_failure;
7867
7868         if (nla_put_string(skb, NFTA_FLOWTABLE_TABLE, flowtable->table->name) ||
7869             nla_put_string(skb, NFTA_FLOWTABLE_NAME, flowtable->name) ||
7870             nla_put_be32(skb, NFTA_FLOWTABLE_USE, htonl(flowtable->use)) ||
7871             nla_put_be64(skb, NFTA_FLOWTABLE_HANDLE, cpu_to_be64(flowtable->handle),
7872                          NFTA_FLOWTABLE_PAD) ||
7873             nla_put_be32(skb, NFTA_FLOWTABLE_FLAGS, htonl(flowtable->data.flags)))
7874                 goto nla_put_failure;
7875
7876         nest = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK);
7877         if (!nest)
7878                 goto nla_put_failure;
7879         if (nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_NUM, htonl(flowtable->hooknum)) ||
7880             nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_PRIORITY, htonl(flowtable->data.priority)))
7881                 goto nla_put_failure;
7882
7883         nest_devs = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK_DEVS);
7884         if (!nest_devs)
7885                 goto nla_put_failure;
7886
7887         list_for_each_entry_rcu(hook, hook_list, list) {
7888                 if (nla_put_string(skb, NFTA_DEVICE_NAME, hook->ops.dev->name))
7889                         goto nla_put_failure;
7890         }
7891         nla_nest_end(skb, nest_devs);
7892         nla_nest_end(skb, nest);
7893
7894         nlmsg_end(skb, nlh);
7895         return 0;
7896
7897 nla_put_failure:
7898         nlmsg_trim(skb, nlh);
7899         return -1;
7900 }
7901
7902 struct nft_flowtable_filter {
7903         char            *table;
7904 };
7905
7906 static int nf_tables_dump_flowtable(struct sk_buff *skb,
7907                                     struct netlink_callback *cb)
7908 {
7909         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
7910         struct nft_flowtable_filter *filter = cb->data;
7911         unsigned int idx = 0, s_idx = cb->args[0];
7912         struct net *net = sock_net(skb->sk);
7913         int family = nfmsg->nfgen_family;
7914         struct nft_flowtable *flowtable;
7915         struct nftables_pernet *nft_net;
7916         const struct nft_table *table;
7917
7918         rcu_read_lock();
7919         nft_net = nft_pernet(net);
7920         cb->seq = READ_ONCE(nft_net->base_seq);
7921
7922         list_for_each_entry_rcu(table, &nft_net->tables, list) {
7923                 if (family != NFPROTO_UNSPEC && family != table->family)
7924                         continue;
7925
7926                 list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
7927                         if (!nft_is_active(net, flowtable))
7928                                 goto cont;
7929                         if (idx < s_idx)
7930                                 goto cont;
7931                         if (idx > s_idx)
7932                                 memset(&cb->args[1], 0,
7933                                        sizeof(cb->args) - sizeof(cb->args[0]));
7934                         if (filter && filter->table &&
7935                             strcmp(filter->table, table->name))
7936                                 goto cont;
7937
7938                         if (nf_tables_fill_flowtable_info(skb, net, NETLINK_CB(cb->skb).portid,
7939                                                           cb->nlh->nlmsg_seq,
7940                                                           NFT_MSG_NEWFLOWTABLE,
7941                                                           NLM_F_MULTI | NLM_F_APPEND,
7942                                                           table->family,
7943                                                           flowtable,
7944                                                           &flowtable->hook_list) < 0)
7945                                 goto done;
7946
7947                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
7948 cont:
7949                         idx++;
7950                 }
7951         }
7952 done:
7953         rcu_read_unlock();
7954
7955         cb->args[0] = idx;
7956         return skb->len;
7957 }
7958
7959 static int nf_tables_dump_flowtable_start(struct netlink_callback *cb)
7960 {
7961         const struct nlattr * const *nla = cb->data;
7962         struct nft_flowtable_filter *filter = NULL;
7963
7964         if (nla[NFTA_FLOWTABLE_TABLE]) {
7965                 filter = kzalloc(sizeof(*filter), GFP_ATOMIC);
7966                 if (!filter)
7967                         return -ENOMEM;
7968
7969                 filter->table = nla_strdup(nla[NFTA_FLOWTABLE_TABLE],
7970                                            GFP_ATOMIC);
7971                 if (!filter->table) {
7972                         kfree(filter);
7973                         return -ENOMEM;
7974                 }
7975         }
7976
7977         cb->data = filter;
7978         return 0;
7979 }
7980
7981 static int nf_tables_dump_flowtable_done(struct netlink_callback *cb)
7982 {
7983         struct nft_flowtable_filter *filter = cb->data;
7984
7985         if (!filter)
7986                 return 0;
7987
7988         kfree(filter->table);
7989         kfree(filter);
7990
7991         return 0;
7992 }
7993
7994 /* called with rcu_read_lock held */
7995 static int nf_tables_getflowtable(struct sk_buff *skb,
7996                                   const struct nfnl_info *info,
7997                                   const struct nlattr * const nla[])
7998 {
7999         u8 genmask = nft_genmask_cur(info->net);
8000         u8 family = info->nfmsg->nfgen_family;
8001         struct nft_flowtable *flowtable;
8002         const struct nft_table *table;
8003         struct net *net = info->net;
8004         struct sk_buff *skb2;
8005         int err;
8006
8007         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
8008                 struct netlink_dump_control c = {
8009                         .start = nf_tables_dump_flowtable_start,
8010                         .dump = nf_tables_dump_flowtable,
8011                         .done = nf_tables_dump_flowtable_done,
8012                         .module = THIS_MODULE,
8013                         .data = (void *)nla,
8014                 };
8015
8016                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
8017         }
8018
8019         if (!nla[NFTA_FLOWTABLE_NAME])
8020                 return -EINVAL;
8021
8022         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8023                                  genmask, 0);
8024         if (IS_ERR(table))
8025                 return PTR_ERR(table);
8026
8027         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
8028                                          genmask);
8029         if (IS_ERR(flowtable))
8030                 return PTR_ERR(flowtable);
8031
8032         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
8033         if (!skb2)
8034                 return -ENOMEM;
8035
8036         err = nf_tables_fill_flowtable_info(skb2, net, NETLINK_CB(skb).portid,
8037                                             info->nlh->nlmsg_seq,
8038                                             NFT_MSG_NEWFLOWTABLE, 0, family,
8039                                             flowtable, &flowtable->hook_list);
8040         if (err < 0)
8041                 goto err_fill_flowtable_info;
8042
8043         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
8044
8045 err_fill_flowtable_info:
8046         kfree_skb(skb2);
8047         return err;
8048 }
8049
8050 static void nf_tables_flowtable_notify(struct nft_ctx *ctx,
8051                                        struct nft_flowtable *flowtable,
8052                                        struct list_head *hook_list,
8053                                        int event)
8054 {
8055         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
8056         struct sk_buff *skb;
8057         u16 flags = 0;
8058         int err;
8059
8060         if (!ctx->report &&
8061             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
8062                 return;
8063
8064         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
8065         if (skb == NULL)
8066                 goto err;
8067
8068         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
8069                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
8070
8071         err = nf_tables_fill_flowtable_info(skb, ctx->net, ctx->portid,
8072                                             ctx->seq, event, flags,
8073                                             ctx->family, flowtable, hook_list);
8074         if (err < 0) {
8075                 kfree_skb(skb);
8076                 goto err;
8077         }
8078
8079         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
8080         return;
8081 err:
8082         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
8083 }
8084
8085 static void nf_tables_flowtable_destroy(struct nft_flowtable *flowtable)
8086 {
8087         struct nft_hook *hook, *next;
8088
8089         flowtable->data.type->free(&flowtable->data);
8090         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
8091                 flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
8092                                             FLOW_BLOCK_UNBIND);
8093                 list_del_rcu(&hook->list);
8094                 kfree(hook);
8095         }
8096         kfree(flowtable->name);
8097         module_put(flowtable->data.type->owner);
8098         kfree(flowtable);
8099 }
8100
8101 static int nf_tables_fill_gen_info(struct sk_buff *skb, struct net *net,
8102                                    u32 portid, u32 seq)
8103 {
8104         struct nftables_pernet *nft_net = nft_pernet(net);
8105         struct nlmsghdr *nlh;
8106         char buf[TASK_COMM_LEN];
8107         int event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWGEN);
8108
8109         nlh = nfnl_msg_put(skb, portid, seq, event, 0, AF_UNSPEC,
8110                            NFNETLINK_V0, nft_base_seq(net));
8111         if (!nlh)
8112                 goto nla_put_failure;
8113
8114         if (nla_put_be32(skb, NFTA_GEN_ID, htonl(nft_net->base_seq)) ||
8115             nla_put_be32(skb, NFTA_GEN_PROC_PID, htonl(task_pid_nr(current))) ||
8116             nla_put_string(skb, NFTA_GEN_PROC_NAME, get_task_comm(buf, current)))
8117                 goto nla_put_failure;
8118
8119         nlmsg_end(skb, nlh);
8120         return 0;
8121
8122 nla_put_failure:
8123         nlmsg_trim(skb, nlh);
8124         return -EMSGSIZE;
8125 }
8126
8127 static void nft_flowtable_event(unsigned long event, struct net_device *dev,
8128                                 struct nft_flowtable *flowtable)
8129 {
8130         struct nft_hook *hook;
8131
8132         list_for_each_entry(hook, &flowtable->hook_list, list) {
8133                 if (hook->ops.dev != dev)
8134                         continue;
8135
8136                 /* flow_offload_netdev_event() cleans up entries for us. */
8137                 nft_unregister_flowtable_hook(dev_net(dev), flowtable, hook);
8138                 list_del_rcu(&hook->list);
8139                 kfree_rcu(hook, rcu);
8140                 break;
8141         }
8142 }
8143
8144 static int nf_tables_flowtable_event(struct notifier_block *this,
8145                                      unsigned long event, void *ptr)
8146 {
8147         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
8148         struct nft_flowtable *flowtable;
8149         struct nftables_pernet *nft_net;
8150         struct nft_table *table;
8151         struct net *net;
8152
8153         if (event != NETDEV_UNREGISTER)
8154                 return 0;
8155
8156         net = dev_net(dev);
8157         nft_net = nft_pernet(net);
8158         mutex_lock(&nft_net->commit_mutex);
8159         list_for_each_entry(table, &nft_net->tables, list) {
8160                 list_for_each_entry(flowtable, &table->flowtables, list) {
8161                         nft_flowtable_event(event, dev, flowtable);
8162                 }
8163         }
8164         mutex_unlock(&nft_net->commit_mutex);
8165
8166         return NOTIFY_DONE;
8167 }
8168
8169 static struct notifier_block nf_tables_flowtable_notifier = {
8170         .notifier_call  = nf_tables_flowtable_event,
8171 };
8172
8173 static void nf_tables_gen_notify(struct net *net, struct sk_buff *skb,
8174                                  int event)
8175 {
8176         struct nlmsghdr *nlh = nlmsg_hdr(skb);
8177         struct sk_buff *skb2;
8178         int err;
8179
8180         if (!nlmsg_report(nlh) &&
8181             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
8182                 return;
8183
8184         skb2 = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
8185         if (skb2 == NULL)
8186                 goto err;
8187
8188         err = nf_tables_fill_gen_info(skb2, net, NETLINK_CB(skb).portid,
8189                                       nlh->nlmsg_seq);
8190         if (err < 0) {
8191                 kfree_skb(skb2);
8192                 goto err;
8193         }
8194
8195         nfnetlink_send(skb2, net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
8196                        nlmsg_report(nlh), GFP_KERNEL);
8197         return;
8198 err:
8199         nfnetlink_set_err(net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
8200                           -ENOBUFS);
8201 }
8202
8203 static int nf_tables_getgen(struct sk_buff *skb, const struct nfnl_info *info,
8204                             const struct nlattr * const nla[])
8205 {
8206         struct sk_buff *skb2;
8207         int err;
8208
8209         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
8210         if (skb2 == NULL)
8211                 return -ENOMEM;
8212
8213         err = nf_tables_fill_gen_info(skb2, info->net, NETLINK_CB(skb).portid,
8214                                       info->nlh->nlmsg_seq);
8215         if (err < 0)
8216                 goto err_fill_gen_info;
8217
8218         return nfnetlink_unicast(skb2, info->net, NETLINK_CB(skb).portid);
8219
8220 err_fill_gen_info:
8221         kfree_skb(skb2);
8222         return err;
8223 }
8224
8225 static const struct nfnl_callback nf_tables_cb[NFT_MSG_MAX] = {
8226         [NFT_MSG_NEWTABLE] = {
8227                 .call           = nf_tables_newtable,
8228                 .type           = NFNL_CB_BATCH,
8229                 .attr_count     = NFTA_TABLE_MAX,
8230                 .policy         = nft_table_policy,
8231         },
8232         [NFT_MSG_GETTABLE] = {
8233                 .call           = nf_tables_gettable,
8234                 .type           = NFNL_CB_RCU,
8235                 .attr_count     = NFTA_TABLE_MAX,
8236                 .policy         = nft_table_policy,
8237         },
8238         [NFT_MSG_DELTABLE] = {
8239                 .call           = nf_tables_deltable,
8240                 .type           = NFNL_CB_BATCH,
8241                 .attr_count     = NFTA_TABLE_MAX,
8242                 .policy         = nft_table_policy,
8243         },
8244         [NFT_MSG_NEWCHAIN] = {
8245                 .call           = nf_tables_newchain,
8246                 .type           = NFNL_CB_BATCH,
8247                 .attr_count     = NFTA_CHAIN_MAX,
8248                 .policy         = nft_chain_policy,
8249         },
8250         [NFT_MSG_GETCHAIN] = {
8251                 .call           = nf_tables_getchain,
8252                 .type           = NFNL_CB_RCU,
8253                 .attr_count     = NFTA_CHAIN_MAX,
8254                 .policy         = nft_chain_policy,
8255         },
8256         [NFT_MSG_DELCHAIN] = {
8257                 .call           = nf_tables_delchain,
8258                 .type           = NFNL_CB_BATCH,
8259                 .attr_count     = NFTA_CHAIN_MAX,
8260                 .policy         = nft_chain_policy,
8261         },
8262         [NFT_MSG_NEWRULE] = {
8263                 .call           = nf_tables_newrule,
8264                 .type           = NFNL_CB_BATCH,
8265                 .attr_count     = NFTA_RULE_MAX,
8266                 .policy         = nft_rule_policy,
8267         },
8268         [NFT_MSG_GETRULE] = {
8269                 .call           = nf_tables_getrule,
8270                 .type           = NFNL_CB_RCU,
8271                 .attr_count     = NFTA_RULE_MAX,
8272                 .policy         = nft_rule_policy,
8273         },
8274         [NFT_MSG_DELRULE] = {
8275                 .call           = nf_tables_delrule,
8276                 .type           = NFNL_CB_BATCH,
8277                 .attr_count     = NFTA_RULE_MAX,
8278                 .policy         = nft_rule_policy,
8279         },
8280         [NFT_MSG_NEWSET] = {
8281                 .call           = nf_tables_newset,
8282                 .type           = NFNL_CB_BATCH,
8283                 .attr_count     = NFTA_SET_MAX,
8284                 .policy         = nft_set_policy,
8285         },
8286         [NFT_MSG_GETSET] = {
8287                 .call           = nf_tables_getset,
8288                 .type           = NFNL_CB_RCU,
8289                 .attr_count     = NFTA_SET_MAX,
8290                 .policy         = nft_set_policy,
8291         },
8292         [NFT_MSG_DELSET] = {
8293                 .call           = nf_tables_delset,
8294                 .type           = NFNL_CB_BATCH,
8295                 .attr_count     = NFTA_SET_MAX,
8296                 .policy         = nft_set_policy,
8297         },
8298         [NFT_MSG_NEWSETELEM] = {
8299                 .call           = nf_tables_newsetelem,
8300                 .type           = NFNL_CB_BATCH,
8301                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
8302                 .policy         = nft_set_elem_list_policy,
8303         },
8304         [NFT_MSG_GETSETELEM] = {
8305                 .call           = nf_tables_getsetelem,
8306                 .type           = NFNL_CB_RCU,
8307                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
8308                 .policy         = nft_set_elem_list_policy,
8309         },
8310         [NFT_MSG_DELSETELEM] = {
8311                 .call           = nf_tables_delsetelem,
8312                 .type           = NFNL_CB_BATCH,
8313                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
8314                 .policy         = nft_set_elem_list_policy,
8315         },
8316         [NFT_MSG_GETGEN] = {
8317                 .call           = nf_tables_getgen,
8318                 .type           = NFNL_CB_RCU,
8319         },
8320         [NFT_MSG_NEWOBJ] = {
8321                 .call           = nf_tables_newobj,
8322                 .type           = NFNL_CB_BATCH,
8323                 .attr_count     = NFTA_OBJ_MAX,
8324                 .policy         = nft_obj_policy,
8325         },
8326         [NFT_MSG_GETOBJ] = {
8327                 .call           = nf_tables_getobj,
8328                 .type           = NFNL_CB_RCU,
8329                 .attr_count     = NFTA_OBJ_MAX,
8330                 .policy         = nft_obj_policy,
8331         },
8332         [NFT_MSG_DELOBJ] = {
8333                 .call           = nf_tables_delobj,
8334                 .type           = NFNL_CB_BATCH,
8335                 .attr_count     = NFTA_OBJ_MAX,
8336                 .policy         = nft_obj_policy,
8337         },
8338         [NFT_MSG_GETOBJ_RESET] = {
8339                 .call           = nf_tables_getobj,
8340                 .type           = NFNL_CB_RCU,
8341                 .attr_count     = NFTA_OBJ_MAX,
8342                 .policy         = nft_obj_policy,
8343         },
8344         [NFT_MSG_NEWFLOWTABLE] = {
8345                 .call           = nf_tables_newflowtable,
8346                 .type           = NFNL_CB_BATCH,
8347                 .attr_count     = NFTA_FLOWTABLE_MAX,
8348                 .policy         = nft_flowtable_policy,
8349         },
8350         [NFT_MSG_GETFLOWTABLE] = {
8351                 .call           = nf_tables_getflowtable,
8352                 .type           = NFNL_CB_RCU,
8353                 .attr_count     = NFTA_FLOWTABLE_MAX,
8354                 .policy         = nft_flowtable_policy,
8355         },
8356         [NFT_MSG_DELFLOWTABLE] = {
8357                 .call           = nf_tables_delflowtable,
8358                 .type           = NFNL_CB_BATCH,
8359                 .attr_count     = NFTA_FLOWTABLE_MAX,
8360                 .policy         = nft_flowtable_policy,
8361         },
8362 };
8363
8364 static int nf_tables_validate(struct net *net)
8365 {
8366         struct nftables_pernet *nft_net = nft_pernet(net);
8367         struct nft_table *table;
8368
8369         switch (nft_net->validate_state) {
8370         case NFT_VALIDATE_SKIP:
8371                 break;
8372         case NFT_VALIDATE_NEED:
8373                 nft_validate_state_update(net, NFT_VALIDATE_DO);
8374                 fallthrough;
8375         case NFT_VALIDATE_DO:
8376                 list_for_each_entry(table, &nft_net->tables, list) {
8377                         if (nft_table_validate(net, table) < 0)
8378                                 return -EAGAIN;
8379                 }
8380                 break;
8381         }
8382
8383         return 0;
8384 }
8385
8386 /* a drop policy has to be deferred until all rules have been activated,
8387  * otherwise a large ruleset that contains a drop-policy base chain will
8388  * cause all packets to get dropped until the full transaction has been
8389  * processed.
8390  *
8391  * We defer the drop policy until the transaction has been finalized.
8392  */
8393 static void nft_chain_commit_drop_policy(struct nft_trans *trans)
8394 {
8395         struct nft_base_chain *basechain;
8396
8397         if (nft_trans_chain_policy(trans) != NF_DROP)
8398                 return;
8399
8400         if (!nft_is_base_chain(trans->ctx.chain))
8401                 return;
8402
8403         basechain = nft_base_chain(trans->ctx.chain);
8404         basechain->policy = NF_DROP;
8405 }
8406
8407 static void nft_chain_commit_update(struct nft_trans *trans)
8408 {
8409         struct nft_base_chain *basechain;
8410
8411         if (nft_trans_chain_name(trans)) {
8412                 rhltable_remove(&trans->ctx.table->chains_ht,
8413                                 &trans->ctx.chain->rhlhead,
8414                                 nft_chain_ht_params);
8415                 swap(trans->ctx.chain->name, nft_trans_chain_name(trans));
8416                 rhltable_insert_key(&trans->ctx.table->chains_ht,
8417                                     trans->ctx.chain->name,
8418                                     &trans->ctx.chain->rhlhead,
8419                                     nft_chain_ht_params);
8420         }
8421
8422         if (!nft_is_base_chain(trans->ctx.chain))
8423                 return;
8424
8425         nft_chain_stats_replace(trans);
8426
8427         basechain = nft_base_chain(trans->ctx.chain);
8428
8429         switch (nft_trans_chain_policy(trans)) {
8430         case NF_DROP:
8431         case NF_ACCEPT:
8432                 basechain->policy = nft_trans_chain_policy(trans);
8433                 break;
8434         }
8435 }
8436
8437 static void nft_obj_commit_update(struct nft_trans *trans)
8438 {
8439         struct nft_object *newobj;
8440         struct nft_object *obj;
8441
8442         obj = nft_trans_obj(trans);
8443         newobj = nft_trans_obj_newobj(trans);
8444
8445         if (obj->ops->update)
8446                 obj->ops->update(obj, newobj);
8447
8448         nft_obj_destroy(&trans->ctx, newobj);
8449 }
8450
8451 static void nft_commit_release(struct nft_trans *trans)
8452 {
8453         switch (trans->msg_type) {
8454         case NFT_MSG_DELTABLE:
8455                 nf_tables_table_destroy(&trans->ctx);
8456                 break;
8457         case NFT_MSG_NEWCHAIN:
8458                 free_percpu(nft_trans_chain_stats(trans));
8459                 kfree(nft_trans_chain_name(trans));
8460                 break;
8461         case NFT_MSG_DELCHAIN:
8462                 nf_tables_chain_destroy(&trans->ctx);
8463                 break;
8464         case NFT_MSG_DELRULE:
8465                 if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
8466                         nft_flow_rule_destroy(nft_trans_flow_rule(trans));
8467
8468                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
8469                 break;
8470         case NFT_MSG_DELSET:
8471                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
8472                 break;
8473         case NFT_MSG_DELSETELEM:
8474                 nf_tables_set_elem_destroy(&trans->ctx,
8475                                            nft_trans_elem_set(trans),
8476                                            nft_trans_elem(trans).priv);
8477                 break;
8478         case NFT_MSG_DELOBJ:
8479                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
8480                 break;
8481         case NFT_MSG_DELFLOWTABLE:
8482                 if (nft_trans_flowtable_update(trans))
8483                         nft_flowtable_hooks_destroy(&nft_trans_flowtable_hooks(trans));
8484                 else
8485                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
8486                 break;
8487         }
8488
8489         if (trans->put_net)
8490                 put_net(trans->ctx.net);
8491
8492         kfree(trans);
8493 }
8494
8495 static void nf_tables_trans_destroy_work(struct work_struct *w)
8496 {
8497         struct nft_trans *trans, *next;
8498         LIST_HEAD(head);
8499
8500         spin_lock(&nf_tables_destroy_list_lock);
8501         list_splice_init(&nf_tables_destroy_list, &head);
8502         spin_unlock(&nf_tables_destroy_list_lock);
8503
8504         if (list_empty(&head))
8505                 return;
8506
8507         synchronize_rcu();
8508
8509         list_for_each_entry_safe(trans, next, &head, list) {
8510                 list_del(&trans->list);
8511                 nft_commit_release(trans);
8512         }
8513 }
8514
8515 void nf_tables_trans_destroy_flush_work(void)
8516 {
8517         flush_work(&trans_destroy_work);
8518 }
8519 EXPORT_SYMBOL_GPL(nf_tables_trans_destroy_flush_work);
8520
8521 static bool nft_expr_reduce(struct nft_regs_track *track,
8522                             const struct nft_expr *expr)
8523 {
8524         return false;
8525 }
8526
8527 static int nf_tables_commit_chain_prepare(struct net *net, struct nft_chain *chain)
8528 {
8529         const struct nft_expr *expr, *last;
8530         struct nft_regs_track track = {};
8531         unsigned int size, data_size;
8532         void *data, *data_boundary;
8533         struct nft_rule_dp *prule;
8534         struct nft_rule *rule;
8535
8536         /* already handled or inactive chain? */
8537         if (chain->blob_next || !nft_is_active_next(net, chain))
8538                 return 0;
8539
8540         data_size = 0;
8541         list_for_each_entry(rule, &chain->rules, list) {
8542                 if (nft_is_active_next(net, rule)) {
8543                         data_size += sizeof(*prule) + rule->dlen;
8544                         if (data_size > INT_MAX)
8545                                 return -ENOMEM;
8546                 }
8547         }
8548         data_size += offsetof(struct nft_rule_dp, data);        /* last rule */
8549
8550         chain->blob_next = nf_tables_chain_alloc_rules(data_size);
8551         if (!chain->blob_next)
8552                 return -ENOMEM;
8553
8554         data = (void *)chain->blob_next->data;
8555         data_boundary = data + data_size;
8556         size = 0;
8557
8558         list_for_each_entry(rule, &chain->rules, list) {
8559                 if (!nft_is_active_next(net, rule))
8560                         continue;
8561
8562                 prule = (struct nft_rule_dp *)data;
8563                 data += offsetof(struct nft_rule_dp, data);
8564                 if (WARN_ON_ONCE(data > data_boundary))
8565                         return -ENOMEM;
8566
8567                 size = 0;
8568                 track.last = nft_expr_last(rule);
8569                 nft_rule_for_each_expr(expr, last, rule) {
8570                         track.cur = expr;
8571
8572                         if (nft_expr_reduce(&track, expr)) {
8573                                 expr = track.cur;
8574                                 continue;
8575                         }
8576
8577                         if (WARN_ON_ONCE(data + expr->ops->size > data_boundary))
8578                                 return -ENOMEM;
8579
8580                         memcpy(data + size, expr, expr->ops->size);
8581                         size += expr->ops->size;
8582                 }
8583                 if (WARN_ON_ONCE(size >= 1 << 12))
8584                         return -ENOMEM;
8585
8586                 prule->handle = rule->handle;
8587                 prule->dlen = size;
8588                 prule->is_last = 0;
8589
8590                 data += size;
8591                 size = 0;
8592                 chain->blob_next->size += (unsigned long)(data - (void *)prule);
8593         }
8594
8595         prule = (struct nft_rule_dp *)data;
8596         data += offsetof(struct nft_rule_dp, data);
8597         if (WARN_ON_ONCE(data > data_boundary))
8598                 return -ENOMEM;
8599
8600         nft_last_rule(chain->blob_next, prule);
8601
8602         return 0;
8603 }
8604
8605 static void nf_tables_commit_chain_prepare_cancel(struct net *net)
8606 {
8607         struct nftables_pernet *nft_net = nft_pernet(net);
8608         struct nft_trans *trans, *next;
8609
8610         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
8611                 struct nft_chain *chain = trans->ctx.chain;
8612
8613                 if (trans->msg_type == NFT_MSG_NEWRULE ||
8614                     trans->msg_type == NFT_MSG_DELRULE) {
8615                         kvfree(chain->blob_next);
8616                         chain->blob_next = NULL;
8617                 }
8618         }
8619 }
8620
8621 static void __nf_tables_commit_chain_free_rules_old(struct rcu_head *h)
8622 {
8623         struct nft_rules_old *o = container_of(h, struct nft_rules_old, h);
8624
8625         kvfree(o->blob);
8626 }
8627
8628 static void nf_tables_commit_chain_free_rules_old(struct nft_rule_blob *blob)
8629 {
8630         struct nft_rules_old *old;
8631
8632         /* rcu_head is after end marker */
8633         old = (void *)blob + sizeof(*blob) + blob->size;
8634         old->blob = blob;
8635
8636         call_rcu(&old->h, __nf_tables_commit_chain_free_rules_old);
8637 }
8638
8639 static void nf_tables_commit_chain(struct net *net, struct nft_chain *chain)
8640 {
8641         struct nft_rule_blob *g0, *g1;
8642         bool next_genbit;
8643
8644         next_genbit = nft_gencursor_next(net);
8645
8646         g0 = rcu_dereference_protected(chain->blob_gen_0,
8647                                        lockdep_commit_lock_is_held(net));
8648         g1 = rcu_dereference_protected(chain->blob_gen_1,
8649                                        lockdep_commit_lock_is_held(net));
8650
8651         /* No changes to this chain? */
8652         if (chain->blob_next == NULL) {
8653                 /* chain had no change in last or next generation */
8654                 if (g0 == g1)
8655                         return;
8656                 /*
8657                  * chain had no change in this generation; make sure next
8658                  * one uses same rules as current generation.
8659                  */
8660                 if (next_genbit) {
8661                         rcu_assign_pointer(chain->blob_gen_1, g0);
8662                         nf_tables_commit_chain_free_rules_old(g1);
8663                 } else {
8664                         rcu_assign_pointer(chain->blob_gen_0, g1);
8665                         nf_tables_commit_chain_free_rules_old(g0);
8666                 }
8667
8668                 return;
8669         }
8670
8671         if (next_genbit)
8672                 rcu_assign_pointer(chain->blob_gen_1, chain->blob_next);
8673         else
8674                 rcu_assign_pointer(chain->blob_gen_0, chain->blob_next);
8675
8676         chain->blob_next = NULL;
8677
8678         if (g0 == g1)
8679                 return;
8680
8681         if (next_genbit)
8682                 nf_tables_commit_chain_free_rules_old(g1);
8683         else
8684                 nf_tables_commit_chain_free_rules_old(g0);
8685 }
8686
8687 static void nft_obj_del(struct nft_object *obj)
8688 {
8689         rhltable_remove(&nft_objname_ht, &obj->rhlhead, nft_objname_ht_params);
8690         list_del_rcu(&obj->list);
8691 }
8692
8693 void nft_chain_del(struct nft_chain *chain)
8694 {
8695         struct nft_table *table = chain->table;
8696
8697         WARN_ON_ONCE(rhltable_remove(&table->chains_ht, &chain->rhlhead,
8698                                      nft_chain_ht_params));
8699         list_del_rcu(&chain->list);
8700 }
8701
8702 static void nf_tables_module_autoload_cleanup(struct net *net)
8703 {
8704         struct nftables_pernet *nft_net = nft_pernet(net);
8705         struct nft_module_request *req, *next;
8706
8707         WARN_ON_ONCE(!list_empty(&nft_net->commit_list));
8708         list_for_each_entry_safe(req, next, &nft_net->module_list, list) {
8709                 WARN_ON_ONCE(!req->done);
8710                 list_del(&req->list);
8711                 kfree(req);
8712         }
8713 }
8714
8715 static void nf_tables_commit_release(struct net *net)
8716 {
8717         struct nftables_pernet *nft_net = nft_pernet(net);
8718         struct nft_trans *trans;
8719
8720         /* all side effects have to be made visible.
8721          * For example, if a chain named 'foo' has been deleted, a
8722          * new transaction must not find it anymore.
8723          *
8724          * Memory reclaim happens asynchronously from work queue
8725          * to prevent expensive synchronize_rcu() in commit phase.
8726          */
8727         if (list_empty(&nft_net->commit_list)) {
8728                 nf_tables_module_autoload_cleanup(net);
8729                 mutex_unlock(&nft_net->commit_mutex);
8730                 return;
8731         }
8732
8733         trans = list_last_entry(&nft_net->commit_list,
8734                                 struct nft_trans, list);
8735         get_net(trans->ctx.net);
8736         WARN_ON_ONCE(trans->put_net);
8737
8738         trans->put_net = true;
8739         spin_lock(&nf_tables_destroy_list_lock);
8740         list_splice_tail_init(&nft_net->commit_list, &nf_tables_destroy_list);
8741         spin_unlock(&nf_tables_destroy_list_lock);
8742
8743         nf_tables_module_autoload_cleanup(net);
8744         schedule_work(&trans_destroy_work);
8745
8746         mutex_unlock(&nft_net->commit_mutex);
8747 }
8748
8749 static void nft_commit_notify(struct net *net, u32 portid)
8750 {
8751         struct nftables_pernet *nft_net = nft_pernet(net);
8752         struct sk_buff *batch_skb = NULL, *nskb, *skb;
8753         unsigned char *data;
8754         int len;
8755
8756         list_for_each_entry_safe(skb, nskb, &nft_net->notify_list, list) {
8757                 if (!batch_skb) {
8758 new_batch:
8759                         batch_skb = skb;
8760                         len = NLMSG_GOODSIZE - skb->len;
8761                         list_del(&skb->list);
8762                         continue;
8763                 }
8764                 len -= skb->len;
8765                 if (len > 0 && NFT_CB(skb).report == NFT_CB(batch_skb).report) {
8766                         data = skb_put(batch_skb, skb->len);
8767                         memcpy(data, skb->data, skb->len);
8768                         list_del(&skb->list);
8769                         kfree_skb(skb);
8770                         continue;
8771                 }
8772                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
8773                                NFT_CB(batch_skb).report, GFP_KERNEL);
8774                 goto new_batch;
8775         }
8776
8777         if (batch_skb) {
8778                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
8779                                NFT_CB(batch_skb).report, GFP_KERNEL);
8780         }
8781
8782         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
8783 }
8784
8785 static int nf_tables_commit_audit_alloc(struct list_head *adl,
8786                                         struct nft_table *table)
8787 {
8788         struct nft_audit_data *adp;
8789
8790         list_for_each_entry(adp, adl, list) {
8791                 if (adp->table == table)
8792                         return 0;
8793         }
8794         adp = kzalloc(sizeof(*adp), GFP_KERNEL);
8795         if (!adp)
8796                 return -ENOMEM;
8797         adp->table = table;
8798         list_add(&adp->list, adl);
8799         return 0;
8800 }
8801
8802 static void nf_tables_commit_audit_free(struct list_head *adl)
8803 {
8804         struct nft_audit_data *adp, *adn;
8805
8806         list_for_each_entry_safe(adp, adn, adl, list) {
8807                 list_del(&adp->list);
8808                 kfree(adp);
8809         }
8810 }
8811
8812 static void nf_tables_commit_audit_collect(struct list_head *adl,
8813                                            struct nft_table *table, u32 op)
8814 {
8815         struct nft_audit_data *adp;
8816
8817         list_for_each_entry(adp, adl, list) {
8818                 if (adp->table == table)
8819                         goto found;
8820         }
8821         WARN_ONCE(1, "table=%s not expected in commit list", table->name);
8822         return;
8823 found:
8824         adp->entries++;
8825         if (!adp->op || adp->op > op)
8826                 adp->op = op;
8827 }
8828
8829 #define AUNFTABLENAMELEN (NFT_TABLE_MAXNAMELEN + 22)
8830
8831 static void nf_tables_commit_audit_log(struct list_head *adl, u32 generation)
8832 {
8833         struct nft_audit_data *adp, *adn;
8834         char aubuf[AUNFTABLENAMELEN];
8835
8836         list_for_each_entry_safe(adp, adn, adl, list) {
8837                 snprintf(aubuf, AUNFTABLENAMELEN, "%s:%u", adp->table->name,
8838                          generation);
8839                 audit_log_nfcfg(aubuf, adp->table->family, adp->entries,
8840                                 nft2audit_op[adp->op], GFP_KERNEL);
8841                 list_del(&adp->list);
8842                 kfree(adp);
8843         }
8844 }
8845
8846 static int nf_tables_commit(struct net *net, struct sk_buff *skb)
8847 {
8848         struct nftables_pernet *nft_net = nft_pernet(net);
8849         struct nft_trans *trans, *next;
8850         struct nft_trans_elem *te;
8851         struct nft_chain *chain;
8852         struct nft_table *table;
8853         unsigned int base_seq;
8854         LIST_HEAD(adl);
8855         int err;
8856
8857         if (list_empty(&nft_net->commit_list)) {
8858                 mutex_unlock(&nft_net->commit_mutex);
8859                 return 0;
8860         }
8861
8862         /* 0. Validate ruleset, otherwise roll back for error reporting. */
8863         if (nf_tables_validate(net) < 0)
8864                 return -EAGAIN;
8865
8866         err = nft_flow_rule_offload_commit(net);
8867         if (err < 0)
8868                 return err;
8869
8870         /* 1.  Allocate space for next generation rules_gen_X[] */
8871         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
8872                 int ret;
8873
8874                 ret = nf_tables_commit_audit_alloc(&adl, trans->ctx.table);
8875                 if (ret) {
8876                         nf_tables_commit_chain_prepare_cancel(net);
8877                         nf_tables_commit_audit_free(&adl);
8878                         return ret;
8879                 }
8880                 if (trans->msg_type == NFT_MSG_NEWRULE ||
8881                     trans->msg_type == NFT_MSG_DELRULE) {
8882                         chain = trans->ctx.chain;
8883
8884                         ret = nf_tables_commit_chain_prepare(net, chain);
8885                         if (ret < 0) {
8886                                 nf_tables_commit_chain_prepare_cancel(net);
8887                                 nf_tables_commit_audit_free(&adl);
8888                                 return ret;
8889                         }
8890                 }
8891         }
8892
8893         /* step 2.  Make rules_gen_X visible to packet path */
8894         list_for_each_entry(table, &nft_net->tables, list) {
8895                 list_for_each_entry(chain, &table->chains, list)
8896                         nf_tables_commit_chain(net, chain);
8897         }
8898
8899         /*
8900          * Bump generation counter, invalidate any dump in progress.
8901          * Cannot fail after this point.
8902          */
8903         base_seq = READ_ONCE(nft_net->base_seq);
8904         while (++base_seq == 0)
8905                 ;
8906
8907         WRITE_ONCE(nft_net->base_seq, base_seq);
8908
8909         /* step 3. Start new generation, rules_gen_X now in use. */
8910         net->nft.gencursor = nft_gencursor_next(net);
8911
8912         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
8913                 nf_tables_commit_audit_collect(&adl, trans->ctx.table,
8914                                                trans->msg_type);
8915                 switch (trans->msg_type) {
8916                 case NFT_MSG_NEWTABLE:
8917                         if (nft_trans_table_update(trans)) {
8918                                 if (!(trans->ctx.table->flags & __NFT_TABLE_F_UPDATE)) {
8919                                         nft_trans_destroy(trans);
8920                                         break;
8921                                 }
8922                                 if (trans->ctx.table->flags & NFT_TABLE_F_DORMANT)
8923                                         nf_tables_table_disable(net, trans->ctx.table);
8924
8925                                 trans->ctx.table->flags &= ~__NFT_TABLE_F_UPDATE;
8926                         } else {
8927                                 nft_clear(net, trans->ctx.table);
8928                         }
8929                         nf_tables_table_notify(&trans->ctx, NFT_MSG_NEWTABLE);
8930                         nft_trans_destroy(trans);
8931                         break;
8932                 case NFT_MSG_DELTABLE:
8933                         list_del_rcu(&trans->ctx.table->list);
8934                         nf_tables_table_notify(&trans->ctx, NFT_MSG_DELTABLE);
8935                         break;
8936                 case NFT_MSG_NEWCHAIN:
8937                         if (nft_trans_chain_update(trans)) {
8938                                 nft_chain_commit_update(trans);
8939                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN);
8940                                 /* trans destroyed after rcu grace period */
8941                         } else {
8942                                 nft_chain_commit_drop_policy(trans);
8943                                 nft_clear(net, trans->ctx.chain);
8944                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN);
8945                                 nft_trans_destroy(trans);
8946                         }
8947                         break;
8948                 case NFT_MSG_DELCHAIN:
8949                         nft_chain_del(trans->ctx.chain);
8950                         nf_tables_chain_notify(&trans->ctx, NFT_MSG_DELCHAIN);
8951                         nf_tables_unregister_hook(trans->ctx.net,
8952                                                   trans->ctx.table,
8953                                                   trans->ctx.chain);
8954                         break;
8955                 case NFT_MSG_NEWRULE:
8956                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
8957                         nf_tables_rule_notify(&trans->ctx,
8958                                               nft_trans_rule(trans),
8959                                               NFT_MSG_NEWRULE);
8960                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
8961                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
8962
8963                         nft_trans_destroy(trans);
8964                         break;
8965                 case NFT_MSG_DELRULE:
8966                         list_del_rcu(&nft_trans_rule(trans)->list);
8967                         nf_tables_rule_notify(&trans->ctx,
8968                                               nft_trans_rule(trans),
8969                                               NFT_MSG_DELRULE);
8970                         nft_rule_expr_deactivate(&trans->ctx,
8971                                                  nft_trans_rule(trans),
8972                                                  NFT_TRANS_COMMIT);
8973                         break;
8974                 case NFT_MSG_NEWSET:
8975                         nft_clear(net, nft_trans_set(trans));
8976                         /* This avoids hitting -EBUSY when deleting the table
8977                          * from the transaction.
8978                          */
8979                         if (nft_set_is_anonymous(nft_trans_set(trans)) &&
8980                             !list_empty(&nft_trans_set(trans)->bindings))
8981                                 trans->ctx.table->use--;
8982
8983                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
8984                                              NFT_MSG_NEWSET, GFP_KERNEL);
8985                         nft_trans_destroy(trans);
8986                         break;
8987                 case NFT_MSG_DELSET:
8988                         list_del_rcu(&nft_trans_set(trans)->list);
8989                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
8990                                              NFT_MSG_DELSET, GFP_KERNEL);
8991                         break;
8992                 case NFT_MSG_NEWSETELEM:
8993                         te = (struct nft_trans_elem *)trans->data;
8994
8995                         nft_setelem_activate(net, te->set, &te->elem);
8996                         nf_tables_setelem_notify(&trans->ctx, te->set,
8997                                                  &te->elem,
8998                                                  NFT_MSG_NEWSETELEM);
8999                         nft_trans_destroy(trans);
9000                         break;
9001                 case NFT_MSG_DELSETELEM:
9002                         te = (struct nft_trans_elem *)trans->data;
9003
9004                         nf_tables_setelem_notify(&trans->ctx, te->set,
9005                                                  &te->elem,
9006                                                  NFT_MSG_DELSETELEM);
9007                         nft_setelem_remove(net, te->set, &te->elem);
9008                         if (!nft_setelem_is_catchall(te->set, &te->elem)) {
9009                                 atomic_dec(&te->set->nelems);
9010                                 te->set->ndeact--;
9011                         }
9012                         break;
9013                 case NFT_MSG_NEWOBJ:
9014                         if (nft_trans_obj_update(trans)) {
9015                                 nft_obj_commit_update(trans);
9016                                 nf_tables_obj_notify(&trans->ctx,
9017                                                      nft_trans_obj(trans),
9018                                                      NFT_MSG_NEWOBJ);
9019                         } else {
9020                                 nft_clear(net, nft_trans_obj(trans));
9021                                 nf_tables_obj_notify(&trans->ctx,
9022                                                      nft_trans_obj(trans),
9023                                                      NFT_MSG_NEWOBJ);
9024                                 nft_trans_destroy(trans);
9025                         }
9026                         break;
9027                 case NFT_MSG_DELOBJ:
9028                         nft_obj_del(nft_trans_obj(trans));
9029                         nf_tables_obj_notify(&trans->ctx, nft_trans_obj(trans),
9030                                              NFT_MSG_DELOBJ);
9031                         break;
9032                 case NFT_MSG_NEWFLOWTABLE:
9033                         if (nft_trans_flowtable_update(trans)) {
9034                                 nft_trans_flowtable(trans)->data.flags =
9035                                         nft_trans_flowtable_flags(trans);
9036                                 nf_tables_flowtable_notify(&trans->ctx,
9037                                                            nft_trans_flowtable(trans),
9038                                                            &nft_trans_flowtable_hooks(trans),
9039                                                            NFT_MSG_NEWFLOWTABLE);
9040                                 list_splice(&nft_trans_flowtable_hooks(trans),
9041                                             &nft_trans_flowtable(trans)->hook_list);
9042                         } else {
9043                                 nft_clear(net, nft_trans_flowtable(trans));
9044                                 nf_tables_flowtable_notify(&trans->ctx,
9045                                                            nft_trans_flowtable(trans),
9046                                                            &nft_trans_flowtable(trans)->hook_list,
9047                                                            NFT_MSG_NEWFLOWTABLE);
9048                         }
9049                         nft_trans_destroy(trans);
9050                         break;
9051                 case NFT_MSG_DELFLOWTABLE:
9052                         if (nft_trans_flowtable_update(trans)) {
9053                                 nf_tables_flowtable_notify(&trans->ctx,
9054                                                            nft_trans_flowtable(trans),
9055                                                            &nft_trans_flowtable_hooks(trans),
9056                                                            NFT_MSG_DELFLOWTABLE);
9057                                 nft_unregister_flowtable_net_hooks(net,
9058                                                                    &nft_trans_flowtable_hooks(trans));
9059                         } else {
9060                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
9061                                 nf_tables_flowtable_notify(&trans->ctx,
9062                                                            nft_trans_flowtable(trans),
9063                                                            &nft_trans_flowtable(trans)->hook_list,
9064                                                            NFT_MSG_DELFLOWTABLE);
9065                                 nft_unregister_flowtable_net_hooks(net,
9066                                                 &nft_trans_flowtable(trans)->hook_list);
9067                         }
9068                         break;
9069                 }
9070         }
9071
9072         nft_commit_notify(net, NETLINK_CB(skb).portid);
9073         nf_tables_gen_notify(net, skb, NFT_MSG_NEWGEN);
9074         nf_tables_commit_audit_log(&adl, nft_net->base_seq);
9075         nf_tables_commit_release(net);
9076
9077         return 0;
9078 }
9079
9080 static void nf_tables_module_autoload(struct net *net)
9081 {
9082         struct nftables_pernet *nft_net = nft_pernet(net);
9083         struct nft_module_request *req, *next;
9084         LIST_HEAD(module_list);
9085
9086         list_splice_init(&nft_net->module_list, &module_list);
9087         mutex_unlock(&nft_net->commit_mutex);
9088         list_for_each_entry_safe(req, next, &module_list, list) {
9089                 request_module("%s", req->module);
9090                 req->done = true;
9091         }
9092         mutex_lock(&nft_net->commit_mutex);
9093         list_splice(&module_list, &nft_net->module_list);
9094 }
9095
9096 static void nf_tables_abort_release(struct nft_trans *trans)
9097 {
9098         switch (trans->msg_type) {
9099         case NFT_MSG_NEWTABLE:
9100                 nf_tables_table_destroy(&trans->ctx);
9101                 break;
9102         case NFT_MSG_NEWCHAIN:
9103                 nf_tables_chain_destroy(&trans->ctx);
9104                 break;
9105         case NFT_MSG_NEWRULE:
9106                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
9107                 break;
9108         case NFT_MSG_NEWSET:
9109                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
9110                 break;
9111         case NFT_MSG_NEWSETELEM:
9112                 nft_set_elem_destroy(nft_trans_elem_set(trans),
9113                                      nft_trans_elem(trans).priv, true);
9114                 break;
9115         case NFT_MSG_NEWOBJ:
9116                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
9117                 break;
9118         case NFT_MSG_NEWFLOWTABLE:
9119                 if (nft_trans_flowtable_update(trans))
9120                         nft_flowtable_hooks_destroy(&nft_trans_flowtable_hooks(trans));
9121                 else
9122                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
9123                 break;
9124         }
9125         kfree(trans);
9126 }
9127
9128 static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action)
9129 {
9130         struct nftables_pernet *nft_net = nft_pernet(net);
9131         struct nft_trans *trans, *next;
9132         struct nft_trans_elem *te;
9133
9134         if (action == NFNL_ABORT_VALIDATE &&
9135             nf_tables_validate(net) < 0)
9136                 return -EAGAIN;
9137
9138         list_for_each_entry_safe_reverse(trans, next, &nft_net->commit_list,
9139                                          list) {
9140                 switch (trans->msg_type) {
9141                 case NFT_MSG_NEWTABLE:
9142                         if (nft_trans_table_update(trans)) {
9143                                 if (!(trans->ctx.table->flags & __NFT_TABLE_F_UPDATE)) {
9144                                         nft_trans_destroy(trans);
9145                                         break;
9146                                 }
9147                                 if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_DORMANT) {
9148                                         nf_tables_table_disable(net, trans->ctx.table);
9149                                         trans->ctx.table->flags |= NFT_TABLE_F_DORMANT;
9150                                 } else if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_AWAKEN) {
9151                                         trans->ctx.table->flags &= ~NFT_TABLE_F_DORMANT;
9152                                 }
9153                                 trans->ctx.table->flags &= ~__NFT_TABLE_F_UPDATE;
9154                                 nft_trans_destroy(trans);
9155                         } else {
9156                                 list_del_rcu(&trans->ctx.table->list);
9157                         }
9158                         break;
9159                 case NFT_MSG_DELTABLE:
9160                         nft_clear(trans->ctx.net, trans->ctx.table);
9161                         nft_trans_destroy(trans);
9162                         break;
9163                 case NFT_MSG_NEWCHAIN:
9164                         if (nft_trans_chain_update(trans)) {
9165                                 free_percpu(nft_trans_chain_stats(trans));
9166                                 kfree(nft_trans_chain_name(trans));
9167                                 nft_trans_destroy(trans);
9168                         } else {
9169                                 if (nft_chain_is_bound(trans->ctx.chain)) {
9170                                         nft_trans_destroy(trans);
9171                                         break;
9172                                 }
9173                                 trans->ctx.table->use--;
9174                                 nft_chain_del(trans->ctx.chain);
9175                                 nf_tables_unregister_hook(trans->ctx.net,
9176                                                           trans->ctx.table,
9177                                                           trans->ctx.chain);
9178                         }
9179                         break;
9180                 case NFT_MSG_DELCHAIN:
9181                         trans->ctx.table->use++;
9182                         nft_clear(trans->ctx.net, trans->ctx.chain);
9183                         nft_trans_destroy(trans);
9184                         break;
9185                 case NFT_MSG_NEWRULE:
9186                         trans->ctx.chain->use--;
9187                         list_del_rcu(&nft_trans_rule(trans)->list);
9188                         nft_rule_expr_deactivate(&trans->ctx,
9189                                                  nft_trans_rule(trans),
9190                                                  NFT_TRANS_ABORT);
9191                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
9192                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
9193                         break;
9194                 case NFT_MSG_DELRULE:
9195                         trans->ctx.chain->use++;
9196                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
9197                         nft_rule_expr_activate(&trans->ctx, nft_trans_rule(trans));
9198                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
9199                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
9200
9201                         nft_trans_destroy(trans);
9202                         break;
9203                 case NFT_MSG_NEWSET:
9204                         trans->ctx.table->use--;
9205                         if (nft_trans_set_bound(trans)) {
9206                                 nft_trans_destroy(trans);
9207                                 break;
9208                         }
9209                         list_del_rcu(&nft_trans_set(trans)->list);
9210                         break;
9211                 case NFT_MSG_DELSET:
9212                         trans->ctx.table->use++;
9213                         nft_clear(trans->ctx.net, nft_trans_set(trans));
9214                         nft_trans_destroy(trans);
9215                         break;
9216                 case NFT_MSG_NEWSETELEM:
9217                         if (nft_trans_elem_set_bound(trans)) {
9218                                 nft_trans_destroy(trans);
9219                                 break;
9220                         }
9221                         te = (struct nft_trans_elem *)trans->data;
9222                         nft_setelem_remove(net, te->set, &te->elem);
9223                         if (!nft_setelem_is_catchall(te->set, &te->elem))
9224                                 atomic_dec(&te->set->nelems);
9225                         break;
9226                 case NFT_MSG_DELSETELEM:
9227                         te = (struct nft_trans_elem *)trans->data;
9228
9229                         nft_setelem_data_activate(net, te->set, &te->elem);
9230                         nft_setelem_activate(net, te->set, &te->elem);
9231                         if (!nft_setelem_is_catchall(te->set, &te->elem))
9232                                 te->set->ndeact--;
9233
9234                         nft_trans_destroy(trans);
9235                         break;
9236                 case NFT_MSG_NEWOBJ:
9237                         if (nft_trans_obj_update(trans)) {
9238                                 nft_obj_destroy(&trans->ctx, nft_trans_obj_newobj(trans));
9239                                 nft_trans_destroy(trans);
9240                         } else {
9241                                 trans->ctx.table->use--;
9242                                 nft_obj_del(nft_trans_obj(trans));
9243                         }
9244                         break;
9245                 case NFT_MSG_DELOBJ:
9246                         trans->ctx.table->use++;
9247                         nft_clear(trans->ctx.net, nft_trans_obj(trans));
9248                         nft_trans_destroy(trans);
9249                         break;
9250                 case NFT_MSG_NEWFLOWTABLE:
9251                         if (nft_trans_flowtable_update(trans)) {
9252                                 nft_unregister_flowtable_net_hooks(net,
9253                                                 &nft_trans_flowtable_hooks(trans));
9254                         } else {
9255                                 trans->ctx.table->use--;
9256                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
9257                                 nft_unregister_flowtable_net_hooks(net,
9258                                                 &nft_trans_flowtable(trans)->hook_list);
9259                         }
9260                         break;
9261                 case NFT_MSG_DELFLOWTABLE:
9262                         if (nft_trans_flowtable_update(trans)) {
9263                                 list_splice(&nft_trans_flowtable_hooks(trans),
9264                                             &nft_trans_flowtable(trans)->hook_list);
9265                         } else {
9266                                 trans->ctx.table->use++;
9267                                 nft_clear(trans->ctx.net, nft_trans_flowtable(trans));
9268                         }
9269                         nft_trans_destroy(trans);
9270                         break;
9271                 }
9272         }
9273
9274         synchronize_rcu();
9275
9276         list_for_each_entry_safe_reverse(trans, next,
9277                                          &nft_net->commit_list, list) {
9278                 list_del(&trans->list);
9279                 nf_tables_abort_release(trans);
9280         }
9281
9282         if (action == NFNL_ABORT_AUTOLOAD)
9283                 nf_tables_module_autoload(net);
9284         else
9285                 nf_tables_module_autoload_cleanup(net);
9286
9287         return 0;
9288 }
9289
9290 static void nf_tables_cleanup(struct net *net)
9291 {
9292         nft_validate_state_update(net, NFT_VALIDATE_SKIP);
9293 }
9294
9295 static int nf_tables_abort(struct net *net, struct sk_buff *skb,
9296                            enum nfnl_abort_action action)
9297 {
9298         struct nftables_pernet *nft_net = nft_pernet(net);
9299         int ret = __nf_tables_abort(net, action);
9300
9301         mutex_unlock(&nft_net->commit_mutex);
9302
9303         return ret;
9304 }
9305
9306 static bool nf_tables_valid_genid(struct net *net, u32 genid)
9307 {
9308         struct nftables_pernet *nft_net = nft_pernet(net);
9309         bool genid_ok;
9310
9311         mutex_lock(&nft_net->commit_mutex);
9312
9313         genid_ok = genid == 0 || nft_net->base_seq == genid;
9314         if (!genid_ok)
9315                 mutex_unlock(&nft_net->commit_mutex);
9316
9317         /* else, commit mutex has to be released by commit or abort function */
9318         return genid_ok;
9319 }
9320
9321 static const struct nfnetlink_subsystem nf_tables_subsys = {
9322         .name           = "nf_tables",
9323         .subsys_id      = NFNL_SUBSYS_NFTABLES,
9324         .cb_count       = NFT_MSG_MAX,
9325         .cb             = nf_tables_cb,
9326         .commit         = nf_tables_commit,
9327         .abort          = nf_tables_abort,
9328         .cleanup        = nf_tables_cleanup,
9329         .valid_genid    = nf_tables_valid_genid,
9330         .owner          = THIS_MODULE,
9331 };
9332
9333 int nft_chain_validate_dependency(const struct nft_chain *chain,
9334                                   enum nft_chain_types type)
9335 {
9336         const struct nft_base_chain *basechain;
9337
9338         if (nft_is_base_chain(chain)) {
9339                 basechain = nft_base_chain(chain);
9340                 if (basechain->type->type != type)
9341                         return -EOPNOTSUPP;
9342         }
9343         return 0;
9344 }
9345 EXPORT_SYMBOL_GPL(nft_chain_validate_dependency);
9346
9347 int nft_chain_validate_hooks(const struct nft_chain *chain,
9348                              unsigned int hook_flags)
9349 {
9350         struct nft_base_chain *basechain;
9351
9352         if (nft_is_base_chain(chain)) {
9353                 basechain = nft_base_chain(chain);
9354
9355                 if ((1 << basechain->ops.hooknum) & hook_flags)
9356                         return 0;
9357
9358                 return -EOPNOTSUPP;
9359         }
9360
9361         return 0;
9362 }
9363 EXPORT_SYMBOL_GPL(nft_chain_validate_hooks);
9364
9365 /*
9366  * Loop detection - walk through the ruleset beginning at the destination chain
9367  * of a new jump until either the source chain is reached (loop) or all
9368  * reachable chains have been traversed.
9369  *
9370  * The loop check is performed whenever a new jump verdict is added to an
9371  * expression or verdict map or a verdict map is bound to a new chain.
9372  */
9373
9374 static int nf_tables_check_loops(const struct nft_ctx *ctx,
9375                                  const struct nft_chain *chain);
9376
9377 static int nft_check_loops(const struct nft_ctx *ctx,
9378                            const struct nft_set_ext *ext)
9379 {
9380         const struct nft_data *data;
9381         int ret;
9382
9383         data = nft_set_ext_data(ext);
9384         switch (data->verdict.code) {
9385         case NFT_JUMP:
9386         case NFT_GOTO:
9387                 ret = nf_tables_check_loops(ctx, data->verdict.chain);
9388                 break;
9389         default:
9390                 ret = 0;
9391                 break;
9392         }
9393
9394         return ret;
9395 }
9396
9397 static int nf_tables_loop_check_setelem(const struct nft_ctx *ctx,
9398                                         struct nft_set *set,
9399                                         const struct nft_set_iter *iter,
9400                                         struct nft_set_elem *elem)
9401 {
9402         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
9403
9404         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
9405             *nft_set_ext_flags(ext) & NFT_SET_ELEM_INTERVAL_END)
9406                 return 0;
9407
9408         return nft_check_loops(ctx, ext);
9409 }
9410
9411 static int nft_set_catchall_loops(const struct nft_ctx *ctx,
9412                                   struct nft_set *set)
9413 {
9414         u8 genmask = nft_genmask_next(ctx->net);
9415         struct nft_set_elem_catchall *catchall;
9416         struct nft_set_ext *ext;
9417         int ret = 0;
9418
9419         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
9420                 ext = nft_set_elem_ext(set, catchall->elem);
9421                 if (!nft_set_elem_active(ext, genmask))
9422                         continue;
9423
9424                 ret = nft_check_loops(ctx, ext);
9425                 if (ret < 0)
9426                         return ret;
9427         }
9428
9429         return ret;
9430 }
9431
9432 static int nf_tables_check_loops(const struct nft_ctx *ctx,
9433                                  const struct nft_chain *chain)
9434 {
9435         const struct nft_rule *rule;
9436         const struct nft_expr *expr, *last;
9437         struct nft_set *set;
9438         struct nft_set_binding *binding;
9439         struct nft_set_iter iter;
9440
9441         if (ctx->chain == chain)
9442                 return -ELOOP;
9443
9444         list_for_each_entry(rule, &chain->rules, list) {
9445                 nft_rule_for_each_expr(expr, last, rule) {
9446                         struct nft_immediate_expr *priv;
9447                         const struct nft_data *data;
9448                         int err;
9449
9450                         if (strcmp(expr->ops->type->name, "immediate"))
9451                                 continue;
9452
9453                         priv = nft_expr_priv(expr);
9454                         if (priv->dreg != NFT_REG_VERDICT)
9455                                 continue;
9456
9457                         data = &priv->data;
9458                         switch (data->verdict.code) {
9459                         case NFT_JUMP:
9460                         case NFT_GOTO:
9461                                 err = nf_tables_check_loops(ctx,
9462                                                         data->verdict.chain);
9463                                 if (err < 0)
9464                                         return err;
9465                                 break;
9466                         default:
9467                                 break;
9468                         }
9469                 }
9470         }
9471
9472         list_for_each_entry(set, &ctx->table->sets, list) {
9473                 if (!nft_is_active_next(ctx->net, set))
9474                         continue;
9475                 if (!(set->flags & NFT_SET_MAP) ||
9476                     set->dtype != NFT_DATA_VERDICT)
9477                         continue;
9478
9479                 list_for_each_entry(binding, &set->bindings, list) {
9480                         if (!(binding->flags & NFT_SET_MAP) ||
9481                             binding->chain != chain)
9482                                 continue;
9483
9484                         iter.genmask    = nft_genmask_next(ctx->net);
9485                         iter.skip       = 0;
9486                         iter.count      = 0;
9487                         iter.err        = 0;
9488                         iter.fn         = nf_tables_loop_check_setelem;
9489
9490                         set->ops->walk(ctx, set, &iter);
9491                         if (!iter.err)
9492                                 iter.err = nft_set_catchall_loops(ctx, set);
9493
9494                         if (iter.err < 0)
9495                                 return iter.err;
9496                 }
9497         }
9498
9499         return 0;
9500 }
9501
9502 /**
9503  *      nft_parse_u32_check - fetch u32 attribute and check for maximum value
9504  *
9505  *      @attr: netlink attribute to fetch value from
9506  *      @max: maximum value to be stored in dest
9507  *      @dest: pointer to the variable
9508  *
9509  *      Parse, check and store a given u32 netlink attribute into variable.
9510  *      This function returns -ERANGE if the value goes over maximum value.
9511  *      Otherwise a 0 is returned and the attribute value is stored in the
9512  *      destination variable.
9513  */
9514 int nft_parse_u32_check(const struct nlattr *attr, int max, u32 *dest)
9515 {
9516         u32 val;
9517
9518         val = ntohl(nla_get_be32(attr));
9519         if (val > max)
9520                 return -ERANGE;
9521
9522         *dest = val;
9523         return 0;
9524 }
9525 EXPORT_SYMBOL_GPL(nft_parse_u32_check);
9526
9527 static int nft_parse_register(const struct nlattr *attr, u32 *preg)
9528 {
9529         unsigned int reg;
9530
9531         reg = ntohl(nla_get_be32(attr));
9532         switch (reg) {
9533         case NFT_REG_VERDICT...NFT_REG_4:
9534                 *preg = reg * NFT_REG_SIZE / NFT_REG32_SIZE;
9535                 break;
9536         case NFT_REG32_00...NFT_REG32_15:
9537                 *preg = reg + NFT_REG_SIZE / NFT_REG32_SIZE - NFT_REG32_00;
9538                 break;
9539         default:
9540                 return -ERANGE;
9541         }
9542
9543         return 0;
9544 }
9545
9546 /**
9547  *      nft_dump_register - dump a register value to a netlink attribute
9548  *
9549  *      @skb: socket buffer
9550  *      @attr: attribute number
9551  *      @reg: register number
9552  *
9553  *      Construct a netlink attribute containing the register number. For
9554  *      compatibility reasons, register numbers being a multiple of 4 are
9555  *      translated to the corresponding 128 bit register numbers.
9556  */
9557 int nft_dump_register(struct sk_buff *skb, unsigned int attr, unsigned int reg)
9558 {
9559         if (reg % (NFT_REG_SIZE / NFT_REG32_SIZE) == 0)
9560                 reg = reg / (NFT_REG_SIZE / NFT_REG32_SIZE);
9561         else
9562                 reg = reg - NFT_REG_SIZE / NFT_REG32_SIZE + NFT_REG32_00;
9563
9564         return nla_put_be32(skb, attr, htonl(reg));
9565 }
9566 EXPORT_SYMBOL_GPL(nft_dump_register);
9567
9568 static int nft_validate_register_load(enum nft_registers reg, unsigned int len)
9569 {
9570         if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
9571                 return -EINVAL;
9572         if (len == 0)
9573                 return -EINVAL;
9574         if (reg * NFT_REG32_SIZE + len > sizeof_field(struct nft_regs, data))
9575                 return -ERANGE;
9576
9577         return 0;
9578 }
9579
9580 int nft_parse_register_load(const struct nlattr *attr, u8 *sreg, u32 len)
9581 {
9582         u32 reg;
9583         int err;
9584
9585         err = nft_parse_register(attr, &reg);
9586         if (err < 0)
9587                 return err;
9588
9589         err = nft_validate_register_load(reg, len);
9590         if (err < 0)
9591                 return err;
9592
9593         *sreg = reg;
9594         return 0;
9595 }
9596 EXPORT_SYMBOL_GPL(nft_parse_register_load);
9597
9598 static int nft_validate_register_store(const struct nft_ctx *ctx,
9599                                        enum nft_registers reg,
9600                                        const struct nft_data *data,
9601                                        enum nft_data_types type,
9602                                        unsigned int len)
9603 {
9604         int err;
9605
9606         switch (reg) {
9607         case NFT_REG_VERDICT:
9608                 if (type != NFT_DATA_VERDICT)
9609                         return -EINVAL;
9610
9611                 if (data != NULL &&
9612                     (data->verdict.code == NFT_GOTO ||
9613                      data->verdict.code == NFT_JUMP)) {
9614                         err = nf_tables_check_loops(ctx, data->verdict.chain);
9615                         if (err < 0)
9616                                 return err;
9617                 }
9618
9619                 return 0;
9620         default:
9621                 if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
9622                         return -EINVAL;
9623                 if (len == 0)
9624                         return -EINVAL;
9625                 if (reg * NFT_REG32_SIZE + len >
9626                     sizeof_field(struct nft_regs, data))
9627                         return -ERANGE;
9628
9629                 if (data != NULL && type != NFT_DATA_VALUE)
9630                         return -EINVAL;
9631                 return 0;
9632         }
9633 }
9634
9635 int nft_parse_register_store(const struct nft_ctx *ctx,
9636                              const struct nlattr *attr, u8 *dreg,
9637                              const struct nft_data *data,
9638                              enum nft_data_types type, unsigned int len)
9639 {
9640         int err;
9641         u32 reg;
9642
9643         err = nft_parse_register(attr, &reg);
9644         if (err < 0)
9645                 return err;
9646
9647         err = nft_validate_register_store(ctx, reg, data, type, len);
9648         if (err < 0)
9649                 return err;
9650
9651         *dreg = reg;
9652         return 0;
9653 }
9654 EXPORT_SYMBOL_GPL(nft_parse_register_store);
9655
9656 static const struct nla_policy nft_verdict_policy[NFTA_VERDICT_MAX + 1] = {
9657         [NFTA_VERDICT_CODE]     = { .type = NLA_U32 },
9658         [NFTA_VERDICT_CHAIN]    = { .type = NLA_STRING,
9659                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
9660         [NFTA_VERDICT_CHAIN_ID] = { .type = NLA_U32 },
9661 };
9662
9663 static int nft_verdict_init(const struct nft_ctx *ctx, struct nft_data *data,
9664                             struct nft_data_desc *desc, const struct nlattr *nla)
9665 {
9666         u8 genmask = nft_genmask_next(ctx->net);
9667         struct nlattr *tb[NFTA_VERDICT_MAX + 1];
9668         struct nft_chain *chain;
9669         int err;
9670
9671         err = nla_parse_nested_deprecated(tb, NFTA_VERDICT_MAX, nla,
9672                                           nft_verdict_policy, NULL);
9673         if (err < 0)
9674                 return err;
9675
9676         if (!tb[NFTA_VERDICT_CODE])
9677                 return -EINVAL;
9678         data->verdict.code = ntohl(nla_get_be32(tb[NFTA_VERDICT_CODE]));
9679
9680         switch (data->verdict.code) {
9681         default:
9682                 switch (data->verdict.code & NF_VERDICT_MASK) {
9683                 case NF_ACCEPT:
9684                 case NF_DROP:
9685                 case NF_QUEUE:
9686                         break;
9687                 default:
9688                         return -EINVAL;
9689                 }
9690                 fallthrough;
9691         case NFT_CONTINUE:
9692         case NFT_BREAK:
9693         case NFT_RETURN:
9694                 break;
9695         case NFT_JUMP:
9696         case NFT_GOTO:
9697                 if (tb[NFTA_VERDICT_CHAIN]) {
9698                         chain = nft_chain_lookup(ctx->net, ctx->table,
9699                                                  tb[NFTA_VERDICT_CHAIN],
9700                                                  genmask);
9701                 } else if (tb[NFTA_VERDICT_CHAIN_ID]) {
9702                         chain = nft_chain_lookup_byid(ctx->net, ctx->table,
9703                                                       tb[NFTA_VERDICT_CHAIN_ID]);
9704                         if (IS_ERR(chain))
9705                                 return PTR_ERR(chain);
9706                 } else {
9707                         return -EINVAL;
9708                 }
9709
9710                 if (IS_ERR(chain))
9711                         return PTR_ERR(chain);
9712                 if (nft_is_base_chain(chain))
9713                         return -EOPNOTSUPP;
9714                 if (desc->flags & NFT_DATA_DESC_SETELEM &&
9715                     chain->flags & NFT_CHAIN_BINDING)
9716                         return -EINVAL;
9717
9718                 chain->use++;
9719                 data->verdict.chain = chain;
9720                 break;
9721         }
9722
9723         desc->len = sizeof(data->verdict);
9724
9725         return 0;
9726 }
9727
9728 static void nft_verdict_uninit(const struct nft_data *data)
9729 {
9730         struct nft_chain *chain;
9731         struct nft_rule *rule;
9732
9733         switch (data->verdict.code) {
9734         case NFT_JUMP:
9735         case NFT_GOTO:
9736                 chain = data->verdict.chain;
9737                 chain->use--;
9738
9739                 if (!nft_chain_is_bound(chain))
9740                         break;
9741
9742                 chain->table->use--;
9743                 list_for_each_entry(rule, &chain->rules, list)
9744                         chain->use--;
9745
9746                 nft_chain_del(chain);
9747                 break;
9748         }
9749 }
9750
9751 int nft_verdict_dump(struct sk_buff *skb, int type, const struct nft_verdict *v)
9752 {
9753         struct nlattr *nest;
9754
9755         nest = nla_nest_start_noflag(skb, type);
9756         if (!nest)
9757                 goto nla_put_failure;
9758
9759         if (nla_put_be32(skb, NFTA_VERDICT_CODE, htonl(v->code)))
9760                 goto nla_put_failure;
9761
9762         switch (v->code) {
9763         case NFT_JUMP:
9764         case NFT_GOTO:
9765                 if (nla_put_string(skb, NFTA_VERDICT_CHAIN,
9766                                    v->chain->name))
9767                         goto nla_put_failure;
9768         }
9769         nla_nest_end(skb, nest);
9770         return 0;
9771
9772 nla_put_failure:
9773         return -1;
9774 }
9775
9776 static int nft_value_init(const struct nft_ctx *ctx,
9777                           struct nft_data *data, struct nft_data_desc *desc,
9778                           const struct nlattr *nla)
9779 {
9780         unsigned int len;
9781
9782         len = nla_len(nla);
9783         if (len == 0)
9784                 return -EINVAL;
9785         if (len > desc->size)
9786                 return -EOVERFLOW;
9787         if (desc->len) {
9788                 if (len != desc->len)
9789                         return -EINVAL;
9790         } else {
9791                 desc->len = len;
9792         }
9793
9794         nla_memcpy(data->data, nla, len);
9795
9796         return 0;
9797 }
9798
9799 static int nft_value_dump(struct sk_buff *skb, const struct nft_data *data,
9800                           unsigned int len)
9801 {
9802         return nla_put(skb, NFTA_DATA_VALUE, len, data->data);
9803 }
9804
9805 static const struct nla_policy nft_data_policy[NFTA_DATA_MAX + 1] = {
9806         [NFTA_DATA_VALUE]       = { .type = NLA_BINARY },
9807         [NFTA_DATA_VERDICT]     = { .type = NLA_NESTED },
9808 };
9809
9810 /**
9811  *      nft_data_init - parse nf_tables data netlink attributes
9812  *
9813  *      @ctx: context of the expression using the data
9814  *      @data: destination struct nft_data
9815  *      @desc: data description
9816  *      @nla: netlink attribute containing data
9817  *
9818  *      Parse the netlink data attributes and initialize a struct nft_data.
9819  *      The type and length of data are returned in the data description.
9820  *
9821  *      The caller can indicate that it only wants to accept data of type
9822  *      NFT_DATA_VALUE by passing NULL for the ctx argument.
9823  */
9824 int nft_data_init(const struct nft_ctx *ctx, struct nft_data *data,
9825                   struct nft_data_desc *desc, const struct nlattr *nla)
9826 {
9827         struct nlattr *tb[NFTA_DATA_MAX + 1];
9828         int err;
9829
9830         if (WARN_ON_ONCE(!desc->size))
9831                 return -EINVAL;
9832
9833         err = nla_parse_nested_deprecated(tb, NFTA_DATA_MAX, nla,
9834                                           nft_data_policy, NULL);
9835         if (err < 0)
9836                 return err;
9837
9838         if (tb[NFTA_DATA_VALUE]) {
9839                 if (desc->type != NFT_DATA_VALUE)
9840                         return -EINVAL;
9841
9842                 err = nft_value_init(ctx, data, desc, tb[NFTA_DATA_VALUE]);
9843         } else if (tb[NFTA_DATA_VERDICT] && ctx != NULL) {
9844                 if (desc->type != NFT_DATA_VERDICT)
9845                         return -EINVAL;
9846
9847                 err = nft_verdict_init(ctx, data, desc, tb[NFTA_DATA_VERDICT]);
9848         } else {
9849                 err = -EINVAL;
9850         }
9851
9852         return err;
9853 }
9854 EXPORT_SYMBOL_GPL(nft_data_init);
9855
9856 /**
9857  *      nft_data_release - release a nft_data item
9858  *
9859  *      @data: struct nft_data to release
9860  *      @type: type of data
9861  *
9862  *      Release a nft_data item. NFT_DATA_VALUE types can be silently discarded,
9863  *      all others need to be released by calling this function.
9864  */
9865 void nft_data_release(const struct nft_data *data, enum nft_data_types type)
9866 {
9867         if (type < NFT_DATA_VERDICT)
9868                 return;
9869         switch (type) {
9870         case NFT_DATA_VERDICT:
9871                 return nft_verdict_uninit(data);
9872         default:
9873                 WARN_ON(1);
9874         }
9875 }
9876 EXPORT_SYMBOL_GPL(nft_data_release);
9877
9878 int nft_data_dump(struct sk_buff *skb, int attr, const struct nft_data *data,
9879                   enum nft_data_types type, unsigned int len)
9880 {
9881         struct nlattr *nest;
9882         int err;
9883
9884         nest = nla_nest_start_noflag(skb, attr);
9885         if (nest == NULL)
9886                 return -1;
9887
9888         switch (type) {
9889         case NFT_DATA_VALUE:
9890                 err = nft_value_dump(skb, data, len);
9891                 break;
9892         case NFT_DATA_VERDICT:
9893                 err = nft_verdict_dump(skb, NFTA_DATA_VERDICT, &data->verdict);
9894                 break;
9895         default:
9896                 err = -EINVAL;
9897                 WARN_ON(1);
9898         }
9899
9900         nla_nest_end(skb, nest);
9901         return err;
9902 }
9903 EXPORT_SYMBOL_GPL(nft_data_dump);
9904
9905 int __nft_release_basechain(struct nft_ctx *ctx)
9906 {
9907         struct nft_rule *rule, *nr;
9908
9909         if (WARN_ON(!nft_is_base_chain(ctx->chain)))
9910                 return 0;
9911
9912         nf_tables_unregister_hook(ctx->net, ctx->chain->table, ctx->chain);
9913         list_for_each_entry_safe(rule, nr, &ctx->chain->rules, list) {
9914                 list_del(&rule->list);
9915                 ctx->chain->use--;
9916                 nf_tables_rule_release(ctx, rule);
9917         }
9918         nft_chain_del(ctx->chain);
9919         ctx->table->use--;
9920         nf_tables_chain_destroy(ctx);
9921
9922         return 0;
9923 }
9924 EXPORT_SYMBOL_GPL(__nft_release_basechain);
9925
9926 static void __nft_release_hook(struct net *net, struct nft_table *table)
9927 {
9928         struct nft_flowtable *flowtable;
9929         struct nft_chain *chain;
9930
9931         list_for_each_entry(chain, &table->chains, list)
9932                 __nf_tables_unregister_hook(net, table, chain, true);
9933         list_for_each_entry(flowtable, &table->flowtables, list)
9934                 __nft_unregister_flowtable_net_hooks(net, &flowtable->hook_list,
9935                                                      true);
9936 }
9937
9938 static void __nft_release_hooks(struct net *net)
9939 {
9940         struct nftables_pernet *nft_net = nft_pernet(net);
9941         struct nft_table *table;
9942
9943         list_for_each_entry(table, &nft_net->tables, list) {
9944                 if (nft_table_has_owner(table))
9945                         continue;
9946
9947                 __nft_release_hook(net, table);
9948         }
9949 }
9950
9951 static void __nft_release_table(struct net *net, struct nft_table *table)
9952 {
9953         struct nft_flowtable *flowtable, *nf;
9954         struct nft_chain *chain, *nc;
9955         struct nft_object *obj, *ne;
9956         struct nft_rule *rule, *nr;
9957         struct nft_set *set, *ns;
9958         struct nft_ctx ctx = {
9959                 .net    = net,
9960                 .family = NFPROTO_NETDEV,
9961         };
9962
9963         ctx.family = table->family;
9964         ctx.table = table;
9965         list_for_each_entry(chain, &table->chains, list) {
9966                 ctx.chain = chain;
9967                 list_for_each_entry_safe(rule, nr, &chain->rules, list) {
9968                         list_del(&rule->list);
9969                         chain->use--;
9970                         nf_tables_rule_release(&ctx, rule);
9971                 }
9972         }
9973         list_for_each_entry_safe(flowtable, nf, &table->flowtables, list) {
9974                 list_del(&flowtable->list);
9975                 table->use--;
9976                 nf_tables_flowtable_destroy(flowtable);
9977         }
9978         list_for_each_entry_safe(set, ns, &table->sets, list) {
9979                 list_del(&set->list);
9980                 table->use--;
9981                 nft_set_destroy(&ctx, set);
9982         }
9983         list_for_each_entry_safe(obj, ne, &table->objects, list) {
9984                 nft_obj_del(obj);
9985                 table->use--;
9986                 nft_obj_destroy(&ctx, obj);
9987         }
9988         list_for_each_entry_safe(chain, nc, &table->chains, list) {
9989                 ctx.chain = chain;
9990                 nft_chain_del(chain);
9991                 table->use--;
9992                 nf_tables_chain_destroy(&ctx);
9993         }
9994         nf_tables_table_destroy(&ctx);
9995 }
9996
9997 static void __nft_release_tables(struct net *net)
9998 {
9999         struct nftables_pernet *nft_net = nft_pernet(net);
10000         struct nft_table *table, *nt;
10001
10002         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
10003                 if (nft_table_has_owner(table))
10004                         continue;
10005
10006                 list_del(&table->list);
10007
10008                 __nft_release_table(net, table);
10009         }
10010 }
10011
10012 static int nft_rcv_nl_event(struct notifier_block *this, unsigned long event,
10013                             void *ptr)
10014 {
10015         struct nft_table *table, *to_delete[8];
10016         struct nftables_pernet *nft_net;
10017         struct netlink_notify *n = ptr;
10018         struct net *net = n->net;
10019         unsigned int deleted;
10020         bool restart = false;
10021
10022         if (event != NETLINK_URELEASE || n->protocol != NETLINK_NETFILTER)
10023                 return NOTIFY_DONE;
10024
10025         nft_net = nft_pernet(net);
10026         deleted = 0;
10027         mutex_lock(&nft_net->commit_mutex);
10028 again:
10029         list_for_each_entry(table, &nft_net->tables, list) {
10030                 if (nft_table_has_owner(table) &&
10031                     n->portid == table->nlpid) {
10032                         __nft_release_hook(net, table);
10033                         list_del_rcu(&table->list);
10034                         to_delete[deleted++] = table;
10035                         if (deleted >= ARRAY_SIZE(to_delete))
10036                                 break;
10037                 }
10038         }
10039         if (deleted) {
10040                 restart = deleted >= ARRAY_SIZE(to_delete);
10041                 synchronize_rcu();
10042                 while (deleted)
10043                         __nft_release_table(net, to_delete[--deleted]);
10044
10045                 if (restart)
10046                         goto again;
10047         }
10048         mutex_unlock(&nft_net->commit_mutex);
10049
10050         return NOTIFY_DONE;
10051 }
10052
10053 static struct notifier_block nft_nl_notifier = {
10054         .notifier_call  = nft_rcv_nl_event,
10055 };
10056
10057 static int __net_init nf_tables_init_net(struct net *net)
10058 {
10059         struct nftables_pernet *nft_net = nft_pernet(net);
10060
10061         INIT_LIST_HEAD(&nft_net->tables);
10062         INIT_LIST_HEAD(&nft_net->commit_list);
10063         INIT_LIST_HEAD(&nft_net->module_list);
10064         INIT_LIST_HEAD(&nft_net->notify_list);
10065         mutex_init(&nft_net->commit_mutex);
10066         nft_net->base_seq = 1;
10067         nft_net->validate_state = NFT_VALIDATE_SKIP;
10068
10069         return 0;
10070 }
10071
10072 static void __net_exit nf_tables_pre_exit_net(struct net *net)
10073 {
10074         struct nftables_pernet *nft_net = nft_pernet(net);
10075
10076         mutex_lock(&nft_net->commit_mutex);
10077         __nft_release_hooks(net);
10078         mutex_unlock(&nft_net->commit_mutex);
10079 }
10080
10081 static void __net_exit nf_tables_exit_net(struct net *net)
10082 {
10083         struct nftables_pernet *nft_net = nft_pernet(net);
10084
10085         mutex_lock(&nft_net->commit_mutex);
10086         if (!list_empty(&nft_net->commit_list))
10087                 __nf_tables_abort(net, NFNL_ABORT_NONE);
10088         __nft_release_tables(net);
10089         mutex_unlock(&nft_net->commit_mutex);
10090         WARN_ON_ONCE(!list_empty(&nft_net->tables));
10091         WARN_ON_ONCE(!list_empty(&nft_net->module_list));
10092         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
10093 }
10094
10095 static struct pernet_operations nf_tables_net_ops = {
10096         .init           = nf_tables_init_net,
10097         .pre_exit       = nf_tables_pre_exit_net,
10098         .exit           = nf_tables_exit_net,
10099         .id             = &nf_tables_net_id,
10100         .size           = sizeof(struct nftables_pernet),
10101 };
10102
10103 static int __init nf_tables_module_init(void)
10104 {
10105         int err;
10106
10107         err = register_pernet_subsys(&nf_tables_net_ops);
10108         if (err < 0)
10109                 return err;
10110
10111         err = nft_chain_filter_init();
10112         if (err < 0)
10113                 goto err_chain_filter;
10114
10115         err = nf_tables_core_module_init();
10116         if (err < 0)
10117                 goto err_core_module;
10118
10119         err = register_netdevice_notifier(&nf_tables_flowtable_notifier);
10120         if (err < 0)
10121                 goto err_netdev_notifier;
10122
10123         err = rhltable_init(&nft_objname_ht, &nft_objname_ht_params);
10124         if (err < 0)
10125                 goto err_rht_objname;
10126
10127         err = nft_offload_init();
10128         if (err < 0)
10129                 goto err_offload;
10130
10131         err = netlink_register_notifier(&nft_nl_notifier);
10132         if (err < 0)
10133                 goto err_netlink_notifier;
10134
10135         /* must be last */
10136         err = nfnetlink_subsys_register(&nf_tables_subsys);
10137         if (err < 0)
10138                 goto err_nfnl_subsys;
10139
10140         nft_chain_route_init();
10141
10142         return err;
10143
10144 err_nfnl_subsys:
10145         netlink_unregister_notifier(&nft_nl_notifier);
10146 err_netlink_notifier:
10147         nft_offload_exit();
10148 err_offload:
10149         rhltable_destroy(&nft_objname_ht);
10150 err_rht_objname:
10151         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
10152 err_netdev_notifier:
10153         nf_tables_core_module_exit();
10154 err_core_module:
10155         nft_chain_filter_fini();
10156 err_chain_filter:
10157         unregister_pernet_subsys(&nf_tables_net_ops);
10158         return err;
10159 }
10160
10161 static void __exit nf_tables_module_exit(void)
10162 {
10163         nfnetlink_subsys_unregister(&nf_tables_subsys);
10164         netlink_unregister_notifier(&nft_nl_notifier);
10165         nft_offload_exit();
10166         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
10167         nft_chain_filter_fini();
10168         nft_chain_route_fini();
10169         unregister_pernet_subsys(&nf_tables_net_ops);
10170         cancel_work_sync(&trans_destroy_work);
10171         rcu_barrier();
10172         rhltable_destroy(&nft_objname_ht);
10173         nf_tables_core_module_exit();
10174 }
10175
10176 module_init(nf_tables_module_init);
10177 module_exit(nf_tables_module_exit);
10178
10179 MODULE_LICENSE("GPL");
10180 MODULE_AUTHOR("Patrick McHardy <kaber@trash.net>");
10181 MODULE_ALIAS_NFNL_SUBSYS(NFNL_SUBSYS_NFTABLES);