net/netfilter/nf_tables_api.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2007-2009 Patrick McHardy <kaber@trash.net>
   4  *
   5  * Development of this code funded by Astaro AG (http://www.astaro.com/)
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/init.h>
  10 #include <linux/list.h>
  11 #include <linux/skbuff.h>
  12 #include <linux/netlink.h>
  13 #include <linux/vmalloc.h>
  14 #include <linux/rhashtable.h>
  15 #include <linux/audit.h>
  16 #include <linux/netfilter.h>
  17 #include <linux/netfilter/nfnetlink.h>
  18 #include <linux/netfilter/nf_tables.h>
  19 #include <net/netfilter/nf_flow_table.h>
  20 #include <net/netfilter/nf_tables_core.h>
  21 #include <net/netfilter/nf_tables.h>
  22 #include <net/netfilter/nf_tables_offload.h>
  23 #include <net/net_namespace.h>
  24 #include <net/sock.h>
  25
  26 #define NFT_MODULE_AUTOLOAD_LIMIT (MODULE_NAME_LEN - sizeof("nft-expr-255-"))
  27
  28 unsigned int nf_tables_net_id __read_mostly;
  29
  30 static LIST_HEAD(nf_tables_expressions);
  31 static LIST_HEAD(nf_tables_objects);
  32 static LIST_HEAD(nf_tables_flowtables);
  33 static LIST_HEAD(nf_tables_destroy_list);
  34 static LIST_HEAD(nf_tables_gc_list);
  35 static DEFINE_SPINLOCK(nf_tables_destroy_list_lock);
  36 static DEFINE_SPINLOCK(nf_tables_gc_list_lock);
  37
  38 enum {
  39         NFT_VALIDATE_SKIP       = 0,
  40         NFT_VALIDATE_NEED,
  41         NFT_VALIDATE_DO,
  42 };
  43
  44 static struct rhltable nft_objname_ht;
  45
  46 static u32 nft_chain_hash(const void *data, u32 len, u32 seed);
  47 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed);
  48 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *, const void *);
  49
  50 static u32 nft_objname_hash(const void *data, u32 len, u32 seed);
  51 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed);
  52 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *, const void *);
  53
  54 static const struct rhashtable_params nft_chain_ht_params = {
  55         .head_offset            = offsetof(struct nft_chain, rhlhead),
  56         .key_offset             = offsetof(struct nft_chain, name),
  57         .hashfn                 = nft_chain_hash,
  58         .obj_hashfn             = nft_chain_hash_obj,
  59         .obj_cmpfn              = nft_chain_hash_cmp,
  60         .automatic_shrinking    = true,
  61 };
  62
  63 static const struct rhashtable_params nft_objname_ht_params = {
  64         .head_offset            = offsetof(struct nft_object, rhlhead),
  65         .key_offset             = offsetof(struct nft_object, key),
  66         .hashfn                 = nft_objname_hash,
  67         .obj_hashfn             = nft_objname_hash_obj,
  68         .obj_cmpfn              = nft_objname_hash_cmp,
  69         .automatic_shrinking    = true,
  70 };
  71
  72 struct nft_audit_data {
  73         struct nft_table *table;
  74         int entries;
  75         int op;
  76         struct list_head list;
  77 };
  78
  79 static const u8 nft2audit_op[NFT_MSG_MAX] = { // enum nf_tables_msg_types
  80         [NFT_MSG_NEWTABLE]      = AUDIT_NFT_OP_TABLE_REGISTER,
  81         [NFT_MSG_GETTABLE]      = AUDIT_NFT_OP_INVALID,
  82         [NFT_MSG_DELTABLE]      = AUDIT_NFT_OP_TABLE_UNREGISTER,
  83         [NFT_MSG_NEWCHAIN]      = AUDIT_NFT_OP_CHAIN_REGISTER,
  84         [NFT_MSG_GETCHAIN]      = AUDIT_NFT_OP_INVALID,
  85         [NFT_MSG_DELCHAIN]      = AUDIT_NFT_OP_CHAIN_UNREGISTER,
  86         [NFT_MSG_NEWRULE]       = AUDIT_NFT_OP_RULE_REGISTER,
  87         [NFT_MSG_GETRULE]       = AUDIT_NFT_OP_INVALID,
  88         [NFT_MSG_DELRULE]       = AUDIT_NFT_OP_RULE_UNREGISTER,
  89         [NFT_MSG_NEWSET]        = AUDIT_NFT_OP_SET_REGISTER,
  90         [NFT_MSG_GETSET]        = AUDIT_NFT_OP_INVALID,
  91         [NFT_MSG_DELSET]        = AUDIT_NFT_OP_SET_UNREGISTER,
  92         [NFT_MSG_NEWSETELEM]    = AUDIT_NFT_OP_SETELEM_REGISTER,
  93         [NFT_MSG_GETSETELEM]    = AUDIT_NFT_OP_INVALID,
  94         [NFT_MSG_DELSETELEM]    = AUDIT_NFT_OP_SETELEM_UNREGISTER,
  95         [NFT_MSG_NEWGEN]        = AUDIT_NFT_OP_GEN_REGISTER,
  96         [NFT_MSG_GETGEN]        = AUDIT_NFT_OP_INVALID,
  97         [NFT_MSG_TRACE]         = AUDIT_NFT_OP_INVALID,
  98         [NFT_MSG_NEWOBJ]        = AUDIT_NFT_OP_OBJ_REGISTER,
  99         [NFT_MSG_GETOBJ]        = AUDIT_NFT_OP_INVALID,
 100         [NFT_MSG_DELOBJ]        = AUDIT_NFT_OP_OBJ_UNREGISTER,
 101         [NFT_MSG_GETOBJ_RESET]  = AUDIT_NFT_OP_OBJ_RESET,
 102         [NFT_MSG_NEWFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_REGISTER,
 103         [NFT_MSG_GETFLOWTABLE]  = AUDIT_NFT_OP_INVALID,
 104         [NFT_MSG_DELFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_UNREGISTER,
 105         [NFT_MSG_GETSETELEM_RESET] = AUDIT_NFT_OP_SETELEM_RESET,
 106 };
 107
 108 static void nft_validate_state_update(struct nft_table *table, u8 new_validate_state)
 109 {
 110         switch (table->validate_state) {
 111         case NFT_VALIDATE_SKIP:
 112                 WARN_ON_ONCE(new_validate_state == NFT_VALIDATE_DO);
 113                 break;
 114         case NFT_VALIDATE_NEED:
 115                 break;
 116         case NFT_VALIDATE_DO:
 117                 if (new_validate_state == NFT_VALIDATE_NEED)
 118                         return;
 119         }
 120
 121         table->validate_state = new_validate_state;
 122 }
 123 static void nf_tables_trans_destroy_work(struct work_struct *w);
 124 static DECLARE_WORK(trans_destroy_work, nf_tables_trans_destroy_work);
 125
 126 static void nft_trans_gc_work(struct work_struct *work);
 127 static DECLARE_WORK(trans_gc_work, nft_trans_gc_work);
 128
 129 static void nft_ctx_init(struct nft_ctx *ctx,
 130                          struct net *net,
 131                          const struct sk_buff *skb,
 132                          const struct nlmsghdr *nlh,
 133                          u8 family,
 134                          struct nft_table *table,
 135                          struct nft_chain *chain,
 136                          const struct nlattr * const *nla)
 137 {
 138         ctx->net        = net;
 139         ctx->family     = family;
 140         ctx->level      = 0;
 141         ctx->table      = table;
 142         ctx->chain      = chain;
 143         ctx->nla        = nla;
 144         ctx->portid     = NETLINK_CB(skb).portid;
 145         ctx->report     = nlmsg_report(nlh);
 146         ctx->flags      = nlh->nlmsg_flags;
 147         ctx->seq        = nlh->nlmsg_seq;
 148 }
 149
 150 static struct nft_trans *nft_trans_alloc_gfp(const struct nft_ctx *ctx,
 151                                              int msg_type, u32 size, gfp_t gfp)
 152 {
 153         struct nft_trans *trans;
 154
 155         trans = kzalloc(sizeof(struct nft_trans) + size, gfp);
 156         if (trans == NULL)
 157                 return NULL;
 158
 159         INIT_LIST_HEAD(&trans->list);
 160         INIT_LIST_HEAD(&trans->binding_list);
 161         trans->msg_type = msg_type;
 162         trans->ctx      = *ctx;
 163
 164         return trans;
 165 }
 166
 167 static struct nft_trans *nft_trans_alloc(const struct nft_ctx *ctx,
 168                                          int msg_type, u32 size)
 169 {
 170         return nft_trans_alloc_gfp(ctx, msg_type, size, GFP_KERNEL);
 171 }
 172
 173 static void nft_trans_list_del(struct nft_trans *trans)
 174 {
 175         list_del(&trans->list);
 176         list_del(&trans->binding_list);
 177 }
 178
 179 static void nft_trans_destroy(struct nft_trans *trans)
 180 {
 181         nft_trans_list_del(trans);
 182         kfree(trans);
 183 }
 184
 185 static void __nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set,
 186                                  bool bind)
 187 {
 188         struct nftables_pernet *nft_net;
 189         struct net *net = ctx->net;
 190         struct nft_trans *trans;
 191
 192         if (!nft_set_is_anonymous(set))
 193                 return;
 194
 195         nft_net = nft_pernet(net);
 196         list_for_each_entry_reverse(trans, &nft_net->commit_list, list) {
 197                 switch (trans->msg_type) {
 198                 case NFT_MSG_NEWSET:
 199                         if (nft_trans_set(trans) == set)
 200                                 nft_trans_set_bound(trans) = bind;
 201                         break;
 202                 case NFT_MSG_NEWSETELEM:
 203                         if (nft_trans_elem_set(trans) == set)
 204                                 nft_trans_elem_set_bound(trans) = bind;
 205                         break;
 206                 }
 207         }
 208 }
 209
 210 static void nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set)
 211 {
 212         return __nft_set_trans_bind(ctx, set, true);
 213 }
 214
 215 static void nft_set_trans_unbind(const struct nft_ctx *ctx, struct nft_set *set)
 216 {
 217         return __nft_set_trans_bind(ctx, set, false);
 218 }
 219
 220 static void __nft_chain_trans_bind(const struct nft_ctx *ctx,
 221                                    struct nft_chain *chain, bool bind)
 222 {
 223         struct nftables_pernet *nft_net;
 224         struct net *net = ctx->net;
 225         struct nft_trans *trans;
 226
 227         if (!nft_chain_binding(chain))
 228                 return;
 229
 230         nft_net = nft_pernet(net);
 231         list_for_each_entry_reverse(trans, &nft_net->commit_list, list) {
 232                 switch (trans->msg_type) {
 233                 case NFT_MSG_NEWCHAIN:
 234                         if (nft_trans_chain(trans) == chain)
 235                                 nft_trans_chain_bound(trans) = bind;
 236                         break;
 237                 case NFT_MSG_NEWRULE:
 238                         if (trans->ctx.chain == chain)
 239                                 nft_trans_rule_bound(trans) = bind;
 240                         break;
 241                 }
 242         }
 243 }
 244
 245 static void nft_chain_trans_bind(const struct nft_ctx *ctx,
 246                                  struct nft_chain *chain)
 247 {
 248         __nft_chain_trans_bind(ctx, chain, true);
 249 }
 250
 251 int nf_tables_bind_chain(const struct nft_ctx *ctx, struct nft_chain *chain)
 252 {
 253         if (!nft_chain_binding(chain))
 254                 return 0;
 255
 256         if (nft_chain_binding(ctx->chain))
 257                 return -EOPNOTSUPP;
 258
 259         if (chain->bound)
 260                 return -EBUSY;
 261
 262         if (!nft_use_inc(&chain->use))
 263                 return -EMFILE;
 264
 265         chain->bound = true;
 266         nft_chain_trans_bind(ctx, chain);
 267
 268         return 0;
 269 }
 270
 271 void nf_tables_unbind_chain(const struct nft_ctx *ctx, struct nft_chain *chain)
 272 {
 273         __nft_chain_trans_bind(ctx, chain, false);
 274 }
 275
 276 static int nft_netdev_register_hooks(struct net *net,
 277                                      struct list_head *hook_list)
 278 {
 279         struct nft_hook *hook;
 280         int err, j;
 281
 282         j = 0;
 283         list_for_each_entry(hook, hook_list, list) {
 284                 err = nf_register_net_hook(net, &hook->ops);
 285                 if (err < 0)
 286                         goto err_register;
 287
 288                 j++;
 289         }
 290         return 0;
 291
 292 err_register:
 293         list_for_each_entry(hook, hook_list, list) {
 294                 if (j-- <= 0)
 295                         break;
 296
 297                 nf_unregister_net_hook(net, &hook->ops);
 298         }
 299         return err;
 300 }
 301
 302 static void nft_netdev_unregister_hooks(struct net *net,
 303                                         struct list_head *hook_list,
 304                                         bool release_netdev)
 305 {
 306         struct nft_hook *hook, *next;
 307
 308         list_for_each_entry_safe(hook, next, hook_list, list) {
 309                 nf_unregister_net_hook(net, &hook->ops);
 310                 if (release_netdev) {
 311                         list_del(&hook->list);
 312                         kfree_rcu(hook, rcu);
 313                 }
 314         }
 315 }
 316
 317 static int nf_tables_register_hook(struct net *net,
 318                                    const struct nft_table *table,
 319                                    struct nft_chain *chain)
 320 {
 321         struct nft_base_chain *basechain;
 322         const struct nf_hook_ops *ops;
 323
 324         if (table->flags & NFT_TABLE_F_DORMANT ||
 325             !nft_is_base_chain(chain))
 326                 return 0;
 327
 328         basechain = nft_base_chain(chain);
 329         ops = &basechain->ops;
 330
 331         if (basechain->type->ops_register)
 332                 return basechain->type->ops_register(net, ops);
 333
 334         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 335                 return nft_netdev_register_hooks(net, &basechain->hook_list);
 336
 337         return nf_register_net_hook(net, &basechain->ops);
 338 }
 339
 340 static void __nf_tables_unregister_hook(struct net *net,
 341                                         const struct nft_table *table,
 342                                         struct nft_chain *chain,
 343                                         bool release_netdev)
 344 {
 345         struct nft_base_chain *basechain;
 346         const struct nf_hook_ops *ops;
 347
 348         if (table->flags & NFT_TABLE_F_DORMANT ||
 349             !nft_is_base_chain(chain))
 350                 return;
 351         basechain = nft_base_chain(chain);
 352         ops = &basechain->ops;
 353
 354         if (basechain->type->ops_unregister)
 355                 return basechain->type->ops_unregister(net, ops);
 356
 357         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 358                 nft_netdev_unregister_hooks(net, &basechain->hook_list,
 359                                             release_netdev);
 360         else
 361                 nf_unregister_net_hook(net, &basechain->ops);
 362 }
 363
 364 static void nf_tables_unregister_hook(struct net *net,
 365                                       const struct nft_table *table,
 366                                       struct nft_chain *chain)
 367 {
 368         return __nf_tables_unregister_hook(net, table, chain, false);
 369 }
 370
 371 static void nft_trans_commit_list_add_tail(struct net *net, struct nft_trans *trans)
 372 {
 373         struct nftables_pernet *nft_net = nft_pernet(net);
 374
 375         switch (trans->msg_type) {
 376         case NFT_MSG_NEWSET:
 377                 if (!nft_trans_set_update(trans) &&
 378                     nft_set_is_anonymous(nft_trans_set(trans)))
 379                         list_add_tail(&trans->binding_list, &nft_net->binding_list);
 380                 break;
 381         case NFT_MSG_NEWCHAIN:
 382                 if (!nft_trans_chain_update(trans) &&
 383                     nft_chain_binding(nft_trans_chain(trans)))
 384                         list_add_tail(&trans->binding_list, &nft_net->binding_list);
 385                 break;
 386         }
 387
 388         list_add_tail(&trans->list, &nft_net->commit_list);
 389 }
 390
 391 static int nft_trans_table_add(struct nft_ctx *ctx, int msg_type)
 392 {
 393         struct nft_trans *trans;
 394
 395         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_table));
 396         if (trans == NULL)
 397                 return -ENOMEM;
 398
 399         if (msg_type == NFT_MSG_NEWTABLE)
 400                 nft_activate_next(ctx->net, ctx->table);
 401
 402         nft_trans_commit_list_add_tail(ctx->net, trans);
 403         return 0;
 404 }
 405
 406 static int nft_deltable(struct nft_ctx *ctx)
 407 {
 408         int err;
 409
 410         err = nft_trans_table_add(ctx, NFT_MSG_DELTABLE);
 411         if (err < 0)
 412                 return err;
 413
 414         nft_deactivate_next(ctx->net, ctx->table);
 415         return err;
 416 }
 417
 418 static struct nft_trans *nft_trans_chain_add(struct nft_ctx *ctx, int msg_type)
 419 {
 420         struct nft_trans *trans;
 421
 422         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_chain));
 423         if (trans == NULL)
 424                 return ERR_PTR(-ENOMEM);
 425
 426         if (msg_type == NFT_MSG_NEWCHAIN) {
 427                 nft_activate_next(ctx->net, ctx->chain);
 428
 429                 if (ctx->nla[NFTA_CHAIN_ID]) {
 430                         nft_trans_chain_id(trans) =
 431                                 ntohl(nla_get_be32(ctx->nla[NFTA_CHAIN_ID]));
 432                 }
 433         }
 434         nft_trans_chain(trans) = ctx->chain;
 435         nft_trans_commit_list_add_tail(ctx->net, trans);
 436
 437         return trans;
 438 }
 439
 440 static int nft_delchain(struct nft_ctx *ctx)
 441 {
 442         struct nft_trans *trans;
 443
 444         trans = nft_trans_chain_add(ctx, NFT_MSG_DELCHAIN);
 445         if (IS_ERR(trans))
 446                 return PTR_ERR(trans);
 447
 448         nft_use_dec(&ctx->table->use);
 449         nft_deactivate_next(ctx->net, ctx->chain);
 450
 451         return 0;
 452 }
 453
 454 void nft_rule_expr_activate(const struct nft_ctx *ctx, struct nft_rule *rule)
 455 {
 456         struct nft_expr *expr;
 457
 458         expr = nft_expr_first(rule);
 459         while (nft_expr_more(rule, expr)) {
 460                 if (expr->ops->activate)
 461                         expr->ops->activate(ctx, expr);
 462
 463                 expr = nft_expr_next(expr);
 464         }
 465 }
 466
 467 void nft_rule_expr_deactivate(const struct nft_ctx *ctx, struct nft_rule *rule,
 468                               enum nft_trans_phase phase)
 469 {
 470         struct nft_expr *expr;
 471
 472         expr = nft_expr_first(rule);
 473         while (nft_expr_more(rule, expr)) {
 474                 if (expr->ops->deactivate)
 475                         expr->ops->deactivate(ctx, expr, phase);
 476
 477                 expr = nft_expr_next(expr);
 478         }
 479 }
 480
 481 static int
 482 nf_tables_delrule_deactivate(struct nft_ctx *ctx, struct nft_rule *rule)
 483 {
 484         /* You cannot delete the same rule twice */
 485         if (nft_is_active_next(ctx->net, rule)) {
 486                 nft_deactivate_next(ctx->net, rule);
 487                 nft_use_dec(&ctx->chain->use);
 488                 return 0;
 489         }
 490         return -ENOENT;
 491 }
 492
 493 static struct nft_trans *nft_trans_rule_add(struct nft_ctx *ctx, int msg_type,
 494                                             struct nft_rule *rule)
 495 {
 496         struct nft_trans *trans;
 497
 498         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_rule));
 499         if (trans == NULL)
 500                 return NULL;
 501
 502         if (msg_type == NFT_MSG_NEWRULE && ctx->nla[NFTA_RULE_ID] != NULL) {
 503                 nft_trans_rule_id(trans) =
 504                         ntohl(nla_get_be32(ctx->nla[NFTA_RULE_ID]));
 505         }
 506         nft_trans_rule(trans) = rule;
 507         nft_trans_commit_list_add_tail(ctx->net, trans);
 508
 509         return trans;
 510 }
 511
 512 static int nft_delrule(struct nft_ctx *ctx, struct nft_rule *rule)
 513 {
 514         struct nft_flow_rule *flow;
 515         struct nft_trans *trans;
 516         int err;
 517
 518         trans = nft_trans_rule_add(ctx, NFT_MSG_DELRULE, rule);
 519         if (trans == NULL)
 520                 return -ENOMEM;
 521
 522         if (ctx->chain->flags & NFT_CHAIN_HW_OFFLOAD) {
 523                 flow = nft_flow_rule_create(ctx->net, rule);
 524                 if (IS_ERR(flow)) {
 525                         nft_trans_destroy(trans);
 526                         return PTR_ERR(flow);
 527                 }
 528
 529                 nft_trans_flow_rule(trans) = flow;
 530         }
 531
 532         err = nf_tables_delrule_deactivate(ctx, rule);
 533         if (err < 0) {
 534                 nft_trans_destroy(trans);
 535                 return err;
 536         }
 537         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_PREPARE);
 538
 539         return 0;
 540 }
 541
 542 static int nft_delrule_by_chain(struct nft_ctx *ctx)
 543 {
 544         struct nft_rule *rule;
 545         int err;
 546
 547         list_for_each_entry(rule, &ctx->chain->rules, list) {
 548                 if (!nft_is_active_next(ctx->net, rule))
 549                         continue;
 550
 551                 err = nft_delrule(ctx, rule);
 552                 if (err < 0)
 553                         return err;
 554         }
 555         return 0;
 556 }
 557
 558 static int __nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
 559                                struct nft_set *set,
 560                                const struct nft_set_desc *desc)
 561 {
 562         struct nft_trans *trans;
 563
 564         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_set));
 565         if (trans == NULL)
 566                 return -ENOMEM;
 567
 568         if (msg_type == NFT_MSG_NEWSET && ctx->nla[NFTA_SET_ID] && !desc) {
 569                 nft_trans_set_id(trans) =
 570                         ntohl(nla_get_be32(ctx->nla[NFTA_SET_ID]));
 571                 nft_activate_next(ctx->net, set);
 572         }
 573         nft_trans_set(trans) = set;
 574         if (desc) {
 575                 nft_trans_set_update(trans) = true;
 576                 nft_trans_set_gc_int(trans) = desc->gc_int;
 577                 nft_trans_set_timeout(trans) = desc->timeout;
 578                 nft_trans_set_size(trans) = desc->size;
 579         }
 580         nft_trans_commit_list_add_tail(ctx->net, trans);
 581
 582         return 0;
 583 }
 584
 585 static int nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
 586                              struct nft_set *set)
 587 {
 588         return __nft_trans_set_add(ctx, msg_type, set, NULL);
 589 }
 590
 591 static int nft_mapelem_deactivate(const struct nft_ctx *ctx,
 592                                   struct nft_set *set,
 593                                   const struct nft_set_iter *iter,
 594                                   struct nft_set_elem *elem)
 595 {
 596         nft_setelem_data_deactivate(ctx->net, set, elem);
 597
 598         return 0;
 599 }
 600
 601 struct nft_set_elem_catchall {
 602         struct list_head        list;
 603         struct rcu_head         rcu;
 604         void                    *elem;
 605 };
 606
 607 static void nft_map_catchall_deactivate(const struct nft_ctx *ctx,
 608                                         struct nft_set *set)
 609 {
 610         u8 genmask = nft_genmask_next(ctx->net);
 611         struct nft_set_elem_catchall *catchall;
 612         struct nft_set_elem elem;
 613         struct nft_set_ext *ext;
 614
 615         list_for_each_entry(catchall, &set->catchall_list, list) {
 616                 ext = nft_set_elem_ext(set, catchall->elem);
 617                 if (!nft_set_elem_active(ext, genmask))
 618                         continue;
 619
 620                 elem.priv = catchall->elem;
 621                 nft_setelem_data_deactivate(ctx->net, set, &elem);
 622                 break;
 623         }
 624 }
 625
 626 static void nft_map_deactivate(const struct nft_ctx *ctx, struct nft_set *set)
 627 {
 628         struct nft_set_iter iter = {
 629                 .genmask        = nft_genmask_next(ctx->net),
 630                 .fn             = nft_mapelem_deactivate,
 631         };
 632
 633         set->ops->walk(ctx, set, &iter);
 634         WARN_ON_ONCE(iter.err);
 635
 636         nft_map_catchall_deactivate(ctx, set);
 637 }
 638
 639 static int nft_delset(const struct nft_ctx *ctx, struct nft_set *set)
 640 {
 641         int err;
 642
 643         err = nft_trans_set_add(ctx, NFT_MSG_DELSET, set);
 644         if (err < 0)
 645                 return err;
 646
 647         if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
 648                 nft_map_deactivate(ctx, set);
 649
 650         nft_deactivate_next(ctx->net, set);
 651         nft_use_dec(&ctx->table->use);
 652
 653         return err;
 654 }
 655
 656 static int nft_trans_obj_add(struct nft_ctx *ctx, int msg_type,
 657                              struct nft_object *obj)
 658 {
 659         struct nft_trans *trans;
 660
 661         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_obj));
 662         if (trans == NULL)
 663                 return -ENOMEM;
 664
 665         if (msg_type == NFT_MSG_NEWOBJ)
 666                 nft_activate_next(ctx->net, obj);
 667
 668         nft_trans_obj(trans) = obj;
 669         nft_trans_commit_list_add_tail(ctx->net, trans);
 670
 671         return 0;
 672 }
 673
 674 static int nft_delobj(struct nft_ctx *ctx, struct nft_object *obj)
 675 {
 676         int err;
 677
 678         err = nft_trans_obj_add(ctx, NFT_MSG_DELOBJ, obj);
 679         if (err < 0)
 680                 return err;
 681
 682         nft_deactivate_next(ctx->net, obj);
 683         nft_use_dec(&ctx->table->use);
 684
 685         return err;
 686 }
 687
 688 static int nft_trans_flowtable_add(struct nft_ctx *ctx, int msg_type,
 689                                    struct nft_flowtable *flowtable)
 690 {
 691         struct nft_trans *trans;
 692
 693         trans = nft_trans_alloc(ctx, msg_type,
 694                                 sizeof(struct nft_trans_flowtable));
 695         if (trans == NULL)
 696                 return -ENOMEM;
 697
 698         if (msg_type == NFT_MSG_NEWFLOWTABLE)
 699                 nft_activate_next(ctx->net, flowtable);
 700
 701         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
 702         nft_trans_flowtable(trans) = flowtable;
 703         nft_trans_commit_list_add_tail(ctx->net, trans);
 704
 705         return 0;
 706 }
 707
 708 static int nft_delflowtable(struct nft_ctx *ctx,
 709                             struct nft_flowtable *flowtable)
 710 {
 711         int err;
 712
 713         err = nft_trans_flowtable_add(ctx, NFT_MSG_DELFLOWTABLE, flowtable);
 714         if (err < 0)
 715                 return err;
 716
 717         nft_deactivate_next(ctx->net, flowtable);
 718         nft_use_dec(&ctx->table->use);
 719
 720         return err;
 721 }
 722
 723 static void __nft_reg_track_clobber(struct nft_regs_track *track, u8 dreg)
 724 {
 725         int i;
 726
 727         for (i = track->regs[dreg].num_reg; i > 0; i--)
 728                 __nft_reg_track_cancel(track, dreg - i);
 729 }
 730
 731 static void __nft_reg_track_update(struct nft_regs_track *track,
 732                                    const struct nft_expr *expr,
 733                                    u8 dreg, u8 num_reg)
 734 {
 735         track->regs[dreg].selector = expr;
 736         track->regs[dreg].bitwise = NULL;
 737         track->regs[dreg].num_reg = num_reg;
 738 }
 739
 740 void nft_reg_track_update(struct nft_regs_track *track,
 741                           const struct nft_expr *expr, u8 dreg, u8 len)
 742 {
 743         unsigned int regcount;
 744         int i;
 745
 746         __nft_reg_track_clobber(track, dreg);
 747
 748         regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
 749         for (i = 0; i < regcount; i++, dreg++)
 750                 __nft_reg_track_update(track, expr, dreg, i);
 751 }
 752 EXPORT_SYMBOL_GPL(nft_reg_track_update);
 753
 754 void nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg, u8 len)
 755 {
 756         unsigned int regcount;
 757         int i;
 758
 759         __nft_reg_track_clobber(track, dreg);
 760
 761         regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
 762         for (i = 0; i < regcount; i++, dreg++)
 763                 __nft_reg_track_cancel(track, dreg);
 764 }
 765 EXPORT_SYMBOL_GPL(nft_reg_track_cancel);
 766
 767 void __nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg)
 768 {
 769         track->regs[dreg].selector = NULL;
 770         track->regs[dreg].bitwise = NULL;
 771         track->regs[dreg].num_reg = 0;
 772 }
 773 EXPORT_SYMBOL_GPL(__nft_reg_track_cancel);
 774
 775 /*
 776  * Tables
 777  */
 778
 779 static struct nft_table *nft_table_lookup(const struct net *net,
 780                                           const struct nlattr *nla,
 781                                           u8 family, u8 genmask, u32 nlpid)
 782 {
 783         struct nftables_pernet *nft_net;
 784         struct nft_table *table;
 785
 786         if (nla == NULL)
 787                 return ERR_PTR(-EINVAL);
 788
 789         nft_net = nft_pernet(net);
 790         list_for_each_entry_rcu(table, &nft_net->tables, list,
 791                                 lockdep_is_held(&nft_net->commit_mutex)) {
 792                 if (!nla_strcmp(nla, table->name) &&
 793                     table->family == family &&
 794                     nft_active_genmask(table, genmask)) {
 795                         if (nft_table_has_owner(table) &&
 796                             nlpid && table->nlpid != nlpid)
 797                                 return ERR_PTR(-EPERM);
 798
 799                         return table;
 800                 }
 801         }
 802
 803         return ERR_PTR(-ENOENT);
 804 }
 805
 806 static struct nft_table *nft_table_lookup_byhandle(const struct net *net,
 807                                                    const struct nlattr *nla,
 808                                                    u8 genmask, u32 nlpid)
 809 {
 810         struct nftables_pernet *nft_net;
 811         struct nft_table *table;
 812
 813         nft_net = nft_pernet(net);
 814         list_for_each_entry(table, &nft_net->tables, list) {
 815                 if (be64_to_cpu(nla_get_be64(nla)) == table->handle &&
 816                     nft_active_genmask(table, genmask)) {
 817                         if (nft_table_has_owner(table) &&
 818                             nlpid && table->nlpid != nlpid)
 819                                 return ERR_PTR(-EPERM);
 820
 821                         return table;
 822                 }
 823         }
 824
 825         return ERR_PTR(-ENOENT);
 826 }
 827
 828 static inline u64 nf_tables_alloc_handle(struct nft_table *table)
 829 {
 830         return ++table->hgenerator;
 831 }
 832
 833 static const struct nft_chain_type *chain_type[NFPROTO_NUMPROTO][NFT_CHAIN_T_MAX];
 834
 835 static const struct nft_chain_type *
 836 __nft_chain_type_get(u8 family, enum nft_chain_types type)
 837 {
 838         if (family >= NFPROTO_NUMPROTO ||
 839             type >= NFT_CHAIN_T_MAX)
 840                 return NULL;
 841
 842         return chain_type[family][type];
 843 }
 844
 845 static const struct nft_chain_type *
 846 __nf_tables_chain_type_lookup(const struct nlattr *nla, u8 family)
 847 {
 848         const struct nft_chain_type *type;
 849         int i;
 850
 851         for (i = 0; i < NFT_CHAIN_T_MAX; i++) {
 852                 type = __nft_chain_type_get(family, i);
 853                 if (!type)
 854                         continue;
 855                 if (!nla_strcmp(nla, type->name))
 856                         return type;
 857         }
 858         return NULL;
 859 }
 860
 861 struct nft_module_request {
 862         struct list_head        list;
 863         char                    module[MODULE_NAME_LEN];
 864         bool                    done;
 865 };
 866
 867 #ifdef CONFIG_MODULES
 868 __printf(2, 3) int nft_request_module(struct net *net, const char *fmt,
 869                                       ...)
 870 {
 871         char module_name[MODULE_NAME_LEN];
 872         struct nftables_pernet *nft_net;
 873         struct nft_module_request *req;
 874         va_list args;
 875         int ret;
 876
 877         va_start(args, fmt);
 878         ret = vsnprintf(module_name, MODULE_NAME_LEN, fmt, args);
 879         va_end(args);
 880         if (ret >= MODULE_NAME_LEN)
 881                 return 0;
 882
 883         nft_net = nft_pernet(net);
 884         list_for_each_entry(req, &nft_net->module_list, list) {
 885                 if (!strcmp(req->module, module_name)) {
 886                         if (req->done)
 887                                 return 0;
 888
 889                         /* A request to load this module already exists. */
 890                         return -EAGAIN;
 891                 }
 892         }
 893
 894         req = kmalloc(sizeof(*req), GFP_KERNEL);
 895         if (!req)
 896                 return -ENOMEM;
 897
 898         req->done = false;
 899         strscpy(req->module, module_name, MODULE_NAME_LEN);
 900         list_add_tail(&req->list, &nft_net->module_list);
 901
 902         return -EAGAIN;
 903 }
 904 EXPORT_SYMBOL_GPL(nft_request_module);
 905 #endif
 906
 907 static void lockdep_nfnl_nft_mutex_not_held(void)
 908 {
 909 #ifdef CONFIG_PROVE_LOCKING
 910         if (debug_locks)
 911                 WARN_ON_ONCE(lockdep_nfnl_is_held(NFNL_SUBSYS_NFTABLES));
 912 #endif
 913 }
 914
 915 static const struct nft_chain_type *
 916 nf_tables_chain_type_lookup(struct net *net, const struct nlattr *nla,
 917                             u8 family, bool autoload)
 918 {
 919         const struct nft_chain_type *type;
 920
 921         type = __nf_tables_chain_type_lookup(nla, family);
 922         if (type != NULL)
 923                 return type;
 924
 925         lockdep_nfnl_nft_mutex_not_held();
 926 #ifdef CONFIG_MODULES
 927         if (autoload) {
 928                 if (nft_request_module(net, "nft-chain-%u-%.*s", family,
 929                                        nla_len(nla),
 930                                        (const char *)nla_data(nla)) == -EAGAIN)
 931                         return ERR_PTR(-EAGAIN);
 932         }
 933 #endif
 934         return ERR_PTR(-ENOENT);
 935 }
 936
 937 static __be16 nft_base_seq(const struct net *net)
 938 {
 939         struct nftables_pernet *nft_net = nft_pernet(net);
 940
 941         return htons(nft_net->base_seq & 0xffff);
 942 }
 943
 944 static const struct nla_policy nft_table_policy[NFTA_TABLE_MAX + 1] = {
 945         [NFTA_TABLE_NAME]       = { .type = NLA_STRING,
 946                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
 947         [NFTA_TABLE_FLAGS]      = { .type = NLA_U32 },
 948         [NFTA_TABLE_HANDLE]     = { .type = NLA_U64 },
 949         [NFTA_TABLE_USERDATA]   = { .type = NLA_BINARY,
 950                                     .len = NFT_USERDATA_MAXLEN }
 951 };
 952
 953 static int nf_tables_fill_table_info(struct sk_buff *skb, struct net *net,
 954                                      u32 portid, u32 seq, int event, u32 flags,
 955                                      int family, const struct nft_table *table)
 956 {
 957         struct nlmsghdr *nlh;
 958
 959         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
 960         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
 961                            NFNETLINK_V0, nft_base_seq(net));
 962         if (!nlh)
 963                 goto nla_put_failure;
 964
 965         if (nla_put_string(skb, NFTA_TABLE_NAME, table->name) ||
 966             nla_put_be32(skb, NFTA_TABLE_USE, htonl(table->use)) ||
 967             nla_put_be64(skb, NFTA_TABLE_HANDLE, cpu_to_be64(table->handle),
 968                          NFTA_TABLE_PAD))
 969                 goto nla_put_failure;
 970
 971         if (event == NFT_MSG_DELTABLE) {
 972                 nlmsg_end(skb, nlh);
 973                 return 0;
 974         }
 975
 976         if (nla_put_be32(skb, NFTA_TABLE_FLAGS,
 977                          htonl(table->flags & NFT_TABLE_F_MASK)))
 978                 goto nla_put_failure;
 979
 980         if (nft_table_has_owner(table) &&
 981             nla_put_be32(skb, NFTA_TABLE_OWNER, htonl(table->nlpid)))
 982                 goto nla_put_failure;
 983
 984         if (table->udata) {
 985                 if (nla_put(skb, NFTA_TABLE_USERDATA, table->udlen, table->udata))
 986                         goto nla_put_failure;
 987         }
 988
 989         nlmsg_end(skb, nlh);
 990         return 0;
 991
 992 nla_put_failure:
 993         nlmsg_trim(skb, nlh);
 994         return -1;
 995 }
 996
 997 struct nftnl_skb_parms {
 998         bool report;
 999 };
1000 #define NFT_CB(skb)     (*(struct nftnl_skb_parms*)&((skb)->cb))
1001
1002 static void nft_notify_enqueue(struct sk_buff *skb, bool report,
1003                                struct list_head *notify_list)
1004 {
1005         NFT_CB(skb).report = report;
1006         list_add_tail(&skb->list, notify_list);
1007 }
1008
1009 static void nf_tables_table_notify(const struct nft_ctx *ctx, int event)
1010 {
1011         struct nftables_pernet *nft_net;
1012         struct sk_buff *skb;
1013         u16 flags = 0;
1014         int err;
1015
1016         if (!ctx->report &&
1017             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
1018                 return;
1019
1020         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
1021         if (skb == NULL)
1022                 goto err;
1023
1024         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
1025                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
1026
1027         err = nf_tables_fill_table_info(skb, ctx->net, ctx->portid, ctx->seq,
1028                                         event, flags, ctx->family, ctx->table);
1029         if (err < 0) {
1030                 kfree_skb(skb);
1031                 goto err;
1032         }
1033
1034         nft_net = nft_pernet(ctx->net);
1035         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
1036         return;
1037 err:
1038         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
1039 }
1040
1041 static int nf_tables_dump_tables(struct sk_buff *skb,
1042                                  struct netlink_callback *cb)
1043 {
1044         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
1045         struct nftables_pernet *nft_net;
1046         const struct nft_table *table;
1047         unsigned int idx = 0, s_idx = cb->args[0];
1048         struct net *net = sock_net(skb->sk);
1049         int family = nfmsg->nfgen_family;
1050
1051         rcu_read_lock();
1052         nft_net = nft_pernet(net);
1053         cb->seq = READ_ONCE(nft_net->base_seq);
1054
1055         list_for_each_entry_rcu(table, &nft_net->tables, list) {
1056                 if (family != NFPROTO_UNSPEC && family != table->family)
1057                         continue;
1058
1059                 if (idx < s_idx)
1060                         goto cont;
1061                 if (idx > s_idx)
1062                         memset(&cb->args[1], 0,
1063                                sizeof(cb->args) - sizeof(cb->args[0]));
1064                 if (!nft_is_active(net, table))
1065                         continue;
1066                 if (nf_tables_fill_table_info(skb, net,
1067                                               NETLINK_CB(cb->skb).portid,
1068                                               cb->nlh->nlmsg_seq,
1069                                               NFT_MSG_NEWTABLE, NLM_F_MULTI,
1070                                               table->family, table) < 0)
1071                         goto done;
1072
1073                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
1074 cont:
1075                 idx++;
1076         }
1077 done:
1078         rcu_read_unlock();
1079         cb->args[0] = idx;
1080         return skb->len;
1081 }
1082
1083 static int nft_netlink_dump_start_rcu(struct sock *nlsk, struct sk_buff *skb,
1084                                       const struct nlmsghdr *nlh,
1085                                       struct netlink_dump_control *c)
1086 {
1087         int err;
1088
1089         if (!try_module_get(THIS_MODULE))
1090                 return -EINVAL;
1091
1092         rcu_read_unlock();
1093         err = netlink_dump_start(nlsk, skb, nlh, c);
1094         rcu_read_lock();
1095         module_put(THIS_MODULE);
1096
1097         return err;
1098 }
1099
1100 /* called with rcu_read_lock held */
1101 static int nf_tables_gettable(struct sk_buff *skb, const struct nfnl_info *info,
1102                               const struct nlattr * const nla[])
1103 {
1104         struct netlink_ext_ack *extack = info->extack;
1105         u8 genmask = nft_genmask_cur(info->net);
1106         u8 family = info->nfmsg->nfgen_family;
1107         const struct nft_table *table;
1108         struct net *net = info->net;
1109         struct sk_buff *skb2;
1110         int err;
1111
1112         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
1113                 struct netlink_dump_control c = {
1114                         .dump = nf_tables_dump_tables,
1115                         .module = THIS_MODULE,
1116                 };
1117
1118                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
1119         }
1120
1121         table = nft_table_lookup(net, nla[NFTA_TABLE_NAME], family, genmask, 0);
1122         if (IS_ERR(table)) {
1123                 NL_SET_BAD_ATTR(extack, nla[NFTA_TABLE_NAME]);
1124                 return PTR_ERR(table);
1125         }
1126
1127         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
1128         if (!skb2)
1129                 return -ENOMEM;
1130
1131         err = nf_tables_fill_table_info(skb2, net, NETLINK_CB(skb).portid,
1132                                         info->nlh->nlmsg_seq, NFT_MSG_NEWTABLE,
1133                                         0, family, table);
1134         if (err < 0)
1135                 goto err_fill_table_info;
1136
1137         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
1138
1139 err_fill_table_info:
1140         kfree_skb(skb2);
1141         return err;
1142 }
1143
1144 static void nft_table_disable(struct net *net, struct nft_table *table, u32 cnt)
1145 {
1146         struct nft_chain *chain;
1147         u32 i = 0;
1148
1149         list_for_each_entry(chain, &table->chains, list) {
1150                 if (!nft_is_active_next(net, chain))
1151                         continue;
1152                 if (!nft_is_base_chain(chain))
1153                         continue;
1154
1155                 if (cnt && i++ == cnt)
1156                         break;
1157
1158                 nf_tables_unregister_hook(net, table, chain);
1159         }
1160 }
1161
1162 static int nf_tables_table_enable(struct net *net, struct nft_table *table)
1163 {
1164         struct nft_chain *chain;
1165         int err, i = 0;
1166
1167         list_for_each_entry(chain, &table->chains, list) {
1168                 if (!nft_is_active_next(net, chain))
1169                         continue;
1170                 if (!nft_is_base_chain(chain))
1171                         continue;
1172
1173                 err = nf_tables_register_hook(net, table, chain);
1174                 if (err < 0)
1175                         goto err_register_hooks;
1176
1177                 i++;
1178         }
1179         return 0;
1180
1181 err_register_hooks:
1182         if (i)
1183                 nft_table_disable(net, table, i);
1184         return err;
1185 }
1186
1187 static void nf_tables_table_disable(struct net *net, struct nft_table *table)
1188 {
1189         table->flags &= ~NFT_TABLE_F_DORMANT;
1190         nft_table_disable(net, table, 0);
1191         table->flags |= NFT_TABLE_F_DORMANT;
1192 }
1193
1194 #define __NFT_TABLE_F_INTERNAL          (NFT_TABLE_F_MASK + 1)
1195 #define __NFT_TABLE_F_WAS_DORMANT       (__NFT_TABLE_F_INTERNAL << 0)
1196 #define __NFT_TABLE_F_WAS_AWAKEN        (__NFT_TABLE_F_INTERNAL << 1)
1197 #define __NFT_TABLE_F_UPDATE            (__NFT_TABLE_F_WAS_DORMANT | \
1198                                          __NFT_TABLE_F_WAS_AWAKEN)
1199
1200 static int nf_tables_updtable(struct nft_ctx *ctx)
1201 {
1202         struct nft_trans *trans;
1203         u32 flags;
1204         int ret;
1205
1206         if (!ctx->nla[NFTA_TABLE_FLAGS])
1207                 return 0;
1208
1209         flags = ntohl(nla_get_be32(ctx->nla[NFTA_TABLE_FLAGS]));
1210         if (flags & ~NFT_TABLE_F_MASK)
1211                 return -EOPNOTSUPP;
1212
1213         if (flags == ctx->table->flags)
1214                 return 0;
1215
1216         if ((nft_table_has_owner(ctx->table) &&
1217              !(flags & NFT_TABLE_F_OWNER)) ||
1218             (!nft_table_has_owner(ctx->table) &&
1219              flags & NFT_TABLE_F_OWNER))
1220                 return -EOPNOTSUPP;
1221
1222         trans = nft_trans_alloc(ctx, NFT_MSG_NEWTABLE,
1223                                 sizeof(struct nft_trans_table));
1224         if (trans == NULL)
1225                 return -ENOMEM;
1226
1227         if ((flags & NFT_TABLE_F_DORMANT) &&
1228             !(ctx->table->flags & NFT_TABLE_F_DORMANT)) {
1229                 ctx->table->flags |= NFT_TABLE_F_DORMANT;
1230                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE))
1231                         ctx->table->flags |= __NFT_TABLE_F_WAS_AWAKEN;
1232         } else if (!(flags & NFT_TABLE_F_DORMANT) &&
1233                    ctx->table->flags & NFT_TABLE_F_DORMANT) {
1234                 ctx->table->flags &= ~NFT_TABLE_F_DORMANT;
1235                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE)) {
1236                         ret = nf_tables_table_enable(ctx->net, ctx->table);
1237                         if (ret < 0)
1238                                 goto err_register_hooks;
1239
1240                         ctx->table->flags |= __NFT_TABLE_F_WAS_DORMANT;
1241                 }
1242         }
1243
1244         nft_trans_table_update(trans) = true;
1245         nft_trans_commit_list_add_tail(ctx->net, trans);
1246
1247         return 0;
1248
1249 err_register_hooks:
1250         nft_trans_destroy(trans);
1251         return ret;
1252 }
1253
1254 static u32 nft_chain_hash(const void *data, u32 len, u32 seed)
1255 {
1256         const char *name = data;
1257
1258         return jhash(name, strlen(name), seed);
1259 }
1260
1261 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed)
1262 {
1263         const struct nft_chain *chain = data;
1264
1265         return nft_chain_hash(chain->name, 0, seed);
1266 }
1267
1268 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *arg,
1269                               const void *ptr)
1270 {
1271         const struct nft_chain *chain = ptr;
1272         const char *name = arg->key;
1273
1274         return strcmp(chain->name, name);
1275 }
1276
1277 static u32 nft_objname_hash(const void *data, u32 len, u32 seed)
1278 {
1279         const struct nft_object_hash_key *k = data;
1280
1281         seed ^= hash_ptr(k->table, 32);
1282
1283         return jhash(k->name, strlen(k->name), seed);
1284 }
1285
1286 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed)
1287 {
1288         const struct nft_object *obj = data;
1289
1290         return nft_objname_hash(&obj->key, 0, seed);
1291 }
1292
1293 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *arg,
1294                                 const void *ptr)
1295 {
1296         const struct nft_object_hash_key *k = arg->key;
1297         const struct nft_object *obj = ptr;
1298
1299         if (obj->key.table != k->table)
1300                 return -1;
1301
1302         return strcmp(obj->key.name, k->name);
1303 }
1304
1305 static bool nft_supported_family(u8 family)
1306 {
1307         return false
1308 #ifdef CONFIG_NF_TABLES_INET
1309                 || family == NFPROTO_INET
1310 #endif
1311 #ifdef CONFIG_NF_TABLES_IPV4
1312                 || family == NFPROTO_IPV4
1313 #endif
1314 #ifdef CONFIG_NF_TABLES_ARP
1315                 || family == NFPROTO_ARP
1316 #endif
1317 #ifdef CONFIG_NF_TABLES_NETDEV
1318                 || family == NFPROTO_NETDEV
1319 #endif
1320 #if IS_ENABLED(CONFIG_NF_TABLES_BRIDGE)
1321                 || family == NFPROTO_BRIDGE
1322 #endif
1323 #ifdef CONFIG_NF_TABLES_IPV6
1324                 || family == NFPROTO_IPV6
1325 #endif
1326                 ;
1327 }
1328
1329 static int nf_tables_newtable(struct sk_buff *skb, const struct nfnl_info *info,
1330                               const struct nlattr * const nla[])
1331 {
1332         struct nftables_pernet *nft_net = nft_pernet(info->net);
1333         struct netlink_ext_ack *extack = info->extack;
1334         u8 genmask = nft_genmask_next(info->net);
1335         u8 family = info->nfmsg->nfgen_family;
1336         struct net *net = info->net;
1337         const struct nlattr *attr;
1338         struct nft_table *table;
1339         struct nft_ctx ctx;
1340         u32 flags = 0;
1341         int err;
1342
1343         if (!nft_supported_family(family))
1344                 return -EOPNOTSUPP;
1345
1346         lockdep_assert_held(&nft_net->commit_mutex);
1347         attr = nla[NFTA_TABLE_NAME];
1348         table = nft_table_lookup(net, attr, family, genmask,
1349                                  NETLINK_CB(skb).portid);
1350         if (IS_ERR(table)) {
1351                 if (PTR_ERR(table) != -ENOENT)
1352                         return PTR_ERR(table);
1353         } else {
1354                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
1355                         NL_SET_BAD_ATTR(extack, attr);
1356                         return -EEXIST;
1357                 }
1358                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
1359                         return -EOPNOTSUPP;
1360
1361                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1362
1363                 return nf_tables_updtable(&ctx);
1364         }
1365
1366         if (nla[NFTA_TABLE_FLAGS]) {
1367                 flags = ntohl(nla_get_be32(nla[NFTA_TABLE_FLAGS]));
1368                 if (flags & ~NFT_TABLE_F_MASK)
1369                         return -EOPNOTSUPP;
1370         }
1371
1372         err = -ENOMEM;
1373         table = kzalloc(sizeof(*table), GFP_KERNEL_ACCOUNT);
1374         if (table == NULL)
1375                 goto err_kzalloc;
1376
1377         table->validate_state = nft_net->validate_state;
1378         table->name = nla_strdup(attr, GFP_KERNEL_ACCOUNT);
1379         if (table->name == NULL)
1380                 goto err_strdup;
1381
1382         if (nla[NFTA_TABLE_USERDATA]) {
1383                 table->udata = nla_memdup(nla[NFTA_TABLE_USERDATA], GFP_KERNEL_ACCOUNT);
1384                 if (table->udata == NULL)
1385                         goto err_table_udata;
1386
1387                 table->udlen = nla_len(nla[NFTA_TABLE_USERDATA]);
1388         }
1389
1390         err = rhltable_init(&table->chains_ht, &nft_chain_ht_params);
1391         if (err)
1392                 goto err_chain_ht;
1393
1394         INIT_LIST_HEAD(&table->chains);
1395         INIT_LIST_HEAD(&table->sets);
1396         INIT_LIST_HEAD(&table->objects);
1397         INIT_LIST_HEAD(&table->flowtables);
1398         table->family = family;
1399         table->flags = flags;
1400         table->handle = ++nft_net->table_handle;
1401         if (table->flags & NFT_TABLE_F_OWNER)
1402                 table->nlpid = NETLINK_CB(skb).portid;
1403
1404         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1405         err = nft_trans_table_add(&ctx, NFT_MSG_NEWTABLE);
1406         if (err < 0)
1407                 goto err_trans;
1408
1409         list_add_tail_rcu(&table->list, &nft_net->tables);
1410         return 0;
1411 err_trans:
1412         rhltable_destroy(&table->chains_ht);
1413 err_chain_ht:
1414         kfree(table->udata);
1415 err_table_udata:
1416         kfree(table->name);
1417 err_strdup:
1418         kfree(table);
1419 err_kzalloc:
1420         return err;
1421 }
1422
1423 static int nft_flush_table(struct nft_ctx *ctx)
1424 {
1425         struct nft_flowtable *flowtable, *nft;
1426         struct nft_chain *chain, *nc;
1427         struct nft_object *obj, *ne;
1428         struct nft_set *set, *ns;
1429         int err;
1430
1431         list_for_each_entry(chain, &ctx->table->chains, list) {
1432                 if (!nft_is_active_next(ctx->net, chain))
1433                         continue;
1434
1435                 if (nft_chain_is_bound(chain))
1436                         continue;
1437
1438                 ctx->chain = chain;
1439
1440                 err = nft_delrule_by_chain(ctx);
1441                 if (err < 0)
1442                         goto out;
1443         }
1444
1445         list_for_each_entry_safe(set, ns, &ctx->table->sets, list) {
1446                 if (!nft_is_active_next(ctx->net, set))
1447                         continue;
1448
1449                 if (nft_set_is_anonymous(set) &&
1450                     !list_empty(&set->bindings))
1451                         continue;
1452
1453                 err = nft_delset(ctx, set);
1454                 if (err < 0)
1455                         goto out;
1456         }
1457
1458         list_for_each_entry_safe(flowtable, nft, &ctx->table->flowtables, list) {
1459                 if (!nft_is_active_next(ctx->net, flowtable))
1460                         continue;
1461
1462                 err = nft_delflowtable(ctx, flowtable);
1463                 if (err < 0)
1464                         goto out;
1465         }
1466
1467         list_for_each_entry_safe(obj, ne, &ctx->table->objects, list) {
1468                 if (!nft_is_active_next(ctx->net, obj))
1469                         continue;
1470
1471                 err = nft_delobj(ctx, obj);
1472                 if (err < 0)
1473                         goto out;
1474         }
1475
1476         list_for_each_entry_safe(chain, nc, &ctx->table->chains, list) {
1477                 if (!nft_is_active_next(ctx->net, chain))
1478                         continue;
1479
1480                 if (nft_chain_is_bound(chain))
1481                         continue;
1482
1483                 ctx->chain = chain;
1484
1485                 err = nft_delchain(ctx);
1486                 if (err < 0)
1487                         goto out;
1488         }
1489
1490         err = nft_deltable(ctx);
1491 out:
1492         return err;
1493 }
1494
1495 static int nft_flush(struct nft_ctx *ctx, int family)
1496 {
1497         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
1498         const struct nlattr * const *nla = ctx->nla;
1499         struct nft_table *table, *nt;
1500         int err = 0;
1501
1502         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
1503                 if (family != AF_UNSPEC && table->family != family)
1504                         continue;
1505
1506                 ctx->family = table->family;
1507
1508                 if (!nft_is_active_next(ctx->net, table))
1509                         continue;
1510
1511                 if (nft_table_has_owner(table) && table->nlpid != ctx->portid)
1512                         continue;
1513
1514                 if (nla[NFTA_TABLE_NAME] &&
1515                     nla_strcmp(nla[NFTA_TABLE_NAME], table->name) != 0)
1516                         continue;
1517
1518                 ctx->table = table;
1519
1520                 err = nft_flush_table(ctx);
1521                 if (err < 0)
1522                         goto out;
1523         }
1524 out:
1525         return err;
1526 }
1527
1528 static int nf_tables_deltable(struct sk_buff *skb, const struct nfnl_info *info,
1529                               const struct nlattr * const nla[])
1530 {
1531         struct netlink_ext_ack *extack = info->extack;
1532         u8 genmask = nft_genmask_next(info->net);
1533         u8 family = info->nfmsg->nfgen_family;
1534         struct net *net = info->net;
1535         const struct nlattr *attr;
1536         struct nft_table *table;
1537         struct nft_ctx ctx;
1538
1539         nft_ctx_init(&ctx, net, skb, info->nlh, 0, NULL, NULL, nla);
1540         if (family == AF_UNSPEC ||
1541             (!nla[NFTA_TABLE_NAME] && !nla[NFTA_TABLE_HANDLE]))
1542                 return nft_flush(&ctx, family);
1543
1544         if (nla[NFTA_TABLE_HANDLE]) {
1545                 attr = nla[NFTA_TABLE_HANDLE];
1546                 table = nft_table_lookup_byhandle(net, attr, genmask,
1547                                                   NETLINK_CB(skb).portid);
1548         } else {
1549                 attr = nla[NFTA_TABLE_NAME];
1550                 table = nft_table_lookup(net, attr, family, genmask,
1551                                          NETLINK_CB(skb).portid);
1552         }
1553
1554         if (IS_ERR(table)) {
1555                 if (PTR_ERR(table) == -ENOENT &&
1556                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYTABLE)
1557                         return 0;
1558
1559                 NL_SET_BAD_ATTR(extack, attr);
1560                 return PTR_ERR(table);
1561         }
1562
1563         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
1564             table->use > 0)
1565                 return -EBUSY;
1566
1567         ctx.family = family;
1568         ctx.table = table;
1569
1570         return nft_flush_table(&ctx);
1571 }
1572
1573 static void nf_tables_table_destroy(struct nft_ctx *ctx)
1574 {
1575         if (WARN_ON(ctx->table->use > 0))
1576                 return;
1577
1578         rhltable_destroy(&ctx->table->chains_ht);
1579         kfree(ctx->table->name);
1580         kfree(ctx->table->udata);
1581         kfree(ctx->table);
1582 }
1583
1584 void nft_register_chain_type(const struct nft_chain_type *ctype)
1585 {
1586         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1587         if (WARN_ON(__nft_chain_type_get(ctype->family, ctype->type))) {
1588                 nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1589                 return;
1590         }
1591         chain_type[ctype->family][ctype->type] = ctype;
1592         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1593 }
1594 EXPORT_SYMBOL_GPL(nft_register_chain_type);
1595
1596 void nft_unregister_chain_type(const struct nft_chain_type *ctype)
1597 {
1598         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1599         chain_type[ctype->family][ctype->type] = NULL;
1600         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1601 }
1602 EXPORT_SYMBOL_GPL(nft_unregister_chain_type);
1603
1604 /*
1605  * Chains
1606  */
1607
1608 static struct nft_chain *
1609 nft_chain_lookup_byhandle(const struct nft_table *table, u64 handle, u8 genmask)
1610 {
1611         struct nft_chain *chain;
1612
1613         list_for_each_entry(chain, &table->chains, list) {
1614                 if (chain->handle == handle &&
1615                     nft_active_genmask(chain, genmask))
1616                         return chain;
1617         }
1618
1619         return ERR_PTR(-ENOENT);
1620 }
1621
1622 static bool lockdep_commit_lock_is_held(const struct net *net)
1623 {
1624 #ifdef CONFIG_PROVE_LOCKING
1625         struct nftables_pernet *nft_net = nft_pernet(net);
1626
1627         return lockdep_is_held(&nft_net->commit_mutex);
1628 #else
1629         return true;
1630 #endif
1631 }
1632
1633 static struct nft_chain *nft_chain_lookup(struct net *net,
1634                                           struct nft_table *table,
1635                                           const struct nlattr *nla, u8 genmask)
1636 {
1637         char search[NFT_CHAIN_MAXNAMELEN + 1];
1638         struct rhlist_head *tmp, *list;
1639         struct nft_chain *chain;
1640
1641         if (nla == NULL)
1642                 return ERR_PTR(-EINVAL);
1643
1644         nla_strscpy(search, nla, sizeof(search));
1645
1646         WARN_ON(!rcu_read_lock_held() &&
1647                 !lockdep_commit_lock_is_held(net));
1648
1649         chain = ERR_PTR(-ENOENT);
1650         rcu_read_lock();
1651         list = rhltable_lookup(&table->chains_ht, search, nft_chain_ht_params);
1652         if (!list)
1653                 goto out_unlock;
1654
1655         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
1656                 if (nft_active_genmask(chain, genmask))
1657                         goto out_unlock;
1658         }
1659         chain = ERR_PTR(-ENOENT);
1660 out_unlock:
1661         rcu_read_unlock();
1662         return chain;
1663 }
1664
1665 static const struct nla_policy nft_chain_policy[NFTA_CHAIN_MAX + 1] = {
1666         [NFTA_CHAIN_TABLE]      = { .type = NLA_STRING,
1667                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
1668         [NFTA_CHAIN_HANDLE]     = { .type = NLA_U64 },
1669         [NFTA_CHAIN_NAME]       = { .type = NLA_STRING,
1670                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
1671         [NFTA_CHAIN_HOOK]       = { .type = NLA_NESTED },
1672         [NFTA_CHAIN_POLICY]     = { .type = NLA_U32 },
1673         [NFTA_CHAIN_TYPE]       = { .type = NLA_STRING,
1674                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
1675         [NFTA_CHAIN_COUNTERS]   = { .type = NLA_NESTED },
1676         [NFTA_CHAIN_FLAGS]      = { .type = NLA_U32 },
1677         [NFTA_CHAIN_ID]         = { .type = NLA_U32 },
1678         [NFTA_CHAIN_USERDATA]   = { .type = NLA_BINARY,
1679                                     .len = NFT_USERDATA_MAXLEN },
1680 };
1681
1682 static const struct nla_policy nft_hook_policy[NFTA_HOOK_MAX + 1] = {
1683         [NFTA_HOOK_HOOKNUM]     = { .type = NLA_U32 },
1684         [NFTA_HOOK_PRIORITY]    = { .type = NLA_U32 },
1685         [NFTA_HOOK_DEV]         = { .type = NLA_STRING,
1686                                     .len = IFNAMSIZ - 1 },
1687 };
1688
1689 static int nft_dump_stats(struct sk_buff *skb, struct nft_stats __percpu *stats)
1690 {
1691         struct nft_stats *cpu_stats, total;
1692         struct nlattr *nest;
1693         unsigned int seq;
1694         u64 pkts, bytes;
1695         int cpu;
1696
1697         if (!stats)
1698                 return 0;
1699
1700         memset(&total, 0, sizeof(total));
1701         for_each_possible_cpu(cpu) {
1702                 cpu_stats = per_cpu_ptr(stats, cpu);
1703                 do {
1704                         seq = u64_stats_fetch_begin(&cpu_stats->syncp);
1705                         pkts = cpu_stats->pkts;
1706                         bytes = cpu_stats->bytes;
1707                 } while (u64_stats_fetch_retry(&cpu_stats->syncp, seq));
1708                 total.pkts += pkts;
1709                 total.bytes += bytes;
1710         }
1711         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_COUNTERS);
1712         if (nest == NULL)
1713                 goto nla_put_failure;
1714
1715         if (nla_put_be64(skb, NFTA_COUNTER_PACKETS, cpu_to_be64(total.pkts),
1716                          NFTA_COUNTER_PAD) ||
1717             nla_put_be64(skb, NFTA_COUNTER_BYTES, cpu_to_be64(total.bytes),
1718                          NFTA_COUNTER_PAD))
1719                 goto nla_put_failure;
1720
1721         nla_nest_end(skb, nest);
1722         return 0;
1723
1724 nla_put_failure:
1725         return -ENOSPC;
1726 }
1727
1728 static int nft_dump_basechain_hook(struct sk_buff *skb, int family,
1729                                    const struct nft_base_chain *basechain,
1730                                    const struct list_head *hook_list)
1731 {
1732         const struct nf_hook_ops *ops = &basechain->ops;
1733         struct nft_hook *hook, *first = NULL;
1734         struct nlattr *nest, *nest_devs;
1735         int n = 0;
1736
1737         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_HOOK);
1738         if (nest == NULL)
1739                 goto nla_put_failure;
1740         if (nla_put_be32(skb, NFTA_HOOK_HOOKNUM, htonl(ops->hooknum)))
1741                 goto nla_put_failure;
1742         if (nla_put_be32(skb, NFTA_HOOK_PRIORITY, htonl(ops->priority)))
1743                 goto nla_put_failure;
1744
1745         if (nft_base_chain_netdev(family, ops->hooknum)) {
1746                 nest_devs = nla_nest_start_noflag(skb, NFTA_HOOK_DEVS);
1747                 if (!nest_devs)
1748                         goto nla_put_failure;
1749
1750                 if (!hook_list)
1751                         hook_list = &basechain->hook_list;
1752
1753                 list_for_each_entry(hook, hook_list, list) {
1754                         if (!first)
1755                                 first = hook;
1756
1757                         if (nla_put_string(skb, NFTA_DEVICE_NAME,
1758                                            hook->ops.dev->name))
1759                                 goto nla_put_failure;
1760                         n++;
1761                 }
1762                 nla_nest_end(skb, nest_devs);
1763
1764                 if (n == 1 &&
1765                     nla_put_string(skb, NFTA_HOOK_DEV, first->ops.dev->name))
1766                         goto nla_put_failure;
1767         }
1768         nla_nest_end(skb, nest);
1769
1770         return 0;
1771 nla_put_failure:
1772         return -1;
1773 }
1774
1775 static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net,
1776                                      u32 portid, u32 seq, int event, u32 flags,
1777                                      int family, const struct nft_table *table,
1778                                      const struct nft_chain *chain,
1779                                      const struct list_head *hook_list)
1780 {
1781         struct nlmsghdr *nlh;
1782
1783         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
1784         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
1785                            NFNETLINK_V0, nft_base_seq(net));
1786         if (!nlh)
1787                 goto nla_put_failure;
1788
1789         if (nla_put_string(skb, NFTA_CHAIN_TABLE, table->name) ||
1790             nla_put_string(skb, NFTA_CHAIN_NAME, chain->name) ||
1791             nla_put_be64(skb, NFTA_CHAIN_HANDLE, cpu_to_be64(chain->handle),
1792                          NFTA_CHAIN_PAD))
1793                 goto nla_put_failure;
1794
1795         if (event == NFT_MSG_DELCHAIN && !hook_list) {
1796                 nlmsg_end(skb, nlh);
1797                 return 0;
1798         }
1799
1800         if (nft_is_base_chain(chain)) {
1801                 const struct nft_base_chain *basechain = nft_base_chain(chain);
1802                 struct nft_stats __percpu *stats;
1803
1804                 if (nft_dump_basechain_hook(skb, family, basechain, hook_list))
1805                         goto nla_put_failure;
1806
1807                 if (nla_put_be32(skb, NFTA_CHAIN_POLICY,
1808                                  htonl(basechain->policy)))
1809                         goto nla_put_failure;
1810
1811                 if (nla_put_string(skb, NFTA_CHAIN_TYPE, basechain->type->name))
1812                         goto nla_put_failure;
1813
1814                 stats = rcu_dereference_check(basechain->stats,
1815                                               lockdep_commit_lock_is_held(net));
1816                 if (nft_dump_stats(skb, stats))
1817                         goto nla_put_failure;
1818         }
1819
1820         if (chain->flags &&
1821             nla_put_be32(skb, NFTA_CHAIN_FLAGS, htonl(chain->flags)))
1822                 goto nla_put_failure;
1823
1824         if (nla_put_be32(skb, NFTA_CHAIN_USE, htonl(chain->use)))
1825                 goto nla_put_failure;
1826
1827         if (chain->udata &&
1828             nla_put(skb, NFTA_CHAIN_USERDATA, chain->udlen, chain->udata))
1829                 goto nla_put_failure;
1830
1831         nlmsg_end(skb, nlh);
1832         return 0;
1833
1834 nla_put_failure:
1835         nlmsg_trim(skb, nlh);
1836         return -1;
1837 }
1838
1839 static void nf_tables_chain_notify(const struct nft_ctx *ctx, int event,
1840                                    const struct list_head *hook_list)
1841 {
1842         struct nftables_pernet *nft_net;
1843         struct sk_buff *skb;
1844         u16 flags = 0;
1845         int err;
1846
1847         if (!ctx->report &&
1848             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
1849                 return;
1850
1851         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
1852         if (skb == NULL)
1853                 goto err;
1854
1855         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
1856                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
1857
1858         err = nf_tables_fill_chain_info(skb, ctx->net, ctx->portid, ctx->seq,
1859                                         event, flags, ctx->family, ctx->table,
1860                                         ctx->chain, hook_list);
1861         if (err < 0) {
1862                 kfree_skb(skb);
1863                 goto err;
1864         }
1865
1866         nft_net = nft_pernet(ctx->net);
1867         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
1868         return;
1869 err:
1870         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
1871 }
1872
1873 static int nf_tables_dump_chains(struct sk_buff *skb,
1874                                  struct netlink_callback *cb)
1875 {
1876         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
1877         unsigned int idx = 0, s_idx = cb->args[0];
1878         struct net *net = sock_net(skb->sk);
1879         int family = nfmsg->nfgen_family;
1880         struct nftables_pernet *nft_net;
1881         const struct nft_table *table;
1882         const struct nft_chain *chain;
1883
1884         rcu_read_lock();
1885         nft_net = nft_pernet(net);
1886         cb->seq = READ_ONCE(nft_net->base_seq);
1887
1888         list_for_each_entry_rcu(table, &nft_net->tables, list) {
1889                 if (family != NFPROTO_UNSPEC && family != table->family)
1890                         continue;
1891
1892                 list_for_each_entry_rcu(chain, &table->chains, list) {
1893                         if (idx < s_idx)
1894                                 goto cont;
1895                         if (idx > s_idx)
1896                                 memset(&cb->args[1], 0,
1897                                        sizeof(cb->args) - sizeof(cb->args[0]));
1898                         if (!nft_is_active(net, chain))
1899                                 continue;
1900                         if (nf_tables_fill_chain_info(skb, net,
1901                                                       NETLINK_CB(cb->skb).portid,
1902                                                       cb->nlh->nlmsg_seq,
1903                                                       NFT_MSG_NEWCHAIN,
1904                                                       NLM_F_MULTI,
1905                                                       table->family, table,
1906                                                       chain, NULL) < 0)
1907                                 goto done;
1908
1909                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
1910 cont:
1911                         idx++;
1912                 }
1913         }
1914 done:
1915         rcu_read_unlock();
1916         cb->args[0] = idx;
1917         return skb->len;
1918 }
1919
1920 /* called with rcu_read_lock held */
1921 static int nf_tables_getchain(struct sk_buff *skb, const struct nfnl_info *info,
1922                               const struct nlattr * const nla[])
1923 {
1924         struct netlink_ext_ack *extack = info->extack;
1925         u8 genmask = nft_genmask_cur(info->net);
1926         u8 family = info->nfmsg->nfgen_family;
1927         const struct nft_chain *chain;
1928         struct net *net = info->net;
1929         struct nft_table *table;
1930         struct sk_buff *skb2;
1931         int err;
1932
1933         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
1934                 struct netlink_dump_control c = {
1935                         .dump = nf_tables_dump_chains,
1936                         .module = THIS_MODULE,
1937                 };
1938
1939                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
1940         }
1941
1942         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask, 0);
1943         if (IS_ERR(table)) {
1944                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
1945                 return PTR_ERR(table);
1946         }
1947
1948         chain = nft_chain_lookup(net, table, nla[NFTA_CHAIN_NAME], genmask);
1949         if (IS_ERR(chain)) {
1950                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
1951                 return PTR_ERR(chain);
1952         }
1953
1954         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
1955         if (!skb2)
1956                 return -ENOMEM;
1957
1958         err = nf_tables_fill_chain_info(skb2, net, NETLINK_CB(skb).portid,
1959                                         info->nlh->nlmsg_seq, NFT_MSG_NEWCHAIN,
1960                                         0, family, table, chain, NULL);
1961         if (err < 0)
1962                 goto err_fill_chain_info;
1963
1964         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
1965
1966 err_fill_chain_info:
1967         kfree_skb(skb2);
1968         return err;
1969 }
1970
1971 static const struct nla_policy nft_counter_policy[NFTA_COUNTER_MAX + 1] = {
1972         [NFTA_COUNTER_PACKETS]  = { .type = NLA_U64 },
1973         [NFTA_COUNTER_BYTES]    = { .type = NLA_U64 },
1974 };
1975
1976 static struct nft_stats __percpu *nft_stats_alloc(const struct nlattr *attr)
1977 {
1978         struct nlattr *tb[NFTA_COUNTER_MAX+1];
1979         struct nft_stats __percpu *newstats;
1980         struct nft_stats *stats;
1981         int err;
1982
1983         err = nla_parse_nested_deprecated(tb, NFTA_COUNTER_MAX, attr,
1984                                           nft_counter_policy, NULL);
1985         if (err < 0)
1986                 return ERR_PTR(err);
1987
1988         if (!tb[NFTA_COUNTER_BYTES] || !tb[NFTA_COUNTER_PACKETS])
1989                 return ERR_PTR(-EINVAL);
1990
1991         newstats = netdev_alloc_pcpu_stats(struct nft_stats);
1992         if (newstats == NULL)
1993                 return ERR_PTR(-ENOMEM);
1994
1995         /* Restore old counters on this cpu, no problem. Per-cpu statistics
1996          * are not exposed to userspace.
1997          */
1998         preempt_disable();
1999         stats = this_cpu_ptr(newstats);
2000         stats->bytes = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_BYTES]));
2001         stats->pkts = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_PACKETS]));
2002         preempt_enable();
2003
2004         return newstats;
2005 }
2006
2007 static void nft_chain_stats_replace(struct nft_trans *trans)
2008 {
2009         struct nft_base_chain *chain = nft_base_chain(trans->ctx.chain);
2010
2011         if (!nft_trans_chain_stats(trans))
2012                 return;
2013
2014         nft_trans_chain_stats(trans) =
2015                 rcu_replace_pointer(chain->stats, nft_trans_chain_stats(trans),
2016                                     lockdep_commit_lock_is_held(trans->ctx.net));
2017
2018         if (!nft_trans_chain_stats(trans))
2019                 static_branch_inc(&nft_counters_enabled);
2020 }
2021
2022 static void nf_tables_chain_free_chain_rules(struct nft_chain *chain)
2023 {
2024         struct nft_rule_blob *g0 = rcu_dereference_raw(chain->blob_gen_0);
2025         struct nft_rule_blob *g1 = rcu_dereference_raw(chain->blob_gen_1);
2026
2027         if (g0 != g1)
2028                 kvfree(g1);
2029         kvfree(g0);
2030
2031         /* should be NULL either via abort or via successful commit */
2032         WARN_ON_ONCE(chain->blob_next);
2033         kvfree(chain->blob_next);
2034 }
2035
2036 void nf_tables_chain_destroy(struct nft_ctx *ctx)
2037 {
2038         struct nft_chain *chain = ctx->chain;
2039         struct nft_hook *hook, *next;
2040
2041         if (WARN_ON(chain->use > 0))
2042                 return;
2043
2044         /* no concurrent access possible anymore */
2045         nf_tables_chain_free_chain_rules(chain);
2046
2047         if (nft_is_base_chain(chain)) {
2048                 struct nft_base_chain *basechain = nft_base_chain(chain);
2049
2050                 if (nft_base_chain_netdev(ctx->family, basechain->ops.hooknum)) {
2051                         list_for_each_entry_safe(hook, next,
2052                                                  &basechain->hook_list, list) {
2053                                 list_del_rcu(&hook->list);
2054                                 kfree_rcu(hook, rcu);
2055                         }
2056                 }
2057                 module_put(basechain->type->owner);
2058                 if (rcu_access_pointer(basechain->stats)) {
2059                         static_branch_dec(&nft_counters_enabled);
2060                         free_percpu(rcu_dereference_raw(basechain->stats));
2061                 }
2062                 kfree(chain->name);
2063                 kfree(chain->udata);
2064                 kfree(basechain);
2065         } else {
2066                 kfree(chain->name);
2067                 kfree(chain->udata);
2068                 kfree(chain);
2069         }
2070 }
2071
2072 static struct nft_hook *nft_netdev_hook_alloc(struct net *net,
2073                                               const struct nlattr *attr)
2074 {
2075         struct net_device *dev;
2076         char ifname[IFNAMSIZ];
2077         struct nft_hook *hook;
2078         int err;
2079
2080         hook = kmalloc(sizeof(struct nft_hook), GFP_KERNEL_ACCOUNT);
2081         if (!hook) {
2082                 err = -ENOMEM;
2083                 goto err_hook_alloc;
2084         }
2085
2086         nla_strscpy(ifname, attr, IFNAMSIZ);
2087         /* nf_tables_netdev_event() is called under rtnl_mutex, this is
2088          * indirectly serializing all the other holders of the commit_mutex with
2089          * the rtnl_mutex.
2090          */
2091         dev = __dev_get_by_name(net, ifname);
2092         if (!dev) {
2093                 err = -ENOENT;
2094                 goto err_hook_dev;
2095         }
2096         hook->ops.dev = dev;
2097
2098         return hook;
2099
2100 err_hook_dev:
2101         kfree(hook);
2102 err_hook_alloc:
2103         return ERR_PTR(err);
2104 }
2105
2106 static struct nft_hook *nft_hook_list_find(struct list_head *hook_list,
2107                                            const struct nft_hook *this)
2108 {
2109         struct nft_hook *hook;
2110
2111         list_for_each_entry(hook, hook_list, list) {
2112                 if (this->ops.dev == hook->ops.dev)
2113                         return hook;
2114         }
2115
2116         return NULL;
2117 }
2118
2119 static int nf_tables_parse_netdev_hooks(struct net *net,
2120                                         const struct nlattr *attr,
2121                                         struct list_head *hook_list,
2122                                         struct netlink_ext_ack *extack)
2123 {
2124         struct nft_hook *hook, *next;
2125         const struct nlattr *tmp;
2126         int rem, n = 0, err;
2127
2128         nla_for_each_nested(tmp, attr, rem) {
2129                 if (nla_type(tmp) != NFTA_DEVICE_NAME) {
2130                         err = -EINVAL;
2131                         goto err_hook;
2132                 }
2133
2134                 hook = nft_netdev_hook_alloc(net, tmp);
2135                 if (IS_ERR(hook)) {
2136                         NL_SET_BAD_ATTR(extack, tmp);
2137                         err = PTR_ERR(hook);
2138                         goto err_hook;
2139                 }
2140                 if (nft_hook_list_find(hook_list, hook)) {
2141                         NL_SET_BAD_ATTR(extack, tmp);
2142                         kfree(hook);
2143                         err = -EEXIST;
2144                         goto err_hook;
2145                 }
2146                 list_add_tail(&hook->list, hook_list);
2147                 n++;
2148
2149                 if (n == NFT_NETDEVICE_MAX) {
2150                         err = -EFBIG;
2151                         goto err_hook;
2152                 }
2153         }
2154
2155         return 0;
2156
2157 err_hook:
2158         list_for_each_entry_safe(hook, next, hook_list, list) {
2159                 list_del(&hook->list);
2160                 kfree(hook);
2161         }
2162         return err;
2163 }
2164
2165 struct nft_chain_hook {
2166         u32                             num;
2167         s32                             priority;
2168         const struct nft_chain_type     *type;
2169         struct list_head                list;
2170 };
2171
2172 static int nft_chain_parse_netdev(struct net *net, struct nlattr *tb[],
2173                                   struct list_head *hook_list,
2174                                   struct netlink_ext_ack *extack, u32 flags)
2175 {
2176         struct nft_hook *hook;
2177         int err;
2178
2179         if (tb[NFTA_HOOK_DEV]) {
2180                 hook = nft_netdev_hook_alloc(net, tb[NFTA_HOOK_DEV]);
2181                 if (IS_ERR(hook)) {
2182                         NL_SET_BAD_ATTR(extack, tb[NFTA_HOOK_DEV]);
2183                         return PTR_ERR(hook);
2184                 }
2185
2186                 list_add_tail(&hook->list, hook_list);
2187         } else if (tb[NFTA_HOOK_DEVS]) {
2188                 err = nf_tables_parse_netdev_hooks(net, tb[NFTA_HOOK_DEVS],
2189                                                    hook_list, extack);
2190                 if (err < 0)
2191                         return err;
2192
2193         }
2194
2195         if (flags & NFT_CHAIN_HW_OFFLOAD &&
2196             list_empty(hook_list))
2197                 return -EINVAL;
2198
2199         return 0;
2200 }
2201
2202 static int nft_chain_parse_hook(struct net *net,
2203                                 struct nft_base_chain *basechain,
2204                                 const struct nlattr * const nla[],
2205                                 struct nft_chain_hook *hook, u8 family,
2206                                 u32 flags, struct netlink_ext_ack *extack)
2207 {
2208         struct nftables_pernet *nft_net = nft_pernet(net);
2209         struct nlattr *ha[NFTA_HOOK_MAX + 1];
2210         const struct nft_chain_type *type;
2211         int err;
2212
2213         lockdep_assert_held(&nft_net->commit_mutex);
2214         lockdep_nfnl_nft_mutex_not_held();
2215
2216         err = nla_parse_nested_deprecated(ha, NFTA_HOOK_MAX,
2217                                           nla[NFTA_CHAIN_HOOK],
2218                                           nft_hook_policy, NULL);
2219         if (err < 0)
2220                 return err;
2221
2222         if (!basechain) {
2223                 if (!ha[NFTA_HOOK_HOOKNUM] ||
2224                     !ha[NFTA_HOOK_PRIORITY]) {
2225                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2226                         return -ENOENT;
2227                 }
2228
2229                 hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
2230                 hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
2231
2232                 type = __nft_chain_type_get(family, NFT_CHAIN_T_DEFAULT);
2233                 if (!type)
2234                         return -EOPNOTSUPP;
2235
2236                 if (nla[NFTA_CHAIN_TYPE]) {
2237                         type = nf_tables_chain_type_lookup(net, nla[NFTA_CHAIN_TYPE],
2238                                                            family, true);
2239                         if (IS_ERR(type)) {
2240                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2241                                 return PTR_ERR(type);
2242                         }
2243                 }
2244                 if (hook->num >= NFT_MAX_HOOKS || !(type->hook_mask & (1 << hook->num)))
2245                         return -EOPNOTSUPP;
2246
2247                 if (type->type == NFT_CHAIN_T_NAT &&
2248                     hook->priority <= NF_IP_PRI_CONNTRACK)
2249                         return -EOPNOTSUPP;
2250         } else {
2251                 if (ha[NFTA_HOOK_HOOKNUM]) {
2252                         hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
2253                         if (hook->num != basechain->ops.hooknum)
2254                                 return -EOPNOTSUPP;
2255                 }
2256                 if (ha[NFTA_HOOK_PRIORITY]) {
2257                         hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
2258                         if (hook->priority != basechain->ops.priority)
2259                                 return -EOPNOTSUPP;
2260                 }
2261
2262                 type = basechain->type;
2263         }
2264
2265         if (!try_module_get(type->owner)) {
2266                 if (nla[NFTA_CHAIN_TYPE])
2267                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2268                 return -ENOENT;
2269         }
2270
2271         hook->type = type;
2272
2273         INIT_LIST_HEAD(&hook->list);
2274         if (nft_base_chain_netdev(family, hook->num)) {
2275                 err = nft_chain_parse_netdev(net, ha, &hook->list, extack, flags);
2276                 if (err < 0) {
2277                         module_put(type->owner);
2278                         return err;
2279                 }
2280         } else if (ha[NFTA_HOOK_DEV] || ha[NFTA_HOOK_DEVS]) {
2281                 module_put(type->owner);
2282                 return -EOPNOTSUPP;
2283         }
2284
2285         return 0;
2286 }
2287
2288 static void nft_chain_release_hook(struct nft_chain_hook *hook)
2289 {
2290         struct nft_hook *h, *next;
2291
2292         list_for_each_entry_safe(h, next, &hook->list, list) {
2293                 list_del(&h->list);
2294                 kfree(h);
2295         }
2296         module_put(hook->type->owner);
2297 }
2298
2299 static void nft_last_rule(const struct nft_chain *chain, const void *ptr)
2300 {
2301         struct nft_rule_dp_last *lrule;
2302
2303         BUILD_BUG_ON(offsetof(struct nft_rule_dp_last, end) != 0);
2304
2305         lrule = (struct nft_rule_dp_last *)ptr;
2306         lrule->end.is_last = 1;
2307         lrule->chain = chain;
2308         /* blob size does not include the trailer rule */
2309 }
2310
2311 static struct nft_rule_blob *nf_tables_chain_alloc_rules(const struct nft_chain *chain,
2312                                                          unsigned int size)
2313 {
2314         struct nft_rule_blob *blob;
2315
2316         if (size > INT_MAX)
2317                 return NULL;
2318
2319         size += sizeof(struct nft_rule_blob) + sizeof(struct nft_rule_dp_last);
2320
2321         blob = kvmalloc(size, GFP_KERNEL_ACCOUNT);
2322         if (!blob)
2323                 return NULL;
2324
2325         blob->size = 0;
2326         nft_last_rule(chain, blob->data);
2327
2328         return blob;
2329 }
2330
2331 static void nft_basechain_hook_init(struct nf_hook_ops *ops, u8 family,
2332                                     const struct nft_chain_hook *hook,
2333                                     struct nft_chain *chain)
2334 {
2335         ops->pf                 = family;
2336         ops->hooknum            = hook->num;
2337         ops->priority           = hook->priority;
2338         ops->priv               = chain;
2339         ops->hook               = hook->type->hooks[ops->hooknum];
2340         ops->hook_ops_type      = NF_HOOK_OP_NF_TABLES;
2341 }
2342
2343 static int nft_basechain_init(struct nft_base_chain *basechain, u8 family,
2344                               struct nft_chain_hook *hook, u32 flags)
2345 {
2346         struct nft_chain *chain;
2347         struct nft_hook *h;
2348
2349         basechain->type = hook->type;
2350         INIT_LIST_HEAD(&basechain->hook_list);
2351         chain = &basechain->chain;
2352
2353         if (nft_base_chain_netdev(family, hook->num)) {
2354                 list_splice_init(&hook->list, &basechain->hook_list);
2355                 list_for_each_entry(h, &basechain->hook_list, list)
2356                         nft_basechain_hook_init(&h->ops, family, hook, chain);
2357         }
2358         nft_basechain_hook_init(&basechain->ops, family, hook, chain);
2359
2360         chain->flags |= NFT_CHAIN_BASE | flags;
2361         basechain->policy = NF_ACCEPT;
2362         if (chain->flags & NFT_CHAIN_HW_OFFLOAD &&
2363             !nft_chain_offload_support(basechain)) {
2364                 list_splice_init(&basechain->hook_list, &hook->list);
2365                 return -EOPNOTSUPP;
2366         }
2367
2368         flow_block_init(&basechain->flow_block);
2369
2370         return 0;
2371 }
2372
2373 int nft_chain_add(struct nft_table *table, struct nft_chain *chain)
2374 {
2375         int err;
2376
2377         err = rhltable_insert_key(&table->chains_ht, chain->name,
2378                                   &chain->rhlhead, nft_chain_ht_params);
2379         if (err)
2380                 return err;
2381
2382         list_add_tail_rcu(&chain->list, &table->chains);
2383
2384         return 0;
2385 }
2386
2387 static u64 chain_id;
2388
2389 static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 genmask,
2390                               u8 policy, u32 flags,
2391                               struct netlink_ext_ack *extack)
2392 {
2393         const struct nlattr * const *nla = ctx->nla;
2394         struct nft_table *table = ctx->table;
2395         struct nft_base_chain *basechain;
2396         struct net *net = ctx->net;
2397         char name[NFT_NAME_MAXLEN];
2398         struct nft_rule_blob *blob;
2399         struct nft_trans *trans;
2400         struct nft_chain *chain;
2401         int err;
2402
2403         if (nla[NFTA_CHAIN_HOOK]) {
2404                 struct nft_stats __percpu *stats = NULL;
2405                 struct nft_chain_hook hook = {};
2406
2407                 if (flags & NFT_CHAIN_BINDING)
2408                         return -EOPNOTSUPP;
2409
2410                 err = nft_chain_parse_hook(net, NULL, nla, &hook, family, flags,
2411                                            extack);
2412                 if (err < 0)
2413                         return err;
2414
2415                 basechain = kzalloc(sizeof(*basechain), GFP_KERNEL_ACCOUNT);
2416                 if (basechain == NULL) {
2417                         nft_chain_release_hook(&hook);
2418                         return -ENOMEM;
2419                 }
2420                 chain = &basechain->chain;
2421
2422                 if (nla[NFTA_CHAIN_COUNTERS]) {
2423                         stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2424                         if (IS_ERR(stats)) {
2425                                 nft_chain_release_hook(&hook);
2426                                 kfree(basechain);
2427                                 return PTR_ERR(stats);
2428                         }
2429                         rcu_assign_pointer(basechain->stats, stats);
2430                 }
2431
2432                 err = nft_basechain_init(basechain, family, &hook, flags);
2433                 if (err < 0) {
2434                         nft_chain_release_hook(&hook);
2435                         kfree(basechain);
2436                         free_percpu(stats);
2437                         return err;
2438                 }
2439                 if (stats)
2440                         static_branch_inc(&nft_counters_enabled);
2441         } else {
2442                 if (flags & NFT_CHAIN_BASE)
2443                         return -EINVAL;
2444                 if (flags & NFT_CHAIN_HW_OFFLOAD)
2445                         return -EOPNOTSUPP;
2446
2447                 chain = kzalloc(sizeof(*chain), GFP_KERNEL_ACCOUNT);
2448                 if (chain == NULL)
2449                         return -ENOMEM;
2450
2451                 chain->flags = flags;
2452         }
2453         ctx->chain = chain;
2454
2455         INIT_LIST_HEAD(&chain->rules);
2456         chain->handle = nf_tables_alloc_handle(table);
2457         chain->table = table;
2458
2459         if (nla[NFTA_CHAIN_NAME]) {
2460                 chain->name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
2461         } else {
2462                 if (!(flags & NFT_CHAIN_BINDING)) {
2463                         err = -EINVAL;
2464                         goto err_destroy_chain;
2465                 }
2466
2467                 snprintf(name, sizeof(name), "__chain%llu", ++chain_id);
2468                 chain->name = kstrdup(name, GFP_KERNEL_ACCOUNT);
2469         }
2470
2471         if (!chain->name) {
2472                 err = -ENOMEM;
2473                 goto err_destroy_chain;
2474         }
2475
2476         if (nla[NFTA_CHAIN_USERDATA]) {
2477                 chain->udata = nla_memdup(nla[NFTA_CHAIN_USERDATA], GFP_KERNEL_ACCOUNT);
2478                 if (chain->udata == NULL) {
2479                         err = -ENOMEM;
2480                         goto err_destroy_chain;
2481                 }
2482                 chain->udlen = nla_len(nla[NFTA_CHAIN_USERDATA]);
2483         }
2484
2485         blob = nf_tables_chain_alloc_rules(chain, 0);
2486         if (!blob) {
2487                 err = -ENOMEM;
2488                 goto err_destroy_chain;
2489         }
2490
2491         RCU_INIT_POINTER(chain->blob_gen_0, blob);
2492         RCU_INIT_POINTER(chain->blob_gen_1, blob);
2493
2494         err = nf_tables_register_hook(net, table, chain);
2495         if (err < 0)
2496                 goto err_destroy_chain;
2497
2498         if (!nft_use_inc(&table->use)) {
2499                 err = -EMFILE;
2500                 goto err_use;
2501         }
2502
2503         trans = nft_trans_chain_add(ctx, NFT_MSG_NEWCHAIN);
2504         if (IS_ERR(trans)) {
2505                 err = PTR_ERR(trans);
2506                 goto err_unregister_hook;
2507         }
2508
2509         nft_trans_chain_policy(trans) = NFT_CHAIN_POLICY_UNSET;
2510         if (nft_is_base_chain(chain))
2511                 nft_trans_chain_policy(trans) = policy;
2512
2513         err = nft_chain_add(table, chain);
2514         if (err < 0) {
2515                 nft_trans_destroy(trans);
2516                 goto err_unregister_hook;
2517         }
2518
2519         return 0;
2520
2521 err_unregister_hook:
2522         nft_use_dec_restore(&table->use);
2523 err_use:
2524         nf_tables_unregister_hook(net, table, chain);
2525 err_destroy_chain:
2526         nf_tables_chain_destroy(ctx);
2527
2528         return err;
2529 }
2530
2531 static int nf_tables_updchain(struct nft_ctx *ctx, u8 genmask, u8 policy,
2532                               u32 flags, const struct nlattr *attr,
2533                               struct netlink_ext_ack *extack)
2534 {
2535         const struct nlattr * const *nla = ctx->nla;
2536         struct nft_base_chain *basechain = NULL;
2537         struct nft_table *table = ctx->table;
2538         struct nft_chain *chain = ctx->chain;
2539         struct nft_chain_hook hook = {};
2540         struct nft_stats *stats = NULL;
2541         struct nft_hook *h, *next;
2542         struct nf_hook_ops *ops;
2543         struct nft_trans *trans;
2544         bool unregister = false;
2545         int err;
2546
2547         if (chain->flags ^ flags)
2548                 return -EOPNOTSUPP;
2549
2550         INIT_LIST_HEAD(&hook.list);
2551
2552         if (nla[NFTA_CHAIN_HOOK]) {
2553                 if (!nft_is_base_chain(chain)) {
2554                         NL_SET_BAD_ATTR(extack, attr);
2555                         return -EEXIST;
2556                 }
2557
2558                 basechain = nft_base_chain(chain);
2559                 err = nft_chain_parse_hook(ctx->net, basechain, nla, &hook,
2560                                            ctx->family, flags, extack);
2561                 if (err < 0)
2562                         return err;
2563
2564                 if (basechain->type != hook.type) {
2565                         nft_chain_release_hook(&hook);
2566                         NL_SET_BAD_ATTR(extack, attr);
2567                         return -EEXIST;
2568                 }
2569
2570                 if (nft_base_chain_netdev(ctx->family, basechain->ops.hooknum)) {
2571                         list_for_each_entry_safe(h, next, &hook.list, list) {
2572                                 h->ops.pf       = basechain->ops.pf;
2573                                 h->ops.hooknum  = basechain->ops.hooknum;
2574                                 h->ops.priority = basechain->ops.priority;
2575                                 h->ops.priv     = basechain->ops.priv;
2576                                 h->ops.hook     = basechain->ops.hook;
2577
2578                                 if (nft_hook_list_find(&basechain->hook_list, h)) {
2579                                         list_del(&h->list);
2580                                         kfree(h);
2581                                 }
2582                         }
2583                 } else {
2584                         ops = &basechain->ops;
2585                         if (ops->hooknum != hook.num ||
2586                             ops->priority != hook.priority) {
2587                                 nft_chain_release_hook(&hook);
2588                                 NL_SET_BAD_ATTR(extack, attr);
2589                                 return -EEXIST;
2590                         }
2591                 }
2592         }
2593
2594         if (nla[NFTA_CHAIN_HANDLE] &&
2595             nla[NFTA_CHAIN_NAME]) {
2596                 struct nft_chain *chain2;
2597
2598                 chain2 = nft_chain_lookup(ctx->net, table,
2599                                           nla[NFTA_CHAIN_NAME], genmask);
2600                 if (!IS_ERR(chain2)) {
2601                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2602                         err = -EEXIST;
2603                         goto err_hooks;
2604                 }
2605         }
2606
2607         if (nla[NFTA_CHAIN_COUNTERS]) {
2608                 if (!nft_is_base_chain(chain)) {
2609                         err = -EOPNOTSUPP;
2610                         goto err_hooks;
2611                 }
2612
2613                 stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2614                 if (IS_ERR(stats)) {
2615                         err = PTR_ERR(stats);
2616                         goto err_hooks;
2617                 }
2618         }
2619
2620         if (!(table->flags & NFT_TABLE_F_DORMANT) &&
2621             nft_is_base_chain(chain) &&
2622             !list_empty(&hook.list)) {
2623                 basechain = nft_base_chain(chain);
2624                 ops = &basechain->ops;
2625
2626                 if (nft_base_chain_netdev(table->family, basechain->ops.hooknum)) {
2627                         err = nft_netdev_register_hooks(ctx->net, &hook.list);
2628                         if (err < 0)
2629                                 goto err_hooks;
2630                 }
2631         }
2632
2633         unregister = true;
2634         err = -ENOMEM;
2635         trans = nft_trans_alloc(ctx, NFT_MSG_NEWCHAIN,
2636                                 sizeof(struct nft_trans_chain));
2637         if (trans == NULL)
2638                 goto err_trans;
2639
2640         nft_trans_chain_stats(trans) = stats;
2641         nft_trans_chain_update(trans) = true;
2642
2643         if (nla[NFTA_CHAIN_POLICY])
2644                 nft_trans_chain_policy(trans) = policy;
2645         else
2646                 nft_trans_chain_policy(trans) = -1;
2647
2648         if (nla[NFTA_CHAIN_HANDLE] &&
2649             nla[NFTA_CHAIN_NAME]) {
2650                 struct nftables_pernet *nft_net = nft_pernet(ctx->net);
2651                 struct nft_trans *tmp;
2652                 char *name;
2653
2654                 err = -ENOMEM;
2655                 name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
2656                 if (!name)
2657                         goto err_trans;
2658
2659                 err = -EEXIST;
2660                 list_for_each_entry(tmp, &nft_net->commit_list, list) {
2661                         if (tmp->msg_type == NFT_MSG_NEWCHAIN &&
2662                             tmp->ctx.table == table &&
2663                             nft_trans_chain_update(tmp) &&
2664                             nft_trans_chain_name(tmp) &&
2665                             strcmp(name, nft_trans_chain_name(tmp)) == 0) {
2666                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2667                                 kfree(name);
2668                                 goto err_trans;
2669                         }
2670                 }
2671
2672                 nft_trans_chain_name(trans) = name;
2673         }
2674
2675         nft_trans_basechain(trans) = basechain;
2676         INIT_LIST_HEAD(&nft_trans_chain_hooks(trans));
2677         list_splice(&hook.list, &nft_trans_chain_hooks(trans));
2678         if (nla[NFTA_CHAIN_HOOK])
2679                 module_put(hook.type->owner);
2680
2681         nft_trans_commit_list_add_tail(ctx->net, trans);
2682
2683         return 0;
2684
2685 err_trans:
2686         free_percpu(stats);
2687         kfree(trans);
2688 err_hooks:
2689         if (nla[NFTA_CHAIN_HOOK]) {
2690                 list_for_each_entry_safe(h, next, &hook.list, list) {
2691                         if (unregister)
2692                                 nf_unregister_net_hook(ctx->net, &h->ops);
2693                         list_del(&h->list);
2694                         kfree_rcu(h, rcu);
2695                 }
2696                 module_put(hook.type->owner);
2697         }
2698
2699         return err;
2700 }
2701
2702 static struct nft_chain *nft_chain_lookup_byid(const struct net *net,
2703                                                const struct nft_table *table,
2704                                                const struct nlattr *nla, u8 genmask)
2705 {
2706         struct nftables_pernet *nft_net = nft_pernet(net);
2707         u32 id = ntohl(nla_get_be32(nla));
2708         struct nft_trans *trans;
2709
2710         list_for_each_entry(trans, &nft_net->commit_list, list) {
2711                 struct nft_chain *chain = trans->ctx.chain;
2712
2713                 if (trans->msg_type == NFT_MSG_NEWCHAIN &&
2714                     chain->table == table &&
2715                     id == nft_trans_chain_id(trans) &&
2716                     nft_active_genmask(chain, genmask))
2717                         return chain;
2718         }
2719         return ERR_PTR(-ENOENT);
2720 }
2721
2722 static int nf_tables_newchain(struct sk_buff *skb, const struct nfnl_info *info,
2723                               const struct nlattr * const nla[])
2724 {
2725         struct nftables_pernet *nft_net = nft_pernet(info->net);
2726         struct netlink_ext_ack *extack = info->extack;
2727         u8 genmask = nft_genmask_next(info->net);
2728         u8 family = info->nfmsg->nfgen_family;
2729         struct nft_chain *chain = NULL;
2730         struct net *net = info->net;
2731         const struct nlattr *attr;
2732         struct nft_table *table;
2733         u8 policy = NF_ACCEPT;
2734         struct nft_ctx ctx;
2735         u64 handle = 0;
2736         u32 flags = 0;
2737
2738         lockdep_assert_held(&nft_net->commit_mutex);
2739
2740         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
2741                                  NETLINK_CB(skb).portid);
2742         if (IS_ERR(table)) {
2743                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2744                 return PTR_ERR(table);
2745         }
2746
2747         chain = NULL;
2748         attr = nla[NFTA_CHAIN_NAME];
2749
2750         if (nla[NFTA_CHAIN_HANDLE]) {
2751                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_CHAIN_HANDLE]));
2752                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2753                 if (IS_ERR(chain)) {
2754                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_HANDLE]);
2755                         return PTR_ERR(chain);
2756                 }
2757                 attr = nla[NFTA_CHAIN_HANDLE];
2758         } else if (nla[NFTA_CHAIN_NAME]) {
2759                 chain = nft_chain_lookup(net, table, attr, genmask);
2760                 if (IS_ERR(chain)) {
2761                         if (PTR_ERR(chain) != -ENOENT) {
2762                                 NL_SET_BAD_ATTR(extack, attr);
2763                                 return PTR_ERR(chain);
2764                         }
2765                         chain = NULL;
2766                 }
2767         } else if (!nla[NFTA_CHAIN_ID]) {
2768                 return -EINVAL;
2769         }
2770
2771         if (nla[NFTA_CHAIN_POLICY]) {
2772                 if (chain != NULL &&
2773                     !nft_is_base_chain(chain)) {
2774                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2775                         return -EOPNOTSUPP;
2776                 }
2777
2778                 if (chain == NULL &&
2779                     nla[NFTA_CHAIN_HOOK] == NULL) {
2780                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2781                         return -EOPNOTSUPP;
2782                 }
2783
2784                 policy = ntohl(nla_get_be32(nla[NFTA_CHAIN_POLICY]));
2785                 switch (policy) {
2786                 case NF_DROP:
2787                 case NF_ACCEPT:
2788                         break;
2789                 default:
2790                         return -EINVAL;
2791                 }
2792         }
2793
2794         if (nla[NFTA_CHAIN_FLAGS])
2795                 flags = ntohl(nla_get_be32(nla[NFTA_CHAIN_FLAGS]));
2796         else if (chain)
2797                 flags = chain->flags;
2798
2799         if (flags & ~NFT_CHAIN_FLAGS)
2800                 return -EOPNOTSUPP;
2801
2802         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
2803
2804         if (chain != NULL) {
2805                 if (chain->flags & NFT_CHAIN_BINDING)
2806                         return -EINVAL;
2807
2808                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
2809                         NL_SET_BAD_ATTR(extack, attr);
2810                         return -EEXIST;
2811                 }
2812                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
2813                         return -EOPNOTSUPP;
2814
2815                 flags |= chain->flags & NFT_CHAIN_BASE;
2816                 return nf_tables_updchain(&ctx, genmask, policy, flags, attr,
2817                                           extack);
2818         }
2819
2820         return nf_tables_addchain(&ctx, family, genmask, policy, flags, extack);
2821 }
2822
2823 static int nft_delchain_hook(struct nft_ctx *ctx,
2824                              struct nft_base_chain *basechain,
2825                              struct netlink_ext_ack *extack)
2826 {
2827         const struct nft_chain *chain = &basechain->chain;
2828         const struct nlattr * const *nla = ctx->nla;
2829         struct nft_chain_hook chain_hook = {};
2830         struct nft_hook *this, *hook;
2831         LIST_HEAD(chain_del_list);
2832         struct nft_trans *trans;
2833         int err;
2834
2835         err = nft_chain_parse_hook(ctx->net, basechain, nla, &chain_hook,
2836                                    ctx->family, chain->flags, extack);
2837         if (err < 0)
2838                 return err;
2839
2840         list_for_each_entry(this, &chain_hook.list, list) {
2841                 hook = nft_hook_list_find(&basechain->hook_list, this);
2842                 if (!hook) {
2843                         err = -ENOENT;
2844                         goto err_chain_del_hook;
2845                 }
2846                 list_move(&hook->list, &chain_del_list);
2847         }
2848
2849         trans = nft_trans_alloc(ctx, NFT_MSG_DELCHAIN,
2850                                 sizeof(struct nft_trans_chain));
2851         if (!trans) {
2852                 err = -ENOMEM;
2853                 goto err_chain_del_hook;
2854         }
2855
2856         nft_trans_basechain(trans) = basechain;
2857         nft_trans_chain_update(trans) = true;
2858         INIT_LIST_HEAD(&nft_trans_chain_hooks(trans));
2859         list_splice(&chain_del_list, &nft_trans_chain_hooks(trans));
2860         nft_chain_release_hook(&chain_hook);
2861
2862         nft_trans_commit_list_add_tail(ctx->net, trans);
2863
2864         return 0;
2865
2866 err_chain_del_hook:
2867         list_splice(&chain_del_list, &basechain->hook_list);
2868         nft_chain_release_hook(&chain_hook);
2869
2870         return err;
2871 }
2872
2873 static int nf_tables_delchain(struct sk_buff *skb, const struct nfnl_info *info,
2874                               const struct nlattr * const nla[])
2875 {
2876         struct netlink_ext_ack *extack = info->extack;
2877         u8 genmask = nft_genmask_next(info->net);
2878         u8 family = info->nfmsg->nfgen_family;
2879         struct net *net = info->net;
2880         const struct nlattr *attr;
2881         struct nft_table *table;
2882         struct nft_chain *chain;
2883         struct nft_rule *rule;
2884         struct nft_ctx ctx;
2885         u64 handle;
2886         u32 use;
2887         int err;
2888
2889         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
2890                                  NETLINK_CB(skb).portid);
2891         if (IS_ERR(table)) {
2892                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2893                 return PTR_ERR(table);
2894         }
2895
2896         if (nla[NFTA_CHAIN_HANDLE]) {
2897                 attr = nla[NFTA_CHAIN_HANDLE];
2898                 handle = be64_to_cpu(nla_get_be64(attr));
2899                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2900         } else {
2901                 attr = nla[NFTA_CHAIN_NAME];
2902                 chain = nft_chain_lookup(net, table, attr, genmask);
2903         }
2904         if (IS_ERR(chain)) {
2905                 if (PTR_ERR(chain) == -ENOENT &&
2906                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYCHAIN)
2907                         return 0;
2908
2909                 NL_SET_BAD_ATTR(extack, attr);
2910                 return PTR_ERR(chain);
2911         }
2912
2913         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
2914
2915         if (nla[NFTA_CHAIN_HOOK]) {
2916                 if (chain->flags & NFT_CHAIN_HW_OFFLOAD)
2917                         return -EOPNOTSUPP;
2918
2919                 if (nft_is_base_chain(chain)) {
2920                         struct nft_base_chain *basechain = nft_base_chain(chain);
2921
2922                         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
2923                                 return nft_delchain_hook(&ctx, basechain, extack);
2924                 }
2925         }
2926
2927         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
2928             chain->use > 0)
2929                 return -EBUSY;
2930
2931         use = chain->use;
2932         list_for_each_entry(rule, &chain->rules, list) {
2933                 if (!nft_is_active_next(net, rule))
2934                         continue;
2935                 use--;
2936
2937                 err = nft_delrule(&ctx, rule);
2938                 if (err < 0)
2939                         return err;
2940         }
2941
2942         /* There are rules and elements that are still holding references to us,
2943          * we cannot do a recursive removal in this case.
2944          */
2945         if (use > 0) {
2946                 NL_SET_BAD_ATTR(extack, attr);
2947                 return -EBUSY;
2948         }
2949
2950         return nft_delchain(&ctx);
2951 }
2952
2953 /*
2954  * Expressions
2955  */
2956
2957 /**
2958  *      nft_register_expr - register nf_tables expr type
2959  *      @type: expr type
2960  *
2961  *      Registers the expr type for use with nf_tables. Returns zero on
2962  *      success or a negative errno code otherwise.
2963  */
2964 int nft_register_expr(struct nft_expr_type *type)
2965 {
2966         nfnl_lock(NFNL_SUBSYS_NFTABLES);
2967         if (type->family == NFPROTO_UNSPEC)
2968                 list_add_tail_rcu(&type->list, &nf_tables_expressions);
2969         else
2970                 list_add_rcu(&type->list, &nf_tables_expressions);
2971         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
2972         return 0;
2973 }
2974 EXPORT_SYMBOL_GPL(nft_register_expr);
2975
2976 /**
2977  *      nft_unregister_expr - unregister nf_tables expr type
2978  *      @type: expr type
2979  *
2980  *      Unregisters the expr typefor use with nf_tables.
2981  */
2982 void nft_unregister_expr(struct nft_expr_type *type)
2983 {
2984         nfnl_lock(NFNL_SUBSYS_NFTABLES);
2985         list_del_rcu(&type->list);
2986         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
2987 }
2988 EXPORT_SYMBOL_GPL(nft_unregister_expr);
2989
2990 static const struct nft_expr_type *__nft_expr_type_get(u8 family,
2991                                                        struct nlattr *nla)
2992 {
2993         const struct nft_expr_type *type, *candidate = NULL;
2994
2995         list_for_each_entry(type, &nf_tables_expressions, list) {
2996                 if (!nla_strcmp(nla, type->name)) {
2997                         if (!type->family && !candidate)
2998                                 candidate = type;
2999                         else if (type->family == family)
3000                                 candidate = type;
3001                 }
3002         }
3003         return candidate;
3004 }
3005
3006 #ifdef CONFIG_MODULES
3007 static int nft_expr_type_request_module(struct net *net, u8 family,
3008                                         struct nlattr *nla)
3009 {
3010         if (nft_request_module(net, "nft-expr-%u-%.*s", family,
3011                                nla_len(nla), (char *)nla_data(nla)) == -EAGAIN)
3012                 return -EAGAIN;
3013
3014         return 0;
3015 }
3016 #endif
3017
3018 static const struct nft_expr_type *nft_expr_type_get(struct net *net,
3019                                                      u8 family,
3020                                                      struct nlattr *nla)
3021 {
3022         const struct nft_expr_type *type;
3023
3024         if (nla == NULL)
3025                 return ERR_PTR(-EINVAL);
3026
3027         type = __nft_expr_type_get(family, nla);
3028         if (type != NULL && try_module_get(type->owner))
3029                 return type;
3030
3031         lockdep_nfnl_nft_mutex_not_held();
3032 #ifdef CONFIG_MODULES
3033         if (type == NULL) {
3034                 if (nft_expr_type_request_module(net, family, nla) == -EAGAIN)
3035                         return ERR_PTR(-EAGAIN);
3036
3037                 if (nft_request_module(net, "nft-expr-%.*s",
3038                                        nla_len(nla),
3039                                        (char *)nla_data(nla)) == -EAGAIN)
3040                         return ERR_PTR(-EAGAIN);
3041         }
3042 #endif
3043         return ERR_PTR(-ENOENT);
3044 }
3045
3046 static const struct nla_policy nft_expr_policy[NFTA_EXPR_MAX + 1] = {
3047         [NFTA_EXPR_NAME]        = { .type = NLA_STRING,
3048                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
3049         [NFTA_EXPR_DATA]        = { .type = NLA_NESTED },
3050 };
3051
3052 static int nf_tables_fill_expr_info(struct sk_buff *skb,
3053                                     const struct nft_expr *expr, bool reset)
3054 {
3055         if (nla_put_string(skb, NFTA_EXPR_NAME, expr->ops->type->name))
3056                 goto nla_put_failure;
3057
3058         if (expr->ops->dump) {
3059                 struct nlattr *data = nla_nest_start_noflag(skb,
3060                                                             NFTA_EXPR_DATA);
3061                 if (data == NULL)
3062                         goto nla_put_failure;
3063                 if (expr->ops->dump(skb, expr, reset) < 0)
3064                         goto nla_put_failure;
3065                 nla_nest_end(skb, data);
3066         }
3067
3068         return skb->len;
3069
3070 nla_put_failure:
3071         return -1;
3072 };
3073
3074 int nft_expr_dump(struct sk_buff *skb, unsigned int attr,
3075                   const struct nft_expr *expr, bool reset)
3076 {
3077         struct nlattr *nest;
3078
3079         nest = nla_nest_start_noflag(skb, attr);
3080         if (!nest)
3081                 goto nla_put_failure;
3082         if (nf_tables_fill_expr_info(skb, expr, reset) < 0)
3083                 goto nla_put_failure;
3084         nla_nest_end(skb, nest);
3085         return 0;
3086
3087 nla_put_failure:
3088         return -1;
3089 }
3090
3091 struct nft_expr_info {
3092         const struct nft_expr_ops       *ops;
3093         const struct nlattr             *attr;
3094         struct nlattr                   *tb[NFT_EXPR_MAXATTR + 1];
3095 };
3096
3097 static int nf_tables_expr_parse(const struct nft_ctx *ctx,
3098                                 const struct nlattr *nla,
3099                                 struct nft_expr_info *info)
3100 {
3101         const struct nft_expr_type *type;
3102         const struct nft_expr_ops *ops;
3103         struct nlattr *tb[NFTA_EXPR_MAX + 1];
3104         int err;
3105
3106         err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
3107                                           nft_expr_policy, NULL);
3108         if (err < 0)
3109                 return err;
3110
3111         type = nft_expr_type_get(ctx->net, ctx->family, tb[NFTA_EXPR_NAME]);
3112         if (IS_ERR(type))
3113                 return PTR_ERR(type);
3114
3115         if (tb[NFTA_EXPR_DATA]) {
3116                 err = nla_parse_nested_deprecated(info->tb, type->maxattr,
3117                                                   tb[NFTA_EXPR_DATA],
3118                                                   type->policy, NULL);
3119                 if (err < 0)
3120                         goto err1;
3121         } else
3122                 memset(info->tb, 0, sizeof(info->tb[0]) * (type->maxattr + 1));
3123
3124         if (type->select_ops != NULL) {
3125                 ops = type->select_ops(ctx,
3126                                        (const struct nlattr * const *)info->tb);
3127                 if (IS_ERR(ops)) {
3128                         err = PTR_ERR(ops);
3129 #ifdef CONFIG_MODULES
3130                         if (err == -EAGAIN)
3131                                 if (nft_expr_type_request_module(ctx->net,
3132                                                                  ctx->family,
3133                                                                  tb[NFTA_EXPR_NAME]) != -EAGAIN)
3134                                         err = -ENOENT;
3135 #endif
3136                         goto err1;
3137                 }
3138         } else
3139                 ops = type->ops;
3140
3141         info->attr = nla;
3142         info->ops = ops;
3143
3144         return 0;
3145
3146 err1:
3147         module_put(type->owner);
3148         return err;
3149 }
3150
3151 int nft_expr_inner_parse(const struct nft_ctx *ctx, const struct nlattr *nla,
3152                          struct nft_expr_info *info)
3153 {
3154         struct nlattr *tb[NFTA_EXPR_MAX + 1];
3155         const struct nft_expr_type *type;
3156         int err;
3157
3158         err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
3159                                           nft_expr_policy, NULL);
3160         if (err < 0)
3161                 return err;
3162
3163         if (!tb[NFTA_EXPR_DATA])
3164                 return -EINVAL;
3165
3166         type = __nft_expr_type_get(ctx->family, tb[NFTA_EXPR_NAME]);
3167         if (!type)
3168                 return -ENOENT;
3169
3170         if (!type->inner_ops)
3171                 return -EOPNOTSUPP;
3172
3173         err = nla_parse_nested_deprecated(info->tb, type->maxattr,
3174                                           tb[NFTA_EXPR_DATA],
3175                                           type->policy, NULL);
3176         if (err < 0)
3177                 goto err_nla_parse;
3178
3179         info->attr = nla;
3180         info->ops = type->inner_ops;
3181
3182         return 0;
3183
3184 err_nla_parse:
3185         return err;
3186 }
3187
3188 static int nf_tables_newexpr(const struct nft_ctx *ctx,
3189                              const struct nft_expr_info *expr_info,
3190                              struct nft_expr *expr)
3191 {
3192         const struct nft_expr_ops *ops = expr_info->ops;
3193         int err;
3194
3195         expr->ops = ops;
3196         if (ops->init) {
3197                 err = ops->init(ctx, expr, (const struct nlattr **)expr_info->tb);
3198                 if (err < 0)
3199                         goto err1;
3200         }
3201
3202         return 0;
3203 err1:
3204         expr->ops = NULL;
3205         return err;
3206 }
3207
3208 static void nf_tables_expr_destroy(const struct nft_ctx *ctx,
3209                                    struct nft_expr *expr)
3210 {
3211         const struct nft_expr_type *type = expr->ops->type;
3212
3213         if (expr->ops->destroy)
3214                 expr->ops->destroy(ctx, expr);
3215         module_put(type->owner);
3216 }
3217
3218 static struct nft_expr *nft_expr_init(const struct nft_ctx *ctx,
3219                                       const struct nlattr *nla)
3220 {
3221         struct nft_expr_info expr_info;
3222         struct nft_expr *expr;
3223         struct module *owner;
3224         int err;
3225
3226         err = nf_tables_expr_parse(ctx, nla, &expr_info);
3227         if (err < 0)
3228                 goto err_expr_parse;
3229
3230         err = -EOPNOTSUPP;
3231         if (!(expr_info.ops->type->flags & NFT_EXPR_STATEFUL))
3232                 goto err_expr_stateful;
3233
3234         err = -ENOMEM;
3235         expr = kzalloc(expr_info.ops->size, GFP_KERNEL_ACCOUNT);
3236         if (expr == NULL)
3237                 goto err_expr_stateful;
3238
3239         err = nf_tables_newexpr(ctx, &expr_info, expr);
3240         if (err < 0)
3241                 goto err_expr_new;
3242
3243         return expr;
3244 err_expr_new:
3245         kfree(expr);
3246 err_expr_stateful:
3247         owner = expr_info.ops->type->owner;
3248         if (expr_info.ops->type->release_ops)
3249                 expr_info.ops->type->release_ops(expr_info.ops);
3250
3251         module_put(owner);
3252 err_expr_parse:
3253         return ERR_PTR(err);
3254 }
3255
3256 int nft_expr_clone(struct nft_expr *dst, struct nft_expr *src)
3257 {
3258         int err;
3259
3260         if (src->ops->clone) {
3261                 dst->ops = src->ops;
3262                 err = src->ops->clone(dst, src);
3263                 if (err < 0)
3264                         return err;
3265         } else {
3266                 memcpy(dst, src, src->ops->size);
3267         }
3268
3269         __module_get(src->ops->type->owner);
3270
3271         return 0;
3272 }
3273
3274 void nft_expr_destroy(const struct nft_ctx *ctx, struct nft_expr *expr)
3275 {
3276         nf_tables_expr_destroy(ctx, expr);
3277         kfree(expr);
3278 }
3279
3280 /*
3281  * Rules
3282  */
3283
3284 static struct nft_rule *__nft_rule_lookup(const struct nft_chain *chain,
3285                                           u64 handle)
3286 {
3287         struct nft_rule *rule;
3288
3289         // FIXME: this sucks
3290         list_for_each_entry_rcu(rule, &chain->rules, list) {
3291                 if (handle == rule->handle)
3292                         return rule;
3293         }
3294
3295         return ERR_PTR(-ENOENT);
3296 }
3297
3298 static struct nft_rule *nft_rule_lookup(const struct nft_chain *chain,
3299                                         const struct nlattr *nla)
3300 {
3301         if (nla == NULL)
3302                 return ERR_PTR(-EINVAL);
3303
3304         return __nft_rule_lookup(chain, be64_to_cpu(nla_get_be64(nla)));
3305 }
3306
3307 static const struct nla_policy nft_rule_policy[NFTA_RULE_MAX + 1] = {
3308         [NFTA_RULE_TABLE]       = { .type = NLA_STRING,
3309                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
3310         [NFTA_RULE_CHAIN]       = { .type = NLA_STRING,
3311                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
3312         [NFTA_RULE_HANDLE]      = { .type = NLA_U64 },
3313         [NFTA_RULE_EXPRESSIONS] = { .type = NLA_NESTED },
3314         [NFTA_RULE_COMPAT]      = { .type = NLA_NESTED },
3315         [NFTA_RULE_POSITION]    = { .type = NLA_U64 },
3316         [NFTA_RULE_USERDATA]    = { .type = NLA_BINARY,
3317                                     .len = NFT_USERDATA_MAXLEN },
3318         [NFTA_RULE_ID]          = { .type = NLA_U32 },
3319         [NFTA_RULE_POSITION_ID] = { .type = NLA_U32 },
3320         [NFTA_RULE_CHAIN_ID]    = { .type = NLA_U32 },
3321 };
3322
3323 static int nf_tables_fill_rule_info(struct sk_buff *skb, struct net *net,
3324                                     u32 portid, u32 seq, int event,
3325                                     u32 flags, int family,
3326                                     const struct nft_table *table,
3327                                     const struct nft_chain *chain,
3328                                     const struct nft_rule *rule, u64 handle,
3329                                     bool reset)
3330 {
3331         struct nlmsghdr *nlh;
3332         const struct nft_expr *expr, *next;
3333         struct nlattr *list;
3334         u16 type = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
3335
3336         nlh = nfnl_msg_put(skb, portid, seq, type, flags, family, NFNETLINK_V0,
3337                            nft_base_seq(net));
3338         if (!nlh)
3339                 goto nla_put_failure;
3340
3341         if (nla_put_string(skb, NFTA_RULE_TABLE, table->name))
3342                 goto nla_put_failure;
3343         if (nla_put_string(skb, NFTA_RULE_CHAIN, chain->name))
3344                 goto nla_put_failure;
3345         if (nla_put_be64(skb, NFTA_RULE_HANDLE, cpu_to_be64(rule->handle),
3346                          NFTA_RULE_PAD))
3347                 goto nla_put_failure;
3348
3349         if (event != NFT_MSG_DELRULE && handle) {
3350                 if (nla_put_be64(skb, NFTA_RULE_POSITION, cpu_to_be64(handle),
3351                                  NFTA_RULE_PAD))
3352                         goto nla_put_failure;
3353         }
3354
3355         if (chain->flags & NFT_CHAIN_HW_OFFLOAD)
3356                 nft_flow_rule_stats(chain, rule);
3357
3358         list = nla_nest_start_noflag(skb, NFTA_RULE_EXPRESSIONS);
3359         if (list == NULL)
3360                 goto nla_put_failure;
3361         nft_rule_for_each_expr(expr, next, rule) {
3362                 if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr, reset) < 0)
3363                         goto nla_put_failure;
3364         }
3365         nla_nest_end(skb, list);
3366
3367         if (rule->udata) {
3368                 struct nft_userdata *udata = nft_userdata(rule);
3369                 if (nla_put(skb, NFTA_RULE_USERDATA, udata->len + 1,
3370                             udata->data) < 0)
3371                         goto nla_put_failure;
3372         }
3373
3374         nlmsg_end(skb, nlh);
3375         return 0;
3376
3377 nla_put_failure:
3378         nlmsg_trim(skb, nlh);
3379         return -1;
3380 }
3381
3382 static void nf_tables_rule_notify(const struct nft_ctx *ctx,
3383                                   const struct nft_rule *rule, int event)
3384 {
3385         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
3386         const struct nft_rule *prule;
3387         struct sk_buff *skb;
3388         u64 handle = 0;
3389         u16 flags = 0;
3390         int err;
3391
3392         if (!ctx->report &&
3393             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
3394                 return;
3395
3396         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
3397         if (skb == NULL)
3398                 goto err;
3399
3400         if (event == NFT_MSG_NEWRULE &&
3401             !list_is_first(&rule->list, &ctx->chain->rules) &&
3402             !list_is_last(&rule->list, &ctx->chain->rules)) {
3403                 prule = list_prev_entry(rule, list);
3404                 handle = prule->handle;
3405         }
3406         if (ctx->flags & (NLM_F_APPEND | NLM_F_REPLACE))
3407                 flags |= NLM_F_APPEND;
3408         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
3409                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
3410
3411         err = nf_tables_fill_rule_info(skb, ctx->net, ctx->portid, ctx->seq,
3412                                        event, flags, ctx->family, ctx->table,
3413                                        ctx->chain, rule, handle, false);
3414         if (err < 0) {
3415                 kfree_skb(skb);
3416                 goto err;
3417         }
3418
3419         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
3420         return;
3421 err:
3422         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
3423 }
3424
3425 static void audit_log_rule_reset(const struct nft_table *table,
3426                                  unsigned int base_seq,
3427                                  unsigned int nentries)
3428 {
3429         char *buf = kasprintf(GFP_ATOMIC, "%s:%u",
3430                               table->name, base_seq);
3431
3432         audit_log_nfcfg(buf, table->family, nentries,
3433                         AUDIT_NFT_OP_RULE_RESET, GFP_ATOMIC);
3434         kfree(buf);
3435 }
3436
3437 struct nft_rule_dump_ctx {
3438         char *table;
3439         char *chain;
3440 };
3441
3442 static int __nf_tables_dump_rules(struct sk_buff *skb,
3443                                   unsigned int *idx,
3444                                   struct netlink_callback *cb,
3445                                   const struct nft_table *table,
3446                                   const struct nft_chain *chain,
3447                                   bool reset)
3448 {
3449         struct net *net = sock_net(skb->sk);
3450         const struct nft_rule *rule, *prule;
3451         unsigned int s_idx = cb->args[0];
3452         u64 handle;
3453
3454         prule = NULL;
3455         list_for_each_entry_rcu(rule, &chain->rules, list) {
3456                 if (!nft_is_active(net, rule))
3457                         goto cont_skip;
3458                 if (*idx < s_idx)
3459                         goto cont;
3460                 if (*idx > s_idx) {
3461                         memset(&cb->args[1], 0,
3462                                         sizeof(cb->args) - sizeof(cb->args[0]));
3463                 }
3464                 if (prule)
3465                         handle = prule->handle;
3466                 else
3467                         handle = 0;
3468
3469                 if (nf_tables_fill_rule_info(skb, net, NETLINK_CB(cb->skb).portid,
3470                                         cb->nlh->nlmsg_seq,
3471                                         NFT_MSG_NEWRULE,
3472                                         NLM_F_MULTI | NLM_F_APPEND,
3473                                         table->family,
3474                                         table, chain, rule, handle, reset) < 0)
3475                         return 1;
3476
3477                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
3478 cont:
3479                 prule = rule;
3480 cont_skip:
3481                 (*idx)++;
3482         }
3483         return 0;
3484 }
3485
3486 static int nf_tables_dump_rules(struct sk_buff *skb,
3487                                 struct netlink_callback *cb)
3488 {
3489         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
3490         const struct nft_rule_dump_ctx *ctx = cb->data;
3491         struct nft_table *table;
3492         const struct nft_chain *chain;
3493         unsigned int idx = 0;
3494         struct net *net = sock_net(skb->sk);
3495         int family = nfmsg->nfgen_family;
3496         struct nftables_pernet *nft_net;
3497         bool reset = false;
3498
3499         if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETRULE_RESET)
3500                 reset = true;
3501
3502         rcu_read_lock();
3503         nft_net = nft_pernet(net);
3504         cb->seq = READ_ONCE(nft_net->base_seq);
3505
3506         list_for_each_entry_rcu(table, &nft_net->tables, list) {
3507                 if (family != NFPROTO_UNSPEC && family != table->family)
3508                         continue;
3509
3510                 if (ctx && ctx->table && strcmp(ctx->table, table->name) != 0)
3511                         continue;
3512
3513                 if (ctx && ctx->table && ctx->chain) {
3514                         struct rhlist_head *list, *tmp;
3515
3516                         list = rhltable_lookup(&table->chains_ht, ctx->chain,
3517                                                nft_chain_ht_params);
3518                         if (!list)
3519                                 goto done;
3520
3521                         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
3522                                 if (!nft_is_active(net, chain))
3523                                         continue;
3524                                 __nf_tables_dump_rules(skb, &idx,
3525                                                        cb, table, chain, reset);
3526                                 break;
3527                         }
3528                         goto done;
3529                 }
3530
3531                 list_for_each_entry_rcu(chain, &table->chains, list) {
3532                         if (__nf_tables_dump_rules(skb, &idx,
3533                                                    cb, table, chain, reset))
3534                                 goto done;
3535                 }
3536
3537                 if (ctx && ctx->table)
3538                         break;
3539         }
3540 done:
3541         rcu_read_unlock();
3542
3543         if (reset && idx > cb->args[0])
3544                 audit_log_rule_reset(table, cb->seq, idx - cb->args[0]);
3545
3546         cb->args[0] = idx;
3547         return skb->len;
3548 }
3549
3550 static int nf_tables_dump_rules_start(struct netlink_callback *cb)
3551 {
3552         const struct nlattr * const *nla = cb->data;
3553         struct nft_rule_dump_ctx *ctx = NULL;
3554
3555         if (nla[NFTA_RULE_TABLE] || nla[NFTA_RULE_CHAIN]) {
3556                 ctx = kzalloc(sizeof(*ctx), GFP_ATOMIC);
3557                 if (!ctx)
3558                         return -ENOMEM;
3559
3560                 if (nla[NFTA_RULE_TABLE]) {
3561                         ctx->table = nla_strdup(nla[NFTA_RULE_TABLE],
3562                                                         GFP_ATOMIC);
3563                         if (!ctx->table) {
3564                                 kfree(ctx);
3565                                 return -ENOMEM;
3566                         }
3567                 }
3568                 if (nla[NFTA_RULE_CHAIN]) {
3569                         ctx->chain = nla_strdup(nla[NFTA_RULE_CHAIN],
3570                                                 GFP_ATOMIC);
3571                         if (!ctx->chain) {
3572                                 kfree(ctx->table);
3573                                 kfree(ctx);
3574                                 return -ENOMEM;
3575                         }
3576                 }
3577         }
3578
3579         cb->data = ctx;
3580         return 0;
3581 }
3582
3583 static int nf_tables_dump_rules_done(struct netlink_callback *cb)
3584 {
3585         struct nft_rule_dump_ctx *ctx = cb->data;
3586
3587         if (ctx) {
3588                 kfree(ctx->table);
3589                 kfree(ctx->chain);
3590                 kfree(ctx);
3591         }
3592         return 0;
3593 }
3594
3595 /* called with rcu_read_lock held */
3596 static int nf_tables_getrule(struct sk_buff *skb, const struct nfnl_info *info,
3597                              const struct nlattr * const nla[])
3598 {
3599         struct netlink_ext_ack *extack = info->extack;
3600         u8 genmask = nft_genmask_cur(info->net);
3601         u8 family = info->nfmsg->nfgen_family;
3602         const struct nft_chain *chain;
3603         const struct nft_rule *rule;
3604         struct net *net = info->net;
3605         struct nft_table *table;
3606         struct sk_buff *skb2;
3607         bool reset = false;
3608         int err;
3609
3610         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
3611                 struct netlink_dump_control c = {
3612                         .start= nf_tables_dump_rules_start,
3613                         .dump = nf_tables_dump_rules,
3614                         .done = nf_tables_dump_rules_done,
3615                         .module = THIS_MODULE,
3616                         .data = (void *)nla,
3617                 };
3618
3619                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
3620         }
3621
3622         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask, 0);
3623         if (IS_ERR(table)) {
3624                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3625                 return PTR_ERR(table);
3626         }
3627
3628         chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN], genmask);
3629         if (IS_ERR(chain)) {
3630                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3631                 return PTR_ERR(chain);
3632         }
3633
3634         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
3635         if (IS_ERR(rule)) {
3636                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3637                 return PTR_ERR(rule);
3638         }
3639
3640         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
3641         if (!skb2)
3642                 return -ENOMEM;
3643
3644         if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETRULE_RESET)
3645                 reset = true;
3646
3647         err = nf_tables_fill_rule_info(skb2, net, NETLINK_CB(skb).portid,
3648                                        info->nlh->nlmsg_seq, NFT_MSG_NEWRULE, 0,
3649                                        family, table, chain, rule, 0, reset);
3650         if (err < 0)
3651                 goto err_fill_rule_info;
3652
3653         if (reset)
3654                 audit_log_rule_reset(table, nft_pernet(net)->base_seq, 1);
3655
3656         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
3657
3658 err_fill_rule_info:
3659         kfree_skb(skb2);
3660         return err;
3661 }
3662
3663 void nf_tables_rule_destroy(const struct nft_ctx *ctx, struct nft_rule *rule)
3664 {
3665         struct nft_expr *expr, *next;
3666
3667         /*
3668          * Careful: some expressions might not be initialized in case this
3669          * is called on error from nf_tables_newrule().
3670          */
3671         expr = nft_expr_first(rule);
3672         while (nft_expr_more(rule, expr)) {
3673                 next = nft_expr_next(expr);
3674                 nf_tables_expr_destroy(ctx, expr);
3675                 expr = next;
3676         }
3677         kfree(rule);
3678 }
3679
3680 static void nf_tables_rule_release(const struct nft_ctx *ctx, struct nft_rule *rule)
3681 {
3682         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_RELEASE);
3683         nf_tables_rule_destroy(ctx, rule);
3684 }
3685
3686 int nft_chain_validate(const struct nft_ctx *ctx, const struct nft_chain *chain)
3687 {
3688         struct nft_expr *expr, *last;
3689         const struct nft_data *data;
3690         struct nft_rule *rule;
3691         int err;
3692
3693         if (ctx->level == NFT_JUMP_STACK_SIZE)
3694                 return -EMLINK;
3695
3696         list_for_each_entry(rule, &chain->rules, list) {
3697                 if (fatal_signal_pending(current))
3698                         return -EINTR;
3699
3700                 if (!nft_is_active_next(ctx->net, rule))
3701                         continue;
3702
3703                 nft_rule_for_each_expr(expr, last, rule) {
3704                         if (!expr->ops->validate)
3705                                 continue;
3706
3707                         err = expr->ops->validate(ctx, expr, &data);
3708                         if (err < 0)
3709                                 return err;
3710                 }
3711         }
3712
3713         return 0;
3714 }
3715 EXPORT_SYMBOL_GPL(nft_chain_validate);
3716
3717 static int nft_table_validate(struct net *net, const struct nft_table *table)
3718 {
3719         struct nft_chain *chain;
3720         struct nft_ctx ctx = {
3721                 .net    = net,
3722                 .family = table->family,
3723         };
3724         int err;
3725
3726         list_for_each_entry(chain, &table->chains, list) {
3727                 if (!nft_is_base_chain(chain))
3728                         continue;
3729
3730                 ctx.chain = chain;
3731                 err = nft_chain_validate(&ctx, chain);
3732                 if (err < 0)
3733                         return err;
3734
3735                 cond_resched();
3736         }
3737
3738         return 0;
3739 }
3740
3741 int nft_setelem_validate(const struct nft_ctx *ctx, struct nft_set *set,
3742                          const struct nft_set_iter *iter,
3743                          struct nft_set_elem *elem)
3744 {
3745         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
3746         struct nft_ctx *pctx = (struct nft_ctx *)ctx;
3747         const struct nft_data *data;
3748         int err;
3749
3750         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
3751             *nft_set_ext_flags(ext) & NFT_SET_ELEM_INTERVAL_END)
3752                 return 0;
3753
3754         data = nft_set_ext_data(ext);
3755         switch (data->verdict.code) {
3756         case NFT_JUMP:
3757         case NFT_GOTO:
3758                 pctx->level++;
3759                 err = nft_chain_validate(ctx, data->verdict.chain);
3760                 if (err < 0)
3761                         return err;
3762                 pctx->level--;
3763                 break;
3764         default:
3765                 break;
3766         }
3767
3768         return 0;
3769 }
3770
3771 int nft_set_catchall_validate(const struct nft_ctx *ctx, struct nft_set *set)
3772 {
3773         u8 genmask = nft_genmask_next(ctx->net);
3774         struct nft_set_elem_catchall *catchall;
3775         struct nft_set_elem elem;
3776         struct nft_set_ext *ext;
3777         int ret = 0;
3778
3779         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
3780                 ext = nft_set_elem_ext(set, catchall->elem);
3781                 if (!nft_set_elem_active(ext, genmask))
3782                         continue;
3783
3784                 elem.priv = catchall->elem;
3785                 ret = nft_setelem_validate(ctx, set, NULL, &elem);
3786                 if (ret < 0)
3787                         return ret;
3788         }
3789
3790         return ret;
3791 }
3792
3793 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
3794                                              const struct nft_chain *chain,
3795                                              const struct nlattr *nla);
3796
3797 #define NFT_RULE_MAXEXPRS       128
3798
3799 static int nf_tables_newrule(struct sk_buff *skb, const struct nfnl_info *info,
3800                              const struct nlattr * const nla[])
3801 {
3802         struct nftables_pernet *nft_net = nft_pernet(info->net);
3803         struct netlink_ext_ack *extack = info->extack;
3804         unsigned int size, i, n, ulen = 0, usize = 0;
3805         u8 genmask = nft_genmask_next(info->net);
3806         struct nft_rule *rule, *old_rule = NULL;
3807         struct nft_expr_info *expr_info = NULL;
3808         u8 family = info->nfmsg->nfgen_family;
3809         struct nft_flow_rule *flow = NULL;
3810         struct net *net = info->net;
3811         struct nft_userdata *udata;
3812         struct nft_table *table;
3813         struct nft_chain *chain;
3814         struct nft_trans *trans;
3815         u64 handle, pos_handle;
3816         struct nft_expr *expr;
3817         struct nft_ctx ctx;
3818         struct nlattr *tmp;
3819         int err, rem;
3820
3821         lockdep_assert_held(&nft_net->commit_mutex);
3822
3823         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
3824                                  NETLINK_CB(skb).portid);
3825         if (IS_ERR(table)) {
3826                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3827                 return PTR_ERR(table);
3828         }
3829
3830         if (nla[NFTA_RULE_CHAIN]) {
3831                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
3832                                          genmask);
3833                 if (IS_ERR(chain)) {
3834                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3835                         return PTR_ERR(chain);
3836                 }
3837
3838         } else if (nla[NFTA_RULE_CHAIN_ID]) {
3839                 chain = nft_chain_lookup_byid(net, table, nla[NFTA_RULE_CHAIN_ID],
3840                                               genmask);
3841                 if (IS_ERR(chain)) {
3842                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN_ID]);
3843                         return PTR_ERR(chain);
3844                 }
3845         } else {
3846                 return -EINVAL;
3847         }
3848
3849         if (nft_chain_is_bound(chain))
3850                 return -EOPNOTSUPP;
3851
3852         if (nla[NFTA_RULE_HANDLE]) {
3853                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_HANDLE]));
3854                 rule = __nft_rule_lookup(chain, handle);
3855                 if (IS_ERR(rule)) {
3856                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3857                         return PTR_ERR(rule);
3858                 }
3859
3860                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
3861                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3862                         return -EEXIST;
3863                 }
3864                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
3865                         old_rule = rule;
3866                 else
3867                         return -EOPNOTSUPP;
3868         } else {
3869                 if (!(info->nlh->nlmsg_flags & NLM_F_CREATE) ||
3870                     info->nlh->nlmsg_flags & NLM_F_REPLACE)
3871                         return -EINVAL;
3872                 handle = nf_tables_alloc_handle(table);
3873
3874                 if (nla[NFTA_RULE_POSITION]) {
3875                         pos_handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_POSITION]));
3876                         old_rule = __nft_rule_lookup(chain, pos_handle);
3877                         if (IS_ERR(old_rule)) {
3878                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION]);
3879                                 return PTR_ERR(old_rule);
3880                         }
3881                 } else if (nla[NFTA_RULE_POSITION_ID]) {
3882                         old_rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_POSITION_ID]);
3883                         if (IS_ERR(old_rule)) {
3884                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION_ID]);
3885                                 return PTR_ERR(old_rule);
3886                         }
3887                 }
3888         }
3889
3890         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
3891
3892         n = 0;
3893         size = 0;
3894         if (nla[NFTA_RULE_EXPRESSIONS]) {
3895                 expr_info = kvmalloc_array(NFT_RULE_MAXEXPRS,
3896                                            sizeof(struct nft_expr_info),
3897                                            GFP_KERNEL);
3898                 if (!expr_info)
3899                         return -ENOMEM;
3900
3901                 nla_for_each_nested(tmp, nla[NFTA_RULE_EXPRESSIONS], rem) {
3902                         err = -EINVAL;
3903                         if (nla_type(tmp) != NFTA_LIST_ELEM)
3904                                 goto err_release_expr;
3905                         if (n == NFT_RULE_MAXEXPRS)
3906                                 goto err_release_expr;
3907                         err = nf_tables_expr_parse(&ctx, tmp, &expr_info[n]);
3908                         if (err < 0) {
3909                                 NL_SET_BAD_ATTR(extack, tmp);
3910                                 goto err_release_expr;
3911                         }
3912                         size += expr_info[n].ops->size;
3913                         n++;
3914                 }
3915         }
3916         /* Check for overflow of dlen field */
3917         err = -EFBIG;
3918         if (size >= 1 << 12)
3919                 goto err_release_expr;
3920
3921         if (nla[NFTA_RULE_USERDATA]) {
3922                 ulen = nla_len(nla[NFTA_RULE_USERDATA]);
3923                 if (ulen > 0)
3924                         usize = sizeof(struct nft_userdata) + ulen;
3925         }
3926
3927         err = -ENOMEM;
3928         rule = kzalloc(sizeof(*rule) + size + usize, GFP_KERNEL_ACCOUNT);
3929         if (rule == NULL)
3930                 goto err_release_expr;
3931
3932         nft_activate_next(net, rule);
3933
3934         rule->handle = handle;
3935         rule->dlen   = size;
3936         rule->udata  = ulen ? 1 : 0;
3937
3938         if (ulen) {
3939                 udata = nft_userdata(rule);
3940                 udata->len = ulen - 1;
3941                 nla_memcpy(udata->data, nla[NFTA_RULE_USERDATA], ulen);
3942         }
3943
3944         expr = nft_expr_first(rule);
3945         for (i = 0; i < n; i++) {
3946                 err = nf_tables_newexpr(&ctx, &expr_info[i], expr);
3947                 if (err < 0) {
3948                         NL_SET_BAD_ATTR(extack, expr_info[i].attr);
3949                         goto err_release_rule;
3950                 }
3951
3952                 if (expr_info[i].ops->validate)
3953                         nft_validate_state_update(table, NFT_VALIDATE_NEED);
3954
3955                 expr_info[i].ops = NULL;
3956                 expr = nft_expr_next(expr);
3957         }
3958
3959         if (chain->flags & NFT_CHAIN_HW_OFFLOAD) {
3960                 flow = nft_flow_rule_create(net, rule);
3961                 if (IS_ERR(flow)) {
3962                         err = PTR_ERR(flow);
3963                         goto err_release_rule;
3964                 }
3965         }
3966
3967         if (!nft_use_inc(&chain->use)) {
3968                 err = -EMFILE;
3969                 goto err_release_rule;
3970         }
3971
3972         if (info->nlh->nlmsg_flags & NLM_F_REPLACE) {
3973                 err = nft_delrule(&ctx, old_rule);
3974                 if (err < 0)
3975                         goto err_destroy_flow_rule;
3976
3977                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
3978                 if (trans == NULL) {
3979                         err = -ENOMEM;
3980                         goto err_destroy_flow_rule;
3981                 }
3982                 list_add_tail_rcu(&rule->list, &old_rule->list);
3983         } else {
3984                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
3985                 if (!trans) {
3986                         err = -ENOMEM;
3987                         goto err_destroy_flow_rule;
3988                 }
3989
3990                 if (info->nlh->nlmsg_flags & NLM_F_APPEND) {
3991                         if (old_rule)
3992                                 list_add_rcu(&rule->list, &old_rule->list);
3993                         else
3994                                 list_add_tail_rcu(&rule->list, &chain->rules);
3995                  } else {
3996                         if (old_rule)
3997                                 list_add_tail_rcu(&rule->list, &old_rule->list);
3998                         else
3999                                 list_add_rcu(&rule->list, &chain->rules);
4000                 }
4001         }
4002         kvfree(expr_info);
4003
4004         if (flow)
4005                 nft_trans_flow_rule(trans) = flow;
4006
4007         if (table->validate_state == NFT_VALIDATE_DO)
4008                 return nft_table_validate(net, table);
4009
4010         return 0;
4011
4012 err_destroy_flow_rule:
4013         nft_use_dec_restore(&chain->use);
4014         if (flow)
4015                 nft_flow_rule_destroy(flow);
4016 err_release_rule:
4017         nft_rule_expr_deactivate(&ctx, rule, NFT_TRANS_PREPARE_ERROR);
4018         nf_tables_rule_destroy(&ctx, rule);
4019 err_release_expr:
4020         for (i = 0; i < n; i++) {
4021                 if (expr_info[i].ops) {
4022                         module_put(expr_info[i].ops->type->owner);
4023                         if (expr_info[i].ops->type->release_ops)
4024                                 expr_info[i].ops->type->release_ops(expr_info[i].ops);
4025                 }
4026         }
4027         kvfree(expr_info);
4028
4029         return err;
4030 }
4031
4032 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
4033                                              const struct nft_chain *chain,
4034                                              const struct nlattr *nla)
4035 {
4036         struct nftables_pernet *nft_net = nft_pernet(net);
4037         u32 id = ntohl(nla_get_be32(nla));
4038         struct nft_trans *trans;
4039
4040         list_for_each_entry(trans, &nft_net->commit_list, list) {
4041                 if (trans->msg_type == NFT_MSG_NEWRULE &&
4042                     trans->ctx.chain == chain &&
4043                     id == nft_trans_rule_id(trans))
4044                         return nft_trans_rule(trans);
4045         }
4046         return ERR_PTR(-ENOENT);
4047 }
4048
4049 static int nf_tables_delrule(struct sk_buff *skb, const struct nfnl_info *info,
4050                              const struct nlattr * const nla[])
4051 {
4052         struct netlink_ext_ack *extack = info->extack;
4053         u8 genmask = nft_genmask_next(info->net);
4054         u8 family = info->nfmsg->nfgen_family;
4055         struct nft_chain *chain = NULL;
4056         struct net *net = info->net;
4057         struct nft_table *table;
4058         struct nft_rule *rule;
4059         struct nft_ctx ctx;
4060         int err = 0;
4061
4062         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
4063                                  NETLINK_CB(skb).portid);
4064         if (IS_ERR(table)) {
4065                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
4066                 return PTR_ERR(table);
4067         }
4068
4069         if (nla[NFTA_RULE_CHAIN]) {
4070                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
4071                                          genmask);
4072                 if (IS_ERR(chain)) {
4073                         if (PTR_ERR(chain) == -ENOENT &&
4074                             NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYRULE)
4075                                 return 0;
4076
4077                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
4078                         return PTR_ERR(chain);
4079                 }
4080                 if (nft_chain_is_bound(chain))
4081                         return -EOPNOTSUPP;
4082         }
4083
4084         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
4085
4086         if (chain) {
4087                 if (nla[NFTA_RULE_HANDLE]) {
4088                         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
4089                         if (IS_ERR(rule)) {
4090                                 if (PTR_ERR(rule) == -ENOENT &&
4091                                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYRULE)
4092                                         return 0;
4093
4094                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
4095                                 return PTR_ERR(rule);
4096                         }
4097
4098                         err = nft_delrule(&ctx, rule);
4099                 } else if (nla[NFTA_RULE_ID]) {
4100                         rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_ID]);
4101                         if (IS_ERR(rule)) {
4102                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_ID]);
4103                                 return PTR_ERR(rule);
4104                         }
4105
4106                         err = nft_delrule(&ctx, rule);
4107                 } else {
4108                         err = nft_delrule_by_chain(&ctx);
4109                 }
4110         } else {
4111                 list_for_each_entry(chain, &table->chains, list) {
4112                         if (!nft_is_active_next(net, chain))
4113                                 continue;
4114                         if (nft_chain_is_bound(chain))
4115                                 continue;
4116
4117                         ctx.chain = chain;
4118                         err = nft_delrule_by_chain(&ctx);
4119                         if (err < 0)
4120                                 break;
4121                 }
4122         }
4123
4124         return err;
4125 }
4126
4127 /*
4128  * Sets
4129  */
4130 static const struct nft_set_type *nft_set_types[] = {
4131         &nft_set_hash_fast_type,
4132         &nft_set_hash_type,
4133         &nft_set_rhash_type,
4134         &nft_set_bitmap_type,
4135         &nft_set_rbtree_type,
4136 #if defined(CONFIG_X86_64) && !defined(CONFIG_UML)
4137         &nft_set_pipapo_avx2_type,
4138 #endif
4139         &nft_set_pipapo_type,
4140 };
4141
4142 #define NFT_SET_FEATURES        (NFT_SET_INTERVAL | NFT_SET_MAP | \
4143                                  NFT_SET_TIMEOUT | NFT_SET_OBJECT | \
4144                                  NFT_SET_EVAL)
4145
4146 static bool nft_set_ops_candidate(const struct nft_set_type *type, u32 flags)
4147 {
4148         return (flags & type->features) == (flags & NFT_SET_FEATURES);
4149 }
4150
4151 /*
4152  * Select a set implementation based on the data characteristics and the
4153  * given policy. The total memory use might not be known if no size is
4154  * given, in that case the amount of memory per element is used.
4155  */
4156 static const struct nft_set_ops *
4157 nft_select_set_ops(const struct nft_ctx *ctx,
4158                    const struct nlattr * const nla[],
4159                    const struct nft_set_desc *desc)
4160 {
4161         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
4162         const struct nft_set_ops *ops, *bops;
4163         struct nft_set_estimate est, best;
4164         const struct nft_set_type *type;
4165         u32 flags = 0;
4166         int i;
4167
4168         lockdep_assert_held(&nft_net->commit_mutex);
4169         lockdep_nfnl_nft_mutex_not_held();
4170
4171         if (nla[NFTA_SET_FLAGS] != NULL)
4172                 flags = ntohl(nla_get_be32(nla[NFTA_SET_FLAGS]));
4173
4174         bops        = NULL;
4175         best.size   = ~0;
4176         best.lookup = ~0;
4177         best.space  = ~0;
4178
4179         for (i = 0; i < ARRAY_SIZE(nft_set_types); i++) {
4180                 type = nft_set_types[i];
4181                 ops = &type->ops;
4182
4183                 if (!nft_set_ops_candidate(type, flags))
4184                         continue;
4185                 if (!ops->estimate(desc, flags, &est))
4186                         continue;
4187
4188                 switch (desc->policy) {
4189                 case NFT_SET_POL_PERFORMANCE:
4190                         if (est.lookup < best.lookup)
4191                                 break;
4192                         if (est.lookup == best.lookup &&
4193                             est.space < best.space)
4194                                 break;
4195                         continue;
4196                 case NFT_SET_POL_MEMORY:
4197                         if (!desc->size) {
4198                                 if (est.space < best.space)
4199                                         break;
4200                                 if (est.space == best.space &&
4201                                     est.lookup < best.lookup)
4202                                         break;
4203                         } else if (est.size < best.size || !bops) {
4204                                 break;
4205                         }
4206                         continue;
4207                 default:
4208                         break;
4209                 }
4210
4211                 bops = ops;
4212                 best = est;
4213         }
4214
4215         if (bops != NULL)
4216                 return bops;
4217
4218         return ERR_PTR(-EOPNOTSUPP);
4219 }
4220
4221 static const struct nla_policy nft_set_policy[NFTA_SET_MAX + 1] = {
4222         [NFTA_SET_TABLE]                = { .type = NLA_STRING,
4223                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
4224         [NFTA_SET_NAME]                 = { .type = NLA_STRING,
4225                                             .len = NFT_SET_MAXNAMELEN - 1 },
4226         [NFTA_SET_FLAGS]                = { .type = NLA_U32 },
4227         [NFTA_SET_KEY_TYPE]             = { .type = NLA_U32 },
4228         [NFTA_SET_KEY_LEN]              = { .type = NLA_U32 },
4229         [NFTA_SET_DATA_TYPE]            = { .type = NLA_U32 },
4230         [NFTA_SET_DATA_LEN]             = { .type = NLA_U32 },
4231         [NFTA_SET_POLICY]               = { .type = NLA_U32 },
4232         [NFTA_SET_DESC]                 = { .type = NLA_NESTED },
4233         [NFTA_SET_ID]                   = { .type = NLA_U32 },
4234         [NFTA_SET_TIMEOUT]              = { .type = NLA_U64 },
4235         [NFTA_SET_GC_INTERVAL]          = { .type = NLA_U32 },
4236         [NFTA_SET_USERDATA]             = { .type = NLA_BINARY,
4237                                             .len  = NFT_USERDATA_MAXLEN },
4238         [NFTA_SET_OBJ_TYPE]             = { .type = NLA_U32 },
4239         [NFTA_SET_HANDLE]               = { .type = NLA_U64 },
4240         [NFTA_SET_EXPR]                 = { .type = NLA_NESTED },
4241         [NFTA_SET_EXPRESSIONS]          = { .type = NLA_NESTED },
4242 };
4243
4244 static const struct nla_policy nft_set_desc_policy[NFTA_SET_DESC_MAX + 1] = {
4245         [NFTA_SET_DESC_SIZE]            = { .type = NLA_U32 },
4246         [NFTA_SET_DESC_CONCAT]          = { .type = NLA_NESTED },
4247 };
4248
4249 static struct nft_set *nft_set_lookup(const struct nft_table *table,
4250                                       const struct nlattr *nla, u8 genmask)
4251 {
4252         struct nft_set *set;
4253
4254         if (nla == NULL)
4255                 return ERR_PTR(-EINVAL);
4256
4257         list_for_each_entry_rcu(set, &table->sets, list) {
4258                 if (!nla_strcmp(nla, set->name) &&
4259                     nft_active_genmask(set, genmask))
4260                         return set;
4261         }
4262         return ERR_PTR(-ENOENT);
4263 }
4264
4265 static struct nft_set *nft_set_lookup_byhandle(const struct nft_table *table,
4266                                                const struct nlattr *nla,
4267                                                u8 genmask)
4268 {
4269         struct nft_set *set;
4270
4271         list_for_each_entry(set, &table->sets, list) {
4272                 if (be64_to_cpu(nla_get_be64(nla)) == set->handle &&
4273                     nft_active_genmask(set, genmask))
4274                         return set;
4275         }
4276         return ERR_PTR(-ENOENT);
4277 }
4278
4279 static struct nft_set *nft_set_lookup_byid(const struct net *net,
4280                                            const struct nft_table *table,
4281                                            const struct nlattr *nla, u8 genmask)
4282 {
4283         struct nftables_pernet *nft_net = nft_pernet(net);
4284         u32 id = ntohl(nla_get_be32(nla));
4285         struct nft_trans *trans;
4286
4287         list_for_each_entry(trans, &nft_net->commit_list, list) {
4288                 if (trans->msg_type == NFT_MSG_NEWSET) {
4289                         struct nft_set *set = nft_trans_set(trans);
4290
4291                         if (id == nft_trans_set_id(trans) &&
4292                             set->table == table &&
4293                             nft_active_genmask(set, genmask))
4294                                 return set;
4295                 }
4296         }
4297         return ERR_PTR(-ENOENT);
4298 }
4299
4300 struct nft_set *nft_set_lookup_global(const struct net *net,
4301                                       const struct nft_table *table,
4302                                       const struct nlattr *nla_set_name,
4303                                       const struct nlattr *nla_set_id,
4304                                       u8 genmask)
4305 {
4306         struct nft_set *set;
4307
4308         set = nft_set_lookup(table, nla_set_name, genmask);
4309         if (IS_ERR(set)) {
4310                 if (!nla_set_id)
4311                         return set;
4312
4313                 set = nft_set_lookup_byid(net, table, nla_set_id, genmask);
4314         }
4315         return set;
4316 }
4317 EXPORT_SYMBOL_GPL(nft_set_lookup_global);
4318
4319 static int nf_tables_set_alloc_name(struct nft_ctx *ctx, struct nft_set *set,
4320                                     const char *name)
4321 {
4322         const struct nft_set *i;
4323         const char *p;
4324         unsigned long *inuse;
4325         unsigned int n = 0, min = 0;
4326
4327         p = strchr(name, '%');
4328         if (p != NULL) {
4329                 if (p[1] != 'd' || strchr(p + 2, '%'))
4330                         return -EINVAL;
4331
4332                 inuse = (unsigned long *)get_zeroed_page(GFP_KERNEL);
4333                 if (inuse == NULL)
4334                         return -ENOMEM;
4335 cont:
4336                 list_for_each_entry(i, &ctx->table->sets, list) {
4337                         int tmp;
4338
4339                         if (!nft_is_active_next(ctx->net, i))
4340                                 continue;
4341                         if (!sscanf(i->name, name, &tmp))
4342                                 continue;
4343                         if (tmp < min || tmp >= min + BITS_PER_BYTE * PAGE_SIZE)
4344                                 continue;
4345
4346                         set_bit(tmp - min, inuse);
4347                 }
4348
4349                 n = find_first_zero_bit(inuse, BITS_PER_BYTE * PAGE_SIZE);
4350                 if (n >= BITS_PER_BYTE * PAGE_SIZE) {
4351                         min += BITS_PER_BYTE * PAGE_SIZE;
4352                         memset(inuse, 0, PAGE_SIZE);
4353                         goto cont;
4354                 }
4355                 free_page((unsigned long)inuse);
4356         }
4357
4358         set->name = kasprintf(GFP_KERNEL_ACCOUNT, name, min + n);
4359         if (!set->name)
4360                 return -ENOMEM;
4361
4362         list_for_each_entry(i, &ctx->table->sets, list) {
4363                 if (!nft_is_active_next(ctx->net, i))
4364                         continue;
4365                 if (!strcmp(set->name, i->name)) {
4366                         kfree(set->name);
4367                         set->name = NULL;
4368                         return -ENFILE;
4369                 }
4370         }
4371         return 0;
4372 }
4373
4374 int nf_msecs_to_jiffies64(const struct nlattr *nla, u64 *result)
4375 {
4376         u64 ms = be64_to_cpu(nla_get_be64(nla));
4377         u64 max = (u64)(~((u64)0));
4378
4379         max = div_u64(max, NSEC_PER_MSEC);
4380         if (ms >= max)
4381                 return -ERANGE;
4382
4383         ms *= NSEC_PER_MSEC;
4384         *result = nsecs_to_jiffies64(ms);
4385         return 0;
4386 }
4387
4388 __be64 nf_jiffies64_to_msecs(u64 input)
4389 {
4390         return cpu_to_be64(jiffies64_to_msecs(input));
4391 }
4392
4393 static int nf_tables_fill_set_concat(struct sk_buff *skb,
4394                                      const struct nft_set *set)
4395 {
4396         struct nlattr *concat, *field;
4397         int i;
4398
4399         concat = nla_nest_start_noflag(skb, NFTA_SET_DESC_CONCAT);
4400         if (!concat)
4401                 return -ENOMEM;
4402
4403         for (i = 0; i < set->field_count; i++) {
4404                 field = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
4405                 if (!field)
4406                         return -ENOMEM;
4407
4408                 if (nla_put_be32(skb, NFTA_SET_FIELD_LEN,
4409                                  htonl(set->field_len[i])))
4410                         return -ENOMEM;
4411
4412                 nla_nest_end(skb, field);
4413         }
4414
4415         nla_nest_end(skb, concat);
4416
4417         return 0;
4418 }
4419
4420 static int nf_tables_fill_set(struct sk_buff *skb, const struct nft_ctx *ctx,
4421                               const struct nft_set *set, u16 event, u16 flags)
4422 {
4423         u64 timeout = READ_ONCE(set->timeout);
4424         u32 gc_int = READ_ONCE(set->gc_int);
4425         u32 portid = ctx->portid;
4426         struct nlmsghdr *nlh;
4427         struct nlattr *nest;
4428         u32 seq = ctx->seq;
4429         int i;
4430
4431         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
4432         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
4433                            NFNETLINK_V0, nft_base_seq(ctx->net));
4434         if (!nlh)
4435                 goto nla_put_failure;
4436
4437         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
4438                 goto nla_put_failure;
4439         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
4440                 goto nla_put_failure;
4441         if (nla_put_be64(skb, NFTA_SET_HANDLE, cpu_to_be64(set->handle),
4442                          NFTA_SET_PAD))
4443                 goto nla_put_failure;
4444
4445         if (event == NFT_MSG_DELSET) {
4446                 nlmsg_end(skb, nlh);
4447                 return 0;
4448         }
4449
4450         if (set->flags != 0)
4451                 if (nla_put_be32(skb, NFTA_SET_FLAGS, htonl(set->flags)))
4452                         goto nla_put_failure;
4453
4454         if (nla_put_be32(skb, NFTA_SET_KEY_TYPE, htonl(set->ktype)))
4455                 goto nla_put_failure;
4456         if (nla_put_be32(skb, NFTA_SET_KEY_LEN, htonl(set->klen)))
4457                 goto nla_put_failure;
4458         if (set->flags & NFT_SET_MAP) {
4459                 if (nla_put_be32(skb, NFTA_SET_DATA_TYPE, htonl(set->dtype)))
4460                         goto nla_put_failure;
4461                 if (nla_put_be32(skb, NFTA_SET_DATA_LEN, htonl(set->dlen)))
4462                         goto nla_put_failure;
4463         }
4464         if (set->flags & NFT_SET_OBJECT &&
4465             nla_put_be32(skb, NFTA_SET_OBJ_TYPE, htonl(set->objtype)))
4466                 goto nla_put_failure;
4467
4468         if (timeout &&
4469             nla_put_be64(skb, NFTA_SET_TIMEOUT,
4470                          nf_jiffies64_to_msecs(timeout),
4471                          NFTA_SET_PAD))
4472                 goto nla_put_failure;
4473         if (gc_int &&
4474             nla_put_be32(skb, NFTA_SET_GC_INTERVAL, htonl(gc_int)))
4475                 goto nla_put_failure;
4476
4477         if (set->policy != NFT_SET_POL_PERFORMANCE) {
4478                 if (nla_put_be32(skb, NFTA_SET_POLICY, htonl(set->policy)))
4479                         goto nla_put_failure;
4480         }
4481
4482         if (set->udata &&
4483             nla_put(skb, NFTA_SET_USERDATA, set->udlen, set->udata))
4484                 goto nla_put_failure;
4485
4486         nest = nla_nest_start_noflag(skb, NFTA_SET_DESC);
4487         if (!nest)
4488                 goto nla_put_failure;
4489         if (set->size &&
4490             nla_put_be32(skb, NFTA_SET_DESC_SIZE, htonl(set->size)))
4491                 goto nla_put_failure;
4492
4493         if (set->field_count > 1 &&
4494             nf_tables_fill_set_concat(skb, set))
4495                 goto nla_put_failure;
4496
4497         nla_nest_end(skb, nest);
4498
4499         if (set->num_exprs == 1) {
4500                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPR);
4501                 if (nf_tables_fill_expr_info(skb, set->exprs[0], false) < 0)
4502                         goto nla_put_failure;
4503
4504                 nla_nest_end(skb, nest);
4505         } else if (set->num_exprs > 1) {
4506                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPRESSIONS);
4507                 if (nest == NULL)
4508                         goto nla_put_failure;
4509
4510                 for (i = 0; i < set->num_exprs; i++) {
4511                         if (nft_expr_dump(skb, NFTA_LIST_ELEM,
4512                                           set->exprs[i], false) < 0)
4513                                 goto nla_put_failure;
4514                 }
4515                 nla_nest_end(skb, nest);
4516         }
4517
4518         nlmsg_end(skb, nlh);
4519         return 0;
4520
4521 nla_put_failure:
4522         nlmsg_trim(skb, nlh);
4523         return -1;
4524 }
4525
4526 static void nf_tables_set_notify(const struct nft_ctx *ctx,
4527                                  const struct nft_set *set, int event,
4528                                  gfp_t gfp_flags)
4529 {
4530         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
4531         u32 portid = ctx->portid;
4532         struct sk_buff *skb;
4533         u16 flags = 0;
4534         int err;
4535
4536         if (!ctx->report &&
4537             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
4538                 return;
4539
4540         skb = nlmsg_new(NLMSG_GOODSIZE, gfp_flags);
4541         if (skb == NULL)
4542                 goto err;
4543
4544         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
4545                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
4546
4547         err = nf_tables_fill_set(skb, ctx, set, event, flags);
4548         if (err < 0) {
4549                 kfree_skb(skb);
4550                 goto err;
4551         }
4552
4553         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
4554         return;
4555 err:
4556         nfnetlink_set_err(ctx->net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
4557 }
4558
4559 static int nf_tables_dump_sets(struct sk_buff *skb, struct netlink_callback *cb)
4560 {
4561         const struct nft_set *set;
4562         unsigned int idx, s_idx = cb->args[0];
4563         struct nft_table *table, *cur_table = (struct nft_table *)cb->args[2];
4564         struct net *net = sock_net(skb->sk);
4565         struct nft_ctx *ctx = cb->data, ctx_set;
4566         struct nftables_pernet *nft_net;
4567
4568         if (cb->args[1])
4569                 return skb->len;
4570
4571         rcu_read_lock();
4572         nft_net = nft_pernet(net);
4573         cb->seq = READ_ONCE(nft_net->base_seq);
4574
4575         list_for_each_entry_rcu(table, &nft_net->tables, list) {
4576                 if (ctx->family != NFPROTO_UNSPEC &&
4577                     ctx->family != table->family)
4578                         continue;
4579
4580                 if (ctx->table && ctx->table != table)
4581                         continue;
4582
4583                 if (cur_table) {
4584                         if (cur_table != table)
4585                                 continue;
4586
4587                         cur_table = NULL;
4588                 }
4589                 idx = 0;
4590                 list_for_each_entry_rcu(set, &table->sets, list) {
4591                         if (idx < s_idx)
4592                                 goto cont;
4593                         if (!nft_is_active(net, set))
4594                                 goto cont;
4595
4596                         ctx_set = *ctx;
4597                         ctx_set.table = table;
4598                         ctx_set.family = table->family;
4599
4600                         if (nf_tables_fill_set(skb, &ctx_set, set,
4601                                                NFT_MSG_NEWSET,
4602                                                NLM_F_MULTI) < 0) {
4603                                 cb->args[0] = idx;
4604                                 cb->args[2] = (unsigned long) table;
4605                                 goto done;
4606                         }
4607                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
4608 cont:
4609                         idx++;
4610                 }
4611                 if (s_idx)
4612                         s_idx = 0;
4613         }
4614         cb->args[1] = 1;
4615 done:
4616         rcu_read_unlock();
4617         return skb->len;
4618 }
4619
4620 static int nf_tables_dump_sets_start(struct netlink_callback *cb)
4621 {
4622         struct nft_ctx *ctx_dump = NULL;
4623
4624         ctx_dump = kmemdup(cb->data, sizeof(*ctx_dump), GFP_ATOMIC);
4625         if (ctx_dump == NULL)
4626                 return -ENOMEM;
4627
4628         cb->data = ctx_dump;
4629         return 0;
4630 }
4631
4632 static int nf_tables_dump_sets_done(struct netlink_callback *cb)
4633 {
4634         kfree(cb->data);
4635         return 0;
4636 }
4637
4638 /* called with rcu_read_lock held */
4639 static int nf_tables_getset(struct sk_buff *skb, const struct nfnl_info *info,
4640                             const struct nlattr * const nla[])
4641 {
4642         struct netlink_ext_ack *extack = info->extack;
4643         u8 genmask = nft_genmask_cur(info->net);
4644         u8 family = info->nfmsg->nfgen_family;
4645         struct nft_table *table = NULL;
4646         struct net *net = info->net;
4647         const struct nft_set *set;
4648         struct sk_buff *skb2;
4649         struct nft_ctx ctx;
4650         int err;
4651
4652         if (nla[NFTA_SET_TABLE]) {
4653                 table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
4654                                          genmask, 0);
4655                 if (IS_ERR(table)) {
4656                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
4657                         return PTR_ERR(table);
4658                 }
4659         }
4660
4661         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
4662
4663         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
4664                 struct netlink_dump_control c = {
4665                         .start = nf_tables_dump_sets_start,
4666                         .dump = nf_tables_dump_sets,
4667                         .done = nf_tables_dump_sets_done,
4668                         .data = &ctx,
4669                         .module = THIS_MODULE,
4670                 };
4671
4672                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
4673         }
4674
4675         /* Only accept unspec with dump */
4676         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
4677                 return -EAFNOSUPPORT;
4678         if (!nla[NFTA_SET_TABLE])
4679                 return -EINVAL;
4680
4681         set = nft_set_lookup(table, nla[NFTA_SET_NAME], genmask);
4682         if (IS_ERR(set))
4683                 return PTR_ERR(set);
4684
4685         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
4686         if (skb2 == NULL)
4687                 return -ENOMEM;
4688
4689         err = nf_tables_fill_set(skb2, &ctx, set, NFT_MSG_NEWSET, 0);
4690         if (err < 0)
4691                 goto err_fill_set_info;
4692
4693         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
4694
4695 err_fill_set_info:
4696         kfree_skb(skb2);
4697         return err;
4698 }
4699
4700 static const struct nla_policy nft_concat_policy[NFTA_SET_FIELD_MAX + 1] = {
4701         [NFTA_SET_FIELD_LEN]    = { .type = NLA_U32 },
4702 };
4703
4704 static int nft_set_desc_concat_parse(const struct nlattr *attr,
4705                                      struct nft_set_desc *desc)
4706 {
4707         struct nlattr *tb[NFTA_SET_FIELD_MAX + 1];
4708         u32 len;
4709         int err;
4710
4711         if (desc->field_count >= ARRAY_SIZE(desc->field_len))
4712                 return -E2BIG;
4713
4714         err = nla_parse_nested_deprecated(tb, NFTA_SET_FIELD_MAX, attr,
4715                                           nft_concat_policy, NULL);
4716         if (err < 0)
4717                 return err;
4718
4719         if (!tb[NFTA_SET_FIELD_LEN])
4720                 return -EINVAL;
4721
4722         len = ntohl(nla_get_be32(tb[NFTA_SET_FIELD_LEN]));
4723         if (!len || len > U8_MAX)
4724                 return -EINVAL;
4725
4726         desc->field_len[desc->field_count++] = len;
4727
4728         return 0;
4729 }
4730
4731 static int nft_set_desc_concat(struct nft_set_desc *desc,
4732                                const struct nlattr *nla)
4733 {
4734         struct nlattr *attr;
4735         u32 num_regs = 0;
4736         int rem, err, i;
4737
4738         nla_for_each_nested(attr, nla, rem) {
4739                 if (nla_type(attr) != NFTA_LIST_ELEM)
4740                         return -EINVAL;
4741
4742                 err = nft_set_desc_concat_parse(attr, desc);
4743                 if (err < 0)
4744                         return err;
4745         }
4746
4747         for (i = 0; i < desc->field_count; i++)
4748                 num_regs += DIV_ROUND_UP(desc->field_len[i], sizeof(u32));
4749
4750         if (num_regs > NFT_REG32_COUNT)
4751                 return -E2BIG;
4752
4753         return 0;
4754 }
4755
4756 static int nf_tables_set_desc_parse(struct nft_set_desc *desc,
4757                                     const struct nlattr *nla)
4758 {
4759         struct nlattr *da[NFTA_SET_DESC_MAX + 1];
4760         int err;
4761
4762         err = nla_parse_nested_deprecated(da, NFTA_SET_DESC_MAX, nla,
4763                                           nft_set_desc_policy, NULL);
4764         if (err < 0)
4765                 return err;
4766
4767         if (da[NFTA_SET_DESC_SIZE] != NULL)
4768                 desc->size = ntohl(nla_get_be32(da[NFTA_SET_DESC_SIZE]));
4769         if (da[NFTA_SET_DESC_CONCAT])
4770                 err = nft_set_desc_concat(desc, da[NFTA_SET_DESC_CONCAT]);
4771
4772         return err;
4773 }
4774
4775 static int nft_set_expr_alloc(struct nft_ctx *ctx, struct nft_set *set,
4776                               const struct nlattr * const *nla,
4777                               struct nft_expr **exprs, int *num_exprs,
4778                               u32 flags)
4779 {
4780         struct nft_expr *expr;
4781         int err, i;
4782
4783         if (nla[NFTA_SET_EXPR]) {
4784                 expr = nft_set_elem_expr_alloc(ctx, set, nla[NFTA_SET_EXPR]);
4785                 if (IS_ERR(expr)) {
4786                         err = PTR_ERR(expr);
4787                         goto err_set_expr_alloc;
4788                 }
4789                 exprs[0] = expr;
4790                 (*num_exprs)++;
4791         } else if (nla[NFTA_SET_EXPRESSIONS]) {
4792                 struct nlattr *tmp;
4793                 int left;
4794
4795                 if (!(flags & NFT_SET_EXPR)) {
4796                         err = -EINVAL;
4797                         goto err_set_expr_alloc;
4798                 }
4799                 i = 0;
4800                 nla_for_each_nested(tmp, nla[NFTA_SET_EXPRESSIONS], left) {
4801                         if (i == NFT_SET_EXPR_MAX) {
4802                                 err = -E2BIG;
4803                                 goto err_set_expr_alloc;
4804                         }
4805                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
4806                                 err = -EINVAL;
4807                                 goto err_set_expr_alloc;
4808                         }
4809                         expr = nft_set_elem_expr_alloc(ctx, set, tmp);
4810                         if (IS_ERR(expr)) {
4811                                 err = PTR_ERR(expr);
4812                                 goto err_set_expr_alloc;
4813                         }
4814                         exprs[i++] = expr;
4815                         (*num_exprs)++;
4816                 }
4817         }
4818
4819         return 0;
4820
4821 err_set_expr_alloc:
4822         for (i = 0; i < *num_exprs; i++)
4823                 nft_expr_destroy(ctx, exprs[i]);
4824
4825         return err;
4826 }
4827
4828 static bool nft_set_is_same(const struct nft_set *set,
4829                             const struct nft_set_desc *desc,
4830                             struct nft_expr *exprs[], u32 num_exprs, u32 flags)
4831 {
4832         int i;
4833
4834         if (set->ktype != desc->ktype ||
4835             set->dtype != desc->dtype ||
4836             set->flags != flags ||
4837             set->klen != desc->klen ||
4838             set->dlen != desc->dlen ||
4839             set->field_count != desc->field_count ||
4840             set->num_exprs != num_exprs)
4841                 return false;
4842
4843         for (i = 0; i < desc->field_count; i++) {
4844                 if (set->field_len[i] != desc->field_len[i])
4845                         return false;
4846         }
4847
4848         for (i = 0; i < num_exprs; i++) {
4849                 if (set->exprs[i]->ops != exprs[i]->ops)
4850                         return false;
4851         }
4852
4853         return true;
4854 }
4855
4856 static int nf_tables_newset(struct sk_buff *skb, const struct nfnl_info *info,
4857                             const struct nlattr * const nla[])
4858 {
4859         struct netlink_ext_ack *extack = info->extack;
4860         u8 genmask = nft_genmask_next(info->net);
4861         u8 family = info->nfmsg->nfgen_family;
4862         const struct nft_set_ops *ops;
4863         struct net *net = info->net;
4864         struct nft_set_desc desc;
4865         struct nft_table *table;
4866         unsigned char *udata;
4867         struct nft_set *set;
4868         struct nft_ctx ctx;
4869         size_t alloc_size;
4870         int num_exprs = 0;
4871         char *name;
4872         int err, i;
4873         u16 udlen;
4874         u32 flags;
4875         u64 size;
4876
4877         if (nla[NFTA_SET_TABLE] == NULL ||
4878             nla[NFTA_SET_NAME] == NULL ||
4879             nla[NFTA_SET_KEY_LEN] == NULL ||
4880             nla[NFTA_SET_ID] == NULL)
4881                 return -EINVAL;
4882
4883         memset(&desc, 0, sizeof(desc));
4884
4885         desc.ktype = NFT_DATA_VALUE;
4886         if (nla[NFTA_SET_KEY_TYPE] != NULL) {
4887                 desc.ktype = ntohl(nla_get_be32(nla[NFTA_SET_KEY_TYPE]));
4888                 if ((desc.ktype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK)
4889                         return -EINVAL;
4890         }
4891
4892         desc.klen = ntohl(nla_get_be32(nla[NFTA_SET_KEY_LEN]));
4893         if (desc.klen == 0 || desc.klen > NFT_DATA_VALUE_MAXLEN)
4894                 return -EINVAL;
4895
4896         flags = 0;
4897         if (nla[NFTA_SET_FLAGS] != NULL) {
4898                 flags = ntohl(nla_get_be32(nla[NFTA_SET_FLAGS]));
4899                 if (flags & ~(NFT_SET_ANONYMOUS | NFT_SET_CONSTANT |
4900                               NFT_SET_INTERVAL | NFT_SET_TIMEOUT |
4901                               NFT_SET_MAP | NFT_SET_EVAL |
4902                               NFT_SET_OBJECT | NFT_SET_CONCAT | NFT_SET_EXPR))
4903                         return -EOPNOTSUPP;
4904                 /* Only one of these operations is supported */
4905                 if ((flags & (NFT_SET_MAP | NFT_SET_OBJECT)) ==
4906                              (NFT_SET_MAP | NFT_SET_OBJECT))
4907                         return -EOPNOTSUPP;
4908                 if ((flags & (NFT_SET_EVAL | NFT_SET_OBJECT)) ==
4909                              (NFT_SET_EVAL | NFT_SET_OBJECT))
4910                         return -EOPNOTSUPP;
4911         }
4912
4913         desc.dtype = 0;
4914         if (nla[NFTA_SET_DATA_TYPE] != NULL) {
4915                 if (!(flags & NFT_SET_MAP))
4916                         return -EINVAL;
4917
4918                 desc.dtype = ntohl(nla_get_be32(nla[NFTA_SET_DATA_TYPE]));
4919                 if ((desc.dtype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK &&
4920                     desc.dtype != NFT_DATA_VERDICT)
4921                         return -EINVAL;
4922
4923                 if (desc.dtype != NFT_DATA_VERDICT) {
4924                         if (nla[NFTA_SET_DATA_LEN] == NULL)
4925                                 return -EINVAL;
4926                         desc.dlen = ntohl(nla_get_be32(nla[NFTA_SET_DATA_LEN]));
4927                         if (desc.dlen == 0 || desc.dlen > NFT_DATA_VALUE_MAXLEN)
4928                                 return -EINVAL;
4929                 } else
4930                         desc.dlen = sizeof(struct nft_verdict);
4931         } else if (flags & NFT_SET_MAP)
4932                 return -EINVAL;
4933
4934         if (nla[NFTA_SET_OBJ_TYPE] != NULL) {
4935                 if (!(flags & NFT_SET_OBJECT))
4936                         return -EINVAL;
4937
4938                 desc.objtype = ntohl(nla_get_be32(nla[NFTA_SET_OBJ_TYPE]));
4939                 if (desc.objtype == NFT_OBJECT_UNSPEC ||
4940                     desc.objtype > NFT_OBJECT_MAX)
4941                         return -EOPNOTSUPP;
4942         } else if (flags & NFT_SET_OBJECT)
4943                 return -EINVAL;
4944         else
4945                 desc.objtype = NFT_OBJECT_UNSPEC;
4946
4947         desc.timeout = 0;
4948         if (nla[NFTA_SET_TIMEOUT] != NULL) {
4949                 if (!(flags & NFT_SET_TIMEOUT))
4950                         return -EINVAL;
4951
4952                 if (flags & NFT_SET_ANONYMOUS)
4953                         return -EOPNOTSUPP;
4954
4955                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_TIMEOUT], &desc.timeout);
4956                 if (err)
4957                         return err;
4958         }
4959         desc.gc_int = 0;
4960         if (nla[NFTA_SET_GC_INTERVAL] != NULL) {
4961                 if (!(flags & NFT_SET_TIMEOUT))
4962                         return -EINVAL;
4963
4964                 if (flags & NFT_SET_ANONYMOUS)
4965                         return -EOPNOTSUPP;
4966
4967                 desc.gc_int = ntohl(nla_get_be32(nla[NFTA_SET_GC_INTERVAL]));
4968         }
4969
4970         desc.policy = NFT_SET_POL_PERFORMANCE;
4971         if (nla[NFTA_SET_POLICY] != NULL)
4972                 desc.policy = ntohl(nla_get_be32(nla[NFTA_SET_POLICY]));
4973
4974         if (nla[NFTA_SET_DESC] != NULL) {
4975                 err = nf_tables_set_desc_parse(&desc, nla[NFTA_SET_DESC]);
4976                 if (err < 0)
4977                         return err;
4978
4979                 if (desc.field_count > 1 && !(flags & NFT_SET_CONCAT))
4980                         return -EINVAL;
4981         } else if (flags & NFT_SET_CONCAT) {
4982                 return -EINVAL;
4983         }
4984
4985         if (nla[NFTA_SET_EXPR] || nla[NFTA_SET_EXPRESSIONS])
4986                 desc.expr = true;
4987
4988         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family, genmask,
4989                                  NETLINK_CB(skb).portid);
4990         if (IS_ERR(table)) {
4991                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
4992                 return PTR_ERR(table);
4993         }
4994
4995         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
4996
4997         set = nft_set_lookup(table, nla[NFTA_SET_NAME], genmask);
4998         if (IS_ERR(set)) {
4999                 if (PTR_ERR(set) != -ENOENT) {
5000                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
5001                         return PTR_ERR(set);
5002                 }
5003         } else {
5004                 struct nft_expr *exprs[NFT_SET_EXPR_MAX] = {};
5005
5006                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
5007                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
5008                         return -EEXIST;
5009                 }
5010                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
5011                         return -EOPNOTSUPP;
5012
5013                 if (nft_set_is_anonymous(set))
5014                         return -EOPNOTSUPP;
5015
5016                 err = nft_set_expr_alloc(&ctx, set, nla, exprs, &num_exprs, flags);
5017                 if (err < 0)
5018                         return err;
5019
5020                 err = 0;
5021                 if (!nft_set_is_same(set, &desc, exprs, num_exprs, flags)) {
5022                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
5023                         err = -EEXIST;
5024                 }
5025
5026                 for (i = 0; i < num_exprs; i++)
5027                         nft_expr_destroy(&ctx, exprs[i]);
5028
5029                 if (err < 0)
5030                         return err;
5031
5032                 return __nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set, &desc);
5033         }
5034
5035         if (!(info->nlh->nlmsg_flags & NLM_F_CREATE))
5036                 return -ENOENT;
5037
5038         ops = nft_select_set_ops(&ctx, nla, &desc);
5039         if (IS_ERR(ops))
5040                 return PTR_ERR(ops);
5041
5042         udlen = 0;
5043         if (nla[NFTA_SET_USERDATA])
5044                 udlen = nla_len(nla[NFTA_SET_USERDATA]);
5045
5046         size = 0;
5047         if (ops->privsize != NULL)
5048                 size = ops->privsize(nla, &desc);
5049         alloc_size = sizeof(*set) + size + udlen;
5050         if (alloc_size < size || alloc_size > INT_MAX)
5051                 return -ENOMEM;
5052
5053         if (!nft_use_inc(&table->use))
5054                 return -EMFILE;
5055
5056         set = kvzalloc(alloc_size, GFP_KERNEL_ACCOUNT);
5057         if (!set) {
5058                 err = -ENOMEM;
5059                 goto err_alloc;
5060         }
5061
5062         name = nla_strdup(nla[NFTA_SET_NAME], GFP_KERNEL_ACCOUNT);
5063         if (!name) {
5064                 err = -ENOMEM;
5065                 goto err_set_name;
5066         }
5067
5068         err = nf_tables_set_alloc_name(&ctx, set, name);
5069         kfree(name);
5070         if (err < 0)
5071                 goto err_set_name;
5072
5073         udata = NULL;
5074         if (udlen) {
5075                 udata = set->data + size;
5076                 nla_memcpy(udata, nla[NFTA_SET_USERDATA], udlen);
5077         }
5078
5079         INIT_LIST_HEAD(&set->bindings);
5080         INIT_LIST_HEAD(&set->catchall_list);
5081         refcount_set(&set->refs, 1);
5082         set->table = table;
5083         write_pnet(&set->net, net);
5084         set->ops = ops;
5085         set->ktype = desc.ktype;
5086         set->klen = desc.klen;
5087         set->dtype = desc.dtype;
5088         set->objtype = desc.objtype;
5089         set->dlen = desc.dlen;
5090         set->flags = flags;
5091         set->size = desc.size;
5092         set->policy = desc.policy;
5093         set->udlen = udlen;
5094         set->udata = udata;
5095         set->timeout = desc.timeout;
5096         set->gc_int = desc.gc_int;
5097
5098         set->field_count = desc.field_count;
5099         for (i = 0; i < desc.field_count; i++)
5100                 set->field_len[i] = desc.field_len[i];
5101
5102         err = ops->init(set, &desc, nla);
5103         if (err < 0)
5104                 goto err_set_init;
5105
5106         err = nft_set_expr_alloc(&ctx, set, nla, set->exprs, &num_exprs, flags);
5107         if (err < 0)
5108                 goto err_set_destroy;
5109
5110         set->num_exprs = num_exprs;
5111         set->handle = nf_tables_alloc_handle(table);
5112         INIT_LIST_HEAD(&set->pending_update);
5113
5114         err = nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set);
5115         if (err < 0)
5116                 goto err_set_expr_alloc;
5117
5118         list_add_tail_rcu(&set->list, &table->sets);
5119
5120         return 0;
5121
5122 err_set_expr_alloc:
5123         for (i = 0; i < set->num_exprs; i++)
5124                 nft_expr_destroy(&ctx, set->exprs[i]);
5125 err_set_destroy:
5126         ops->destroy(&ctx, set);
5127 err_set_init:
5128         kfree(set->name);
5129 err_set_name:
5130         kvfree(set);
5131 err_alloc:
5132         nft_use_dec_restore(&table->use);
5133
5134         return err;
5135 }
5136
5137 static void nft_set_catchall_destroy(const struct nft_ctx *ctx,
5138                                      struct nft_set *set)
5139 {
5140         struct nft_set_elem_catchall *next, *catchall;
5141
5142         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
5143                 list_del_rcu(&catchall->list);
5144                 nf_tables_set_elem_destroy(ctx, set, catchall->elem);
5145                 kfree_rcu(catchall, rcu);
5146         }
5147 }
5148
5149 static void nft_set_put(struct nft_set *set)
5150 {
5151         if (refcount_dec_and_test(&set->refs)) {
5152                 kfree(set->name);
5153                 kvfree(set);
5154         }
5155 }
5156
5157 static void nft_set_destroy(const struct nft_ctx *ctx, struct nft_set *set)
5158 {
5159         int i;
5160
5161         if (WARN_ON(set->use > 0))
5162                 return;
5163
5164         for (i = 0; i < set->num_exprs; i++)
5165                 nft_expr_destroy(ctx, set->exprs[i]);
5166
5167         set->ops->destroy(ctx, set);
5168         nft_set_catchall_destroy(ctx, set);
5169         nft_set_put(set);
5170 }
5171
5172 static int nf_tables_delset(struct sk_buff *skb, const struct nfnl_info *info,
5173                             const struct nlattr * const nla[])
5174 {
5175         struct netlink_ext_ack *extack = info->extack;
5176         u8 genmask = nft_genmask_next(info->net);
5177         u8 family = info->nfmsg->nfgen_family;
5178         struct net *net = info->net;
5179         const struct nlattr *attr;
5180         struct nft_table *table;
5181         struct nft_set *set;
5182         struct nft_ctx ctx;
5183
5184         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
5185                 return -EAFNOSUPPORT;
5186
5187         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
5188                                  genmask, NETLINK_CB(skb).portid);
5189         if (IS_ERR(table)) {
5190                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
5191                 return PTR_ERR(table);
5192         }
5193
5194         if (nla[NFTA_SET_HANDLE]) {
5195                 attr = nla[NFTA_SET_HANDLE];
5196                 set = nft_set_lookup_byhandle(table, attr, genmask);
5197         } else {
5198                 attr = nla[NFTA_SET_NAME];
5199                 set = nft_set_lookup(table, attr, genmask);
5200         }
5201
5202         if (IS_ERR(set)) {
5203                 if (PTR_ERR(set) == -ENOENT &&
5204                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYSET)
5205                         return 0;
5206
5207                 NL_SET_BAD_ATTR(extack, attr);
5208                 return PTR_ERR(set);
5209         }
5210         if (set->use ||
5211             (info->nlh->nlmsg_flags & NLM_F_NONREC &&
5212              atomic_read(&set->nelems) > 0)) {
5213                 NL_SET_BAD_ATTR(extack, attr);
5214                 return -EBUSY;
5215         }
5216
5217         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
5218
5219         return nft_delset(&ctx, set);
5220 }
5221
5222 static int nft_validate_register_store(const struct nft_ctx *ctx,
5223                                        enum nft_registers reg,
5224                                        const struct nft_data *data,
5225                                        enum nft_data_types type,
5226                                        unsigned int len);
5227
5228 static int nft_setelem_data_validate(const struct nft_ctx *ctx,
5229                                      struct nft_set *set,
5230                                      struct nft_set_elem *elem)
5231 {
5232         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
5233         enum nft_registers dreg;
5234
5235         dreg = nft_type_to_reg(set->dtype);
5236         return nft_validate_register_store(ctx, dreg, nft_set_ext_data(ext),
5237                                            set->dtype == NFT_DATA_VERDICT ?
5238                                            NFT_DATA_VERDICT : NFT_DATA_VALUE,
5239                                            set->dlen);
5240 }
5241
5242 static int nf_tables_bind_check_setelem(const struct nft_ctx *ctx,
5243                                         struct nft_set *set,
5244                                         const struct nft_set_iter *iter,
5245                                         struct nft_set_elem *elem)
5246 {
5247         return nft_setelem_data_validate(ctx, set, elem);
5248 }
5249
5250 static int nft_set_catchall_bind_check(const struct nft_ctx *ctx,
5251                                        struct nft_set *set)
5252 {
5253         u8 genmask = nft_genmask_next(ctx->net);
5254         struct nft_set_elem_catchall *catchall;
5255         struct nft_set_elem elem;
5256         struct nft_set_ext *ext;
5257         int ret = 0;
5258
5259         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5260                 ext = nft_set_elem_ext(set, catchall->elem);
5261                 if (!nft_set_elem_active(ext, genmask))
5262                         continue;
5263
5264                 elem.priv = catchall->elem;
5265                 ret = nft_setelem_data_validate(ctx, set, &elem);
5266                 if (ret < 0)
5267                         break;
5268         }
5269
5270         return ret;
5271 }
5272
5273 int nf_tables_bind_set(const struct nft_ctx *ctx, struct nft_set *set,
5274                        struct nft_set_binding *binding)
5275 {
5276         struct nft_set_binding *i;
5277         struct nft_set_iter iter;
5278
5279         if (!list_empty(&set->bindings) && nft_set_is_anonymous(set))
5280                 return -EBUSY;
5281
5282         if (binding->flags & NFT_SET_MAP) {
5283                 /* If the set is already bound to the same chain all
5284                  * jumps are already validated for that chain.
5285                  */
5286                 list_for_each_entry(i, &set->bindings, list) {
5287                         if (i->flags & NFT_SET_MAP &&
5288                             i->chain == binding->chain)
5289                                 goto bind;
5290                 }
5291
5292                 iter.genmask    = nft_genmask_next(ctx->net);
5293                 iter.skip       = 0;
5294                 iter.count      = 0;
5295                 iter.err        = 0;
5296                 iter.fn         = nf_tables_bind_check_setelem;
5297
5298                 set->ops->walk(ctx, set, &iter);
5299                 if (!iter.err)
5300                         iter.err = nft_set_catchall_bind_check(ctx, set);
5301
5302                 if (iter.err < 0)
5303                         return iter.err;
5304         }
5305 bind:
5306         if (!nft_use_inc(&set->use))
5307                 return -EMFILE;
5308
5309         binding->chain = ctx->chain;
5310         list_add_tail_rcu(&binding->list, &set->bindings);
5311         nft_set_trans_bind(ctx, set);
5312
5313         return 0;
5314 }
5315 EXPORT_SYMBOL_GPL(nf_tables_bind_set);
5316
5317 static void nf_tables_unbind_set(const struct nft_ctx *ctx, struct nft_set *set,
5318                                  struct nft_set_binding *binding, bool event)
5319 {
5320         list_del_rcu(&binding->list);
5321
5322         if (list_empty(&set->bindings) && nft_set_is_anonymous(set)) {
5323                 list_del_rcu(&set->list);
5324                 if (event)
5325                         nf_tables_set_notify(ctx, set, NFT_MSG_DELSET,
5326                                              GFP_KERNEL);
5327         }
5328 }
5329
5330 static void nft_setelem_data_activate(const struct net *net,
5331                                       const struct nft_set *set,
5332                                       struct nft_set_elem *elem);
5333
5334 static int nft_mapelem_activate(const struct nft_ctx *ctx,
5335                                 struct nft_set *set,
5336                                 const struct nft_set_iter *iter,
5337                                 struct nft_set_elem *elem)
5338 {
5339         nft_setelem_data_activate(ctx->net, set, elem);
5340
5341         return 0;
5342 }
5343
5344 static void nft_map_catchall_activate(const struct nft_ctx *ctx,
5345                                       struct nft_set *set)
5346 {
5347         u8 genmask = nft_genmask_next(ctx->net);
5348         struct nft_set_elem_catchall *catchall;
5349         struct nft_set_elem elem;
5350         struct nft_set_ext *ext;
5351
5352         list_for_each_entry(catchall, &set->catchall_list, list) {
5353                 ext = nft_set_elem_ext(set, catchall->elem);
5354                 if (!nft_set_elem_active(ext, genmask))
5355                         continue;
5356
5357                 elem.priv = catchall->elem;
5358                 nft_setelem_data_activate(ctx->net, set, &elem);
5359                 break;
5360         }
5361 }
5362
5363 static void nft_map_activate(const struct nft_ctx *ctx, struct nft_set *set)
5364 {
5365         struct nft_set_iter iter = {
5366                 .genmask        = nft_genmask_next(ctx->net),
5367                 .fn             = nft_mapelem_activate,
5368         };
5369
5370         set->ops->walk(ctx, set, &iter);
5371         WARN_ON_ONCE(iter.err);
5372
5373         nft_map_catchall_activate(ctx, set);
5374 }
5375
5376 void nf_tables_activate_set(const struct nft_ctx *ctx, struct nft_set *set)
5377 {
5378         if (nft_set_is_anonymous(set)) {
5379                 if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
5380                         nft_map_activate(ctx, set);
5381
5382                 nft_clear(ctx->net, set);
5383         }
5384
5385         nft_use_inc_restore(&set->use);
5386 }
5387 EXPORT_SYMBOL_GPL(nf_tables_activate_set);
5388
5389 void nf_tables_deactivate_set(const struct nft_ctx *ctx, struct nft_set *set,
5390                               struct nft_set_binding *binding,
5391                               enum nft_trans_phase phase)
5392 {
5393         switch (phase) {
5394         case NFT_TRANS_PREPARE_ERROR:
5395                 nft_set_trans_unbind(ctx, set);
5396                 if (nft_set_is_anonymous(set))
5397                         nft_deactivate_next(ctx->net, set);
5398                 else
5399                         list_del_rcu(&binding->list);
5400
5401                 nft_use_dec(&set->use);
5402                 break;
5403         case NFT_TRANS_PREPARE:
5404                 if (nft_set_is_anonymous(set)) {
5405                         if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
5406                                 nft_map_deactivate(ctx, set);
5407
5408                         nft_deactivate_next(ctx->net, set);
5409                 }
5410                 nft_use_dec(&set->use);
5411                 return;
5412         case NFT_TRANS_ABORT:
5413         case NFT_TRANS_RELEASE:
5414                 if (nft_set_is_anonymous(set) &&
5415                     set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
5416                         nft_map_deactivate(ctx, set);
5417
5418                 nft_use_dec(&set->use);
5419                 fallthrough;
5420         default:
5421                 nf_tables_unbind_set(ctx, set, binding,
5422                                      phase == NFT_TRANS_COMMIT);
5423         }
5424 }
5425 EXPORT_SYMBOL_GPL(nf_tables_deactivate_set);
5426
5427 void nf_tables_destroy_set(const struct nft_ctx *ctx, struct nft_set *set)
5428 {
5429         if (list_empty(&set->bindings) && nft_set_is_anonymous(set))
5430                 nft_set_destroy(ctx, set);
5431 }
5432 EXPORT_SYMBOL_GPL(nf_tables_destroy_set);
5433
5434 const struct nft_set_ext_type nft_set_ext_types[] = {
5435         [NFT_SET_EXT_KEY]               = {
5436                 .align  = __alignof__(u32),
5437         },
5438         [NFT_SET_EXT_DATA]              = {
5439                 .align  = __alignof__(u32),
5440         },
5441         [NFT_SET_EXT_EXPRESSIONS]       = {
5442                 .align  = __alignof__(struct nft_set_elem_expr),
5443         },
5444         [NFT_SET_EXT_OBJREF]            = {
5445                 .len    = sizeof(struct nft_object *),
5446                 .align  = __alignof__(struct nft_object *),
5447         },
5448         [NFT_SET_EXT_FLAGS]             = {
5449                 .len    = sizeof(u8),
5450                 .align  = __alignof__(u8),
5451         },
5452         [NFT_SET_EXT_TIMEOUT]           = {
5453                 .len    = sizeof(u64),
5454                 .align  = __alignof__(u64),
5455         },
5456         [NFT_SET_EXT_EXPIRATION]        = {
5457                 .len    = sizeof(u64),
5458                 .align  = __alignof__(u64),
5459         },
5460         [NFT_SET_EXT_USERDATA]          = {
5461                 .len    = sizeof(struct nft_userdata),
5462                 .align  = __alignof__(struct nft_userdata),
5463         },
5464         [NFT_SET_EXT_KEY_END]           = {
5465                 .align  = __alignof__(u32),
5466         },
5467 };
5468
5469 /*
5470  * Set elements
5471  */
5472
5473 static const struct nla_policy nft_set_elem_policy[NFTA_SET_ELEM_MAX + 1] = {
5474         [NFTA_SET_ELEM_KEY]             = { .type = NLA_NESTED },
5475         [NFTA_SET_ELEM_DATA]            = { .type = NLA_NESTED },
5476         [NFTA_SET_ELEM_FLAGS]           = { .type = NLA_U32 },
5477         [NFTA_SET_ELEM_TIMEOUT]         = { .type = NLA_U64 },
5478         [NFTA_SET_ELEM_EXPIRATION]      = { .type = NLA_U64 },
5479         [NFTA_SET_ELEM_USERDATA]        = { .type = NLA_BINARY,
5480                                             .len = NFT_USERDATA_MAXLEN },
5481         [NFTA_SET_ELEM_EXPR]            = { .type = NLA_NESTED },
5482         [NFTA_SET_ELEM_OBJREF]          = { .type = NLA_STRING,
5483                                             .len = NFT_OBJ_MAXNAMELEN - 1 },
5484         [NFTA_SET_ELEM_KEY_END]         = { .type = NLA_NESTED },
5485         [NFTA_SET_ELEM_EXPRESSIONS]     = { .type = NLA_NESTED },
5486 };
5487
5488 static const struct nla_policy nft_set_elem_list_policy[NFTA_SET_ELEM_LIST_MAX + 1] = {
5489         [NFTA_SET_ELEM_LIST_TABLE]      = { .type = NLA_STRING,
5490                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
5491         [NFTA_SET_ELEM_LIST_SET]        = { .type = NLA_STRING,
5492                                             .len = NFT_SET_MAXNAMELEN - 1 },
5493         [NFTA_SET_ELEM_LIST_ELEMENTS]   = { .type = NLA_NESTED },
5494         [NFTA_SET_ELEM_LIST_SET_ID]     = { .type = NLA_U32 },
5495 };
5496
5497 static int nft_set_elem_expr_dump(struct sk_buff *skb,
5498                                   const struct nft_set *set,
5499                                   const struct nft_set_ext *ext,
5500                                   bool reset)
5501 {
5502         struct nft_set_elem_expr *elem_expr;
5503         u32 size, num_exprs = 0;
5504         struct nft_expr *expr;
5505         struct nlattr *nest;
5506
5507         elem_expr = nft_set_ext_expr(ext);
5508         nft_setelem_expr_foreach(expr, elem_expr, size)
5509                 num_exprs++;
5510
5511         if (num_exprs == 1) {
5512                 expr = nft_setelem_expr_at(elem_expr, 0);
5513                 if (nft_expr_dump(skb, NFTA_SET_ELEM_EXPR, expr, reset) < 0)
5514                         return -1;
5515
5516                 return 0;
5517         } else if (num_exprs > 1) {
5518                 nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_EXPRESSIONS);
5519                 if (nest == NULL)
5520                         goto nla_put_failure;
5521
5522                 nft_setelem_expr_foreach(expr, elem_expr, size) {
5523                         expr = nft_setelem_expr_at(elem_expr, size);
5524                         if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr, reset) < 0)
5525                                 goto nla_put_failure;
5526                 }
5527                 nla_nest_end(skb, nest);
5528         }
5529         return 0;
5530
5531 nla_put_failure:
5532         return -1;
5533 }
5534
5535 static int nf_tables_fill_setelem(struct sk_buff *skb,
5536                                   const struct nft_set *set,
5537                                   const struct nft_set_elem *elem,
5538                                   bool reset)
5539 {
5540         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
5541         unsigned char *b = skb_tail_pointer(skb);
5542         struct nlattr *nest;
5543         u64 timeout = 0;
5544
5545         nest = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
5546         if (nest == NULL)
5547                 goto nla_put_failure;
5548
5549         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
5550             nft_data_dump(skb, NFTA_SET_ELEM_KEY, nft_set_ext_key(ext),
5551                           NFT_DATA_VALUE, set->klen) < 0)
5552                 goto nla_put_failure;
5553
5554         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
5555             nft_data_dump(skb, NFTA_SET_ELEM_KEY_END, nft_set_ext_key_end(ext),
5556                           NFT_DATA_VALUE, set->klen) < 0)
5557                 goto nla_put_failure;
5558
5559         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
5560             nft_data_dump(skb, NFTA_SET_ELEM_DATA, nft_set_ext_data(ext),
5561                           set->dtype == NFT_DATA_VERDICT ? NFT_DATA_VERDICT : NFT_DATA_VALUE,
5562                           set->dlen) < 0)
5563                 goto nla_put_failure;
5564
5565         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS) &&
5566             nft_set_elem_expr_dump(skb, set, ext, reset))
5567                 goto nla_put_failure;
5568
5569         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
5570             nla_put_string(skb, NFTA_SET_ELEM_OBJREF,
5571                            (*nft_set_ext_obj(ext))->key.name) < 0)
5572                 goto nla_put_failure;
5573
5574         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
5575             nla_put_be32(skb, NFTA_SET_ELEM_FLAGS,
5576                          htonl(*nft_set_ext_flags(ext))))
5577                 goto nla_put_failure;
5578
5579         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT)) {
5580                 timeout = *nft_set_ext_timeout(ext);
5581                 if (nla_put_be64(skb, NFTA_SET_ELEM_TIMEOUT,
5582                                  nf_jiffies64_to_msecs(timeout),
5583                                  NFTA_SET_ELEM_PAD))
5584                         goto nla_put_failure;
5585         } else if (set->flags & NFT_SET_TIMEOUT) {
5586                 timeout = READ_ONCE(set->timeout);
5587         }
5588
5589         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
5590                 u64 expires, now = get_jiffies_64();
5591
5592                 expires = *nft_set_ext_expiration(ext);
5593                 if (time_before64(now, expires))
5594                         expires -= now;
5595                 else
5596                         expires = 0;
5597
5598                 if (nla_put_be64(skb, NFTA_SET_ELEM_EXPIRATION,
5599                                  nf_jiffies64_to_msecs(expires),
5600                                  NFTA_SET_ELEM_PAD))
5601                         goto nla_put_failure;
5602
5603                 if (reset)
5604                         *nft_set_ext_expiration(ext) = now + timeout;
5605         }
5606
5607         if (nft_set_ext_exists(ext, NFT_SET_EXT_USERDATA)) {
5608                 struct nft_userdata *udata;
5609
5610                 udata = nft_set_ext_userdata(ext);
5611                 if (nla_put(skb, NFTA_SET_ELEM_USERDATA,
5612                             udata->len + 1, udata->data))
5613                         goto nla_put_failure;
5614         }
5615
5616         nla_nest_end(skb, nest);
5617         return 0;
5618
5619 nla_put_failure:
5620         nlmsg_trim(skb, b);
5621         return -EMSGSIZE;
5622 }
5623
5624 struct nft_set_dump_args {
5625         const struct netlink_callback   *cb;
5626         struct nft_set_iter             iter;
5627         struct sk_buff                  *skb;
5628         bool                            reset;
5629 };
5630
5631 static int nf_tables_dump_setelem(const struct nft_ctx *ctx,
5632                                   struct nft_set *set,
5633                                   const struct nft_set_iter *iter,
5634                                   struct nft_set_elem *elem)
5635 {
5636         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
5637         struct nft_set_dump_args *args;
5638
5639         if (nft_set_elem_expired(ext))
5640                 return 0;
5641
5642         args = container_of(iter, struct nft_set_dump_args, iter);
5643         return nf_tables_fill_setelem(args->skb, set, elem, args->reset);
5644 }
5645
5646 static void audit_log_nft_set_reset(const struct nft_table *table,
5647                                     unsigned int base_seq,
5648                                     unsigned int nentries)
5649 {
5650         char *buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, base_seq);
5651
5652         audit_log_nfcfg(buf, table->family, nentries,
5653                         AUDIT_NFT_OP_SETELEM_RESET, GFP_ATOMIC);
5654         kfree(buf);
5655 }
5656
5657 struct nft_set_dump_ctx {
5658         const struct nft_set    *set;
5659         struct nft_ctx          ctx;
5660 };
5661
5662 static int nft_set_catchall_dump(struct net *net, struct sk_buff *skb,
5663                                  const struct nft_set *set, bool reset,
5664                                  unsigned int base_seq)
5665 {
5666         struct nft_set_elem_catchall *catchall;
5667         u8 genmask = nft_genmask_cur(net);
5668         struct nft_set_elem elem;
5669         struct nft_set_ext *ext;
5670         int ret = 0;
5671
5672         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5673                 ext = nft_set_elem_ext(set, catchall->elem);
5674                 if (!nft_set_elem_active(ext, genmask) ||
5675                     nft_set_elem_expired(ext))
5676                         continue;
5677
5678                 elem.priv = catchall->elem;
5679                 ret = nf_tables_fill_setelem(skb, set, &elem, reset);
5680                 if (reset && !ret)
5681                         audit_log_nft_set_reset(set->table, base_seq, 1);
5682                 break;
5683         }
5684
5685         return ret;
5686 }
5687
5688 static int nf_tables_dump_set(struct sk_buff *skb, struct netlink_callback *cb)
5689 {
5690         struct nft_set_dump_ctx *dump_ctx = cb->data;
5691         struct net *net = sock_net(skb->sk);
5692         struct nftables_pernet *nft_net;
5693         struct nft_table *table;
5694         struct nft_set *set;
5695         struct nft_set_dump_args args;
5696         bool set_found = false;
5697         struct nlmsghdr *nlh;
5698         struct nlattr *nest;
5699         bool reset = false;
5700         u32 portid, seq;
5701         int event;
5702
5703         rcu_read_lock();
5704         nft_net = nft_pernet(net);
5705         cb->seq = READ_ONCE(nft_net->base_seq);
5706
5707         list_for_each_entry_rcu(table, &nft_net->tables, list) {
5708                 if (dump_ctx->ctx.family != NFPROTO_UNSPEC &&
5709                     dump_ctx->ctx.family != table->family)
5710                         continue;
5711
5712                 if (table != dump_ctx->ctx.table)
5713                         continue;
5714
5715                 list_for_each_entry_rcu(set, &table->sets, list) {
5716                         if (set == dump_ctx->set) {
5717                                 set_found = true;
5718                                 break;
5719                         }
5720                 }
5721                 break;
5722         }
5723
5724         if (!set_found) {
5725                 rcu_read_unlock();
5726                 return -ENOENT;
5727         }
5728
5729         event  = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWSETELEM);
5730         portid = NETLINK_CB(cb->skb).portid;
5731         seq    = cb->nlh->nlmsg_seq;
5732
5733         nlh = nfnl_msg_put(skb, portid, seq, event, NLM_F_MULTI,
5734                            table->family, NFNETLINK_V0, nft_base_seq(net));
5735         if (!nlh)
5736                 goto nla_put_failure;
5737
5738         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_TABLE, table->name))
5739                 goto nla_put_failure;
5740         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_SET, set->name))
5741                 goto nla_put_failure;
5742
5743         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
5744         if (nest == NULL)
5745                 goto nla_put_failure;
5746
5747         if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETSETELEM_RESET)
5748                 reset = true;
5749
5750         args.cb                 = cb;
5751         args.skb                = skb;
5752         args.reset              = reset;
5753         args.iter.genmask       = nft_genmask_cur(net);
5754         args.iter.skip          = cb->args[0];
5755         args.iter.count         = 0;
5756         args.iter.err           = 0;
5757         args.iter.fn            = nf_tables_dump_setelem;
5758         set->ops->walk(&dump_ctx->ctx, set, &args.iter);
5759
5760         if (!args.iter.err && args.iter.count == cb->args[0])
5761                 args.iter.err = nft_set_catchall_dump(net, skb, set,
5762                                                       reset, cb->seq);
5763         rcu_read_unlock();
5764
5765         nla_nest_end(skb, nest);
5766         nlmsg_end(skb, nlh);
5767
5768         if (reset && args.iter.count > args.iter.skip)
5769                 audit_log_nft_set_reset(table, cb->seq,
5770                                         args.iter.count - args.iter.skip);
5771
5772         if (args.iter.err && args.iter.err != -EMSGSIZE)
5773                 return args.iter.err;
5774         if (args.iter.count == cb->args[0])
5775                 return 0;
5776
5777         cb->args[0] = args.iter.count;
5778         return skb->len;
5779
5780 nla_put_failure:
5781         rcu_read_unlock();
5782         return -ENOSPC;
5783 }
5784
5785 static int nf_tables_dump_set_start(struct netlink_callback *cb)
5786 {
5787         struct nft_set_dump_ctx *dump_ctx = cb->data;
5788
5789         cb->data = kmemdup(dump_ctx, sizeof(*dump_ctx), GFP_ATOMIC);
5790
5791         return cb->data ? 0 : -ENOMEM;
5792 }
5793
5794 static int nf_tables_dump_set_done(struct netlink_callback *cb)
5795 {
5796         kfree(cb->data);
5797         return 0;
5798 }
5799
5800 static int nf_tables_fill_setelem_info(struct sk_buff *skb,
5801                                        const struct nft_ctx *ctx, u32 seq,
5802                                        u32 portid, int event, u16 flags,
5803                                        const struct nft_set *set,
5804                                        const struct nft_set_elem *elem,
5805                                        bool reset)
5806 {
5807         struct nlmsghdr *nlh;
5808         struct nlattr *nest;
5809         int err;
5810
5811         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
5812         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
5813                            NFNETLINK_V0, nft_base_seq(ctx->net));
5814         if (!nlh)
5815                 goto nla_put_failure;
5816
5817         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
5818                 goto nla_put_failure;
5819         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
5820                 goto nla_put_failure;
5821
5822         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
5823         if (nest == NULL)
5824                 goto nla_put_failure;
5825
5826         err = nf_tables_fill_setelem(skb, set, elem, reset);
5827         if (err < 0)
5828                 goto nla_put_failure;
5829
5830         nla_nest_end(skb, nest);
5831
5832         nlmsg_end(skb, nlh);
5833         return 0;
5834
5835 nla_put_failure:
5836         nlmsg_trim(skb, nlh);
5837         return -1;
5838 }
5839
5840 static int nft_setelem_parse_flags(const struct nft_set *set,
5841                                    const struct nlattr *attr, u32 *flags)
5842 {
5843         if (attr == NULL)
5844                 return 0;
5845
5846         *flags = ntohl(nla_get_be32(attr));
5847         if (*flags & ~(NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
5848                 return -EOPNOTSUPP;
5849         if (!(set->flags & NFT_SET_INTERVAL) &&
5850             *flags & NFT_SET_ELEM_INTERVAL_END)
5851                 return -EINVAL;
5852         if ((*flags & (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL)) ==
5853             (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
5854                 return -EINVAL;
5855
5856         return 0;
5857 }
5858
5859 static int nft_setelem_parse_key(struct nft_ctx *ctx, struct nft_set *set,
5860                                  struct nft_data *key, struct nlattr *attr)
5861 {
5862         struct nft_data_desc desc = {
5863                 .type   = NFT_DATA_VALUE,
5864                 .size   = NFT_DATA_VALUE_MAXLEN,
5865                 .len    = set->klen,
5866         };
5867
5868         return nft_data_init(ctx, key, &desc, attr);
5869 }
5870
5871 static int nft_setelem_parse_data(struct nft_ctx *ctx, struct nft_set *set,
5872                                   struct nft_data_desc *desc,
5873                                   struct nft_data *data,
5874                                   struct nlattr *attr)
5875 {
5876         u32 dtype;
5877
5878         if (set->dtype == NFT_DATA_VERDICT)
5879                 dtype = NFT_DATA_VERDICT;
5880         else
5881                 dtype = NFT_DATA_VALUE;
5882
5883         desc->type = dtype;
5884         desc->size = NFT_DATA_VALUE_MAXLEN;
5885         desc->len = set->dlen;
5886         desc->flags = NFT_DATA_DESC_SETELEM;
5887
5888         return nft_data_init(ctx, data, desc, attr);
5889 }
5890
5891 static void *nft_setelem_catchall_get(const struct net *net,
5892                                       const struct nft_set *set)
5893 {
5894         struct nft_set_elem_catchall *catchall;
5895         u8 genmask = nft_genmask_cur(net);
5896         struct nft_set_ext *ext;
5897         void *priv = NULL;
5898
5899         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5900                 ext = nft_set_elem_ext(set, catchall->elem);
5901                 if (!nft_set_elem_active(ext, genmask) ||
5902                     nft_set_elem_expired(ext))
5903                         continue;
5904
5905                 priv = catchall->elem;
5906                 break;
5907         }
5908
5909         return priv;
5910 }
5911
5912 static int nft_setelem_get(struct nft_ctx *ctx, struct nft_set *set,
5913                            struct nft_set_elem *elem, u32 flags)
5914 {
5915         void *priv;
5916
5917         if (!(flags & NFT_SET_ELEM_CATCHALL)) {
5918                 priv = set->ops->get(ctx->net, set, elem, flags);
5919                 if (IS_ERR(priv))
5920                         return PTR_ERR(priv);
5921         } else {
5922                 priv = nft_setelem_catchall_get(ctx->net, set);
5923                 if (!priv)
5924                         return -ENOENT;
5925         }
5926         elem->priv = priv;
5927
5928         return 0;
5929 }
5930
5931 static int nft_get_set_elem(struct nft_ctx *ctx, struct nft_set *set,
5932                             const struct nlattr *attr, bool reset)
5933 {
5934         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
5935         struct nft_set_elem elem;
5936         struct sk_buff *skb;
5937         uint32_t flags = 0;
5938         int err;
5939
5940         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
5941                                           nft_set_elem_policy, NULL);
5942         if (err < 0)
5943                 return err;
5944
5945         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
5946         if (err < 0)
5947                 return err;
5948
5949         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
5950                 return -EINVAL;
5951
5952         if (nla[NFTA_SET_ELEM_KEY]) {
5953                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
5954                                             nla[NFTA_SET_ELEM_KEY]);
5955                 if (err < 0)
5956                         return err;
5957         }
5958
5959         if (nla[NFTA_SET_ELEM_KEY_END]) {
5960                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
5961                                             nla[NFTA_SET_ELEM_KEY_END]);
5962                 if (err < 0)
5963                         return err;
5964         }
5965
5966         err = nft_setelem_get(ctx, set, &elem, flags);
5967         if (err < 0)
5968                 return err;
5969
5970         err = -ENOMEM;
5971         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_ATOMIC);
5972         if (skb == NULL)
5973                 return err;
5974
5975         err = nf_tables_fill_setelem_info(skb, ctx, ctx->seq, ctx->portid,
5976                                           NFT_MSG_NEWSETELEM, 0, set, &elem,
5977                                           reset);
5978         if (err < 0)
5979                 goto err_fill_setelem;
5980
5981         return nfnetlink_unicast(skb, ctx->net, ctx->portid);
5982
5983 err_fill_setelem:
5984         kfree_skb(skb);
5985         return err;
5986 }
5987
5988 /* called with rcu_read_lock held */
5989 static int nf_tables_getsetelem(struct sk_buff *skb,
5990                                 const struct nfnl_info *info,
5991                                 const struct nlattr * const nla[])
5992 {
5993         struct netlink_ext_ack *extack = info->extack;
5994         u8 genmask = nft_genmask_cur(info->net);
5995         u8 family = info->nfmsg->nfgen_family;
5996         int rem, err = 0, nelems = 0;
5997         struct net *net = info->net;
5998         struct nft_table *table;
5999         struct nft_set *set;
6000         struct nlattr *attr;
6001         struct nft_ctx ctx;
6002         bool reset = false;
6003
6004         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
6005                                  genmask, 0);
6006         if (IS_ERR(table)) {
6007                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
6008                 return PTR_ERR(table);
6009         }
6010
6011         set = nft_set_lookup(table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
6012         if (IS_ERR(set))
6013                 return PTR_ERR(set);
6014
6015         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
6016
6017         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
6018                 struct netlink_dump_control c = {
6019                         .start = nf_tables_dump_set_start,
6020                         .dump = nf_tables_dump_set,
6021                         .done = nf_tables_dump_set_done,
6022                         .module = THIS_MODULE,
6023                 };
6024                 struct nft_set_dump_ctx dump_ctx = {
6025                         .set = set,
6026                         .ctx = ctx,
6027                 };
6028
6029                 c.data = &dump_ctx;
6030                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
6031         }
6032
6033         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
6034                 return -EINVAL;
6035
6036         if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETSETELEM_RESET)
6037                 reset = true;
6038
6039         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
6040                 err = nft_get_set_elem(&ctx, set, attr, reset);
6041                 if (err < 0) {
6042                         NL_SET_BAD_ATTR(extack, attr);
6043                         break;
6044                 }
6045                 nelems++;
6046         }
6047
6048         if (reset)
6049                 audit_log_nft_set_reset(table, nft_pernet(net)->base_seq,
6050                                         nelems);
6051
6052         return err;
6053 }
6054
6055 static void nf_tables_setelem_notify(const struct nft_ctx *ctx,
6056                                      const struct nft_set *set,
6057                                      const struct nft_set_elem *elem,
6058                                      int event)
6059 {
6060         struct nftables_pernet *nft_net;
6061         struct net *net = ctx->net;
6062         u32 portid = ctx->portid;
6063         struct sk_buff *skb;
6064         u16 flags = 0;
6065         int err;
6066
6067         if (!ctx->report && !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
6068                 return;
6069
6070         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
6071         if (skb == NULL)
6072                 goto err;
6073
6074         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
6075                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
6076
6077         err = nf_tables_fill_setelem_info(skb, ctx, 0, portid, event, flags,
6078                                           set, elem, false);
6079         if (err < 0) {
6080                 kfree_skb(skb);
6081                 goto err;
6082         }
6083
6084         nft_net = nft_pernet(net);
6085         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
6086         return;
6087 err:
6088         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
6089 }
6090
6091 static struct nft_trans *nft_trans_elem_alloc(struct nft_ctx *ctx,
6092                                               int msg_type,
6093                                               struct nft_set *set)
6094 {
6095         struct nft_trans *trans;
6096
6097         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_elem));
6098         if (trans == NULL)
6099                 return NULL;
6100
6101         nft_trans_elem_set(trans) = set;
6102         return trans;
6103 }
6104
6105 struct nft_expr *nft_set_elem_expr_alloc(const struct nft_ctx *ctx,
6106                                          const struct nft_set *set,
6107                                          const struct nlattr *attr)
6108 {
6109         struct nft_expr *expr;
6110         int err;
6111
6112         expr = nft_expr_init(ctx, attr);
6113         if (IS_ERR(expr))
6114                 return expr;
6115
6116         err = -EOPNOTSUPP;
6117         if (expr->ops->type->flags & NFT_EXPR_GC) {
6118                 if (set->flags & NFT_SET_TIMEOUT)
6119                         goto err_set_elem_expr;
6120                 if (!set->ops->gc_init)
6121                         goto err_set_elem_expr;
6122                 set->ops->gc_init(set);
6123         }
6124
6125         return expr;
6126
6127 err_set_elem_expr:
6128         nft_expr_destroy(ctx, expr);
6129         return ERR_PTR(err);
6130 }
6131
6132 static int nft_set_ext_check(const struct nft_set_ext_tmpl *tmpl, u8 id, u32 len)
6133 {
6134         len += nft_set_ext_types[id].len;
6135         if (len > tmpl->ext_len[id] ||
6136             len > U8_MAX)
6137                 return -1;
6138
6139         return 0;
6140 }
6141
6142 static int nft_set_ext_memcpy(const struct nft_set_ext_tmpl *tmpl, u8 id,
6143                               void *to, const void *from, u32 len)
6144 {
6145         if (nft_set_ext_check(tmpl, id, len) < 0)
6146                 return -1;
6147
6148         memcpy(to, from, len);
6149
6150         return 0;
6151 }
6152
6153 void *nft_set_elem_init(const struct nft_set *set,
6154                         const struct nft_set_ext_tmpl *tmpl,
6155                         const u32 *key, const u32 *key_end,
6156                         const u32 *data, u64 timeout, u64 expiration, gfp_t gfp)
6157 {
6158         struct nft_set_ext *ext;
6159         void *elem;
6160
6161         elem = kzalloc(set->ops->elemsize + tmpl->len, gfp);
6162         if (elem == NULL)
6163                 return ERR_PTR(-ENOMEM);
6164
6165         ext = nft_set_elem_ext(set, elem);
6166         nft_set_ext_init(ext, tmpl);
6167
6168         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
6169             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY,
6170                                nft_set_ext_key(ext), key, set->klen) < 0)
6171                 goto err_ext_check;
6172
6173         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
6174             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY_END,
6175                                nft_set_ext_key_end(ext), key_end, set->klen) < 0)
6176                 goto err_ext_check;
6177
6178         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
6179             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_DATA,
6180                                nft_set_ext_data(ext), data, set->dlen) < 0)
6181                 goto err_ext_check;
6182
6183         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
6184                 *nft_set_ext_expiration(ext) = get_jiffies_64() + expiration;
6185                 if (expiration == 0)
6186                         *nft_set_ext_expiration(ext) += timeout;
6187         }
6188         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT))
6189                 *nft_set_ext_timeout(ext) = timeout;
6190
6191         return elem;
6192
6193 err_ext_check:
6194         kfree(elem);
6195
6196         return ERR_PTR(-EINVAL);
6197 }
6198
6199 static void __nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
6200                                         struct nft_expr *expr)
6201 {
6202         if (expr->ops->destroy_clone) {
6203                 expr->ops->destroy_clone(ctx, expr);
6204                 module_put(expr->ops->type->owner);
6205         } else {
6206                 nf_tables_expr_destroy(ctx, expr);
6207         }
6208 }
6209
6210 static void nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
6211                                       struct nft_set_elem_expr *elem_expr)
6212 {
6213         struct nft_expr *expr;
6214         u32 size;
6215
6216         nft_setelem_expr_foreach(expr, elem_expr, size)
6217                 __nft_set_elem_expr_destroy(ctx, expr);
6218 }
6219
6220 /* Drop references and destroy. Called from gc, dynset and abort path. */
6221 void nft_set_elem_destroy(const struct nft_set *set, void *elem,
6222                           bool destroy_expr)
6223 {
6224         struct nft_set_ext *ext = nft_set_elem_ext(set, elem);
6225         struct nft_ctx ctx = {
6226                 .net    = read_pnet(&set->net),
6227                 .family = set->table->family,
6228         };
6229
6230         nft_data_release(nft_set_ext_key(ext), NFT_DATA_VALUE);
6231         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
6232                 nft_data_release(nft_set_ext_data(ext), set->dtype);
6233         if (destroy_expr && nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
6234                 nft_set_elem_expr_destroy(&ctx, nft_set_ext_expr(ext));
6235
6236         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
6237                 nft_use_dec(&(*nft_set_ext_obj(ext))->use);
6238         kfree(elem);
6239 }
6240 EXPORT_SYMBOL_GPL(nft_set_elem_destroy);
6241
6242 /* Destroy element. References have been already dropped in the preparation
6243  * path via nft_setelem_data_deactivate().
6244  */
6245 void nf_tables_set_elem_destroy(const struct nft_ctx *ctx,
6246                                 const struct nft_set *set, void *elem)
6247 {
6248         struct nft_set_ext *ext = nft_set_elem_ext(set, elem);
6249
6250         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
6251                 nft_set_elem_expr_destroy(ctx, nft_set_ext_expr(ext));
6252
6253         kfree(elem);
6254 }
6255
6256 int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set,
6257                             struct nft_expr *expr_array[])
6258 {
6259         struct nft_expr *expr;
6260         int err, i, k;
6261
6262         for (i = 0; i < set->num_exprs; i++) {
6263                 expr = kzalloc(set->exprs[i]->ops->size, GFP_KERNEL_ACCOUNT);
6264                 if (!expr)
6265                         goto err_expr;
6266
6267                 err = nft_expr_clone(expr, set->exprs[i]);
6268                 if (err < 0) {
6269                         kfree(expr);
6270                         goto err_expr;
6271                 }
6272                 expr_array[i] = expr;
6273         }
6274
6275         return 0;
6276
6277 err_expr:
6278         for (k = i - 1; k >= 0; k--)
6279                 nft_expr_destroy(ctx, expr_array[k]);
6280
6281         return -ENOMEM;
6282 }
6283
6284 static int nft_set_elem_expr_setup(struct nft_ctx *ctx,
6285                                    const struct nft_set_ext_tmpl *tmpl,
6286                                    const struct nft_set_ext *ext,
6287                                    struct nft_expr *expr_array[],
6288                                    u32 num_exprs)
6289 {
6290         struct nft_set_elem_expr *elem_expr = nft_set_ext_expr(ext);
6291         u32 len = sizeof(struct nft_set_elem_expr);
6292         struct nft_expr *expr;
6293         int i, err;
6294
6295         if (num_exprs == 0)
6296                 return 0;
6297
6298         for (i = 0; i < num_exprs; i++)
6299                 len += expr_array[i]->ops->size;
6300
6301         if (nft_set_ext_check(tmpl, NFT_SET_EXT_EXPRESSIONS, len) < 0)
6302                 return -EINVAL;
6303
6304         for (i = 0; i < num_exprs; i++) {
6305                 expr = nft_setelem_expr_at(elem_expr, elem_expr->size);
6306                 err = nft_expr_clone(expr, expr_array[i]);
6307                 if (err < 0)
6308                         goto err_elem_expr_setup;
6309
6310                 elem_expr->size += expr_array[i]->ops->size;
6311                 nft_expr_destroy(ctx, expr_array[i]);
6312                 expr_array[i] = NULL;
6313         }
6314
6315         return 0;
6316
6317 err_elem_expr_setup:
6318         for (; i < num_exprs; i++) {
6319                 nft_expr_destroy(ctx, expr_array[i]);
6320                 expr_array[i] = NULL;
6321         }
6322
6323         return -ENOMEM;
6324 }
6325
6326 struct nft_set_ext *nft_set_catchall_lookup(const struct net *net,
6327                                             const struct nft_set *set)
6328 {
6329         struct nft_set_elem_catchall *catchall;
6330         u8 genmask = nft_genmask_cur(net);
6331         struct nft_set_ext *ext;
6332
6333         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
6334                 ext = nft_set_elem_ext(set, catchall->elem);
6335                 if (nft_set_elem_active(ext, genmask) &&
6336                     !nft_set_elem_expired(ext) &&
6337                     !nft_set_elem_is_dead(ext))
6338                         return ext;
6339         }
6340
6341         return NULL;
6342 }
6343 EXPORT_SYMBOL_GPL(nft_set_catchall_lookup);
6344
6345 static int nft_setelem_catchall_insert(const struct net *net,
6346                                        struct nft_set *set,
6347                                        const struct nft_set_elem *elem,
6348                                        struct nft_set_ext **pext)
6349 {
6350         struct nft_set_elem_catchall *catchall;
6351         u8 genmask = nft_genmask_next(net);
6352         struct nft_set_ext *ext;
6353
6354         list_for_each_entry(catchall, &set->catchall_list, list) {
6355                 ext = nft_set_elem_ext(set, catchall->elem);
6356                 if (nft_set_elem_active(ext, genmask)) {
6357                         *pext = ext;
6358                         return -EEXIST;
6359                 }
6360         }
6361
6362         catchall = kmalloc(sizeof(*catchall), GFP_KERNEL);
6363         if (!catchall)
6364                 return -ENOMEM;
6365
6366         catchall->elem = elem->priv;
6367         list_add_tail_rcu(&catchall->list, &set->catchall_list);
6368
6369         return 0;
6370 }
6371
6372 static int nft_setelem_insert(const struct net *net,
6373                               struct nft_set *set,
6374                               const struct nft_set_elem *elem,
6375                               struct nft_set_ext **ext, unsigned int flags)
6376 {
6377         int ret;
6378
6379         if (flags & NFT_SET_ELEM_CATCHALL)
6380                 ret = nft_setelem_catchall_insert(net, set, elem, ext);
6381         else
6382                 ret = set->ops->insert(net, set, elem, ext);
6383
6384         return ret;
6385 }
6386
6387 static bool nft_setelem_is_catchall(const struct nft_set *set,
6388                                     const struct nft_set_elem *elem)
6389 {
6390         struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6391
6392         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
6393             *nft_set_ext_flags(ext) & NFT_SET_ELEM_CATCHALL)
6394                 return true;
6395
6396         return false;
6397 }
6398
6399 static void nft_setelem_activate(struct net *net, struct nft_set *set,
6400                                  struct nft_set_elem *elem)
6401 {
6402         struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6403
6404         if (nft_setelem_is_catchall(set, elem)) {
6405                 nft_set_elem_change_active(net, set, ext);
6406         } else {
6407                 set->ops->activate(net, set, elem);
6408         }
6409 }
6410
6411 static int nft_setelem_catchall_deactivate(const struct net *net,
6412                                            struct nft_set *set,
6413                                            struct nft_set_elem *elem)
6414 {
6415         struct nft_set_elem_catchall *catchall;
6416         struct nft_set_ext *ext;
6417
6418         list_for_each_entry(catchall, &set->catchall_list, list) {
6419                 ext = nft_set_elem_ext(set, catchall->elem);
6420                 if (!nft_is_active(net, ext))
6421                         continue;
6422
6423                 kfree(elem->priv);
6424                 elem->priv = catchall->elem;
6425                 nft_set_elem_change_active(net, set, ext);
6426                 return 0;
6427         }
6428
6429         return -ENOENT;
6430 }
6431
6432 static int __nft_setelem_deactivate(const struct net *net,
6433                                     struct nft_set *set,
6434                                     struct nft_set_elem *elem)
6435 {
6436         void *priv;
6437
6438         priv = set->ops->deactivate(net, set, elem);
6439         if (!priv)
6440                 return -ENOENT;
6441
6442         kfree(elem->priv);
6443         elem->priv = priv;
6444         set->ndeact++;
6445
6446         return 0;
6447 }
6448
6449 static int nft_setelem_deactivate(const struct net *net,
6450                                   struct nft_set *set,
6451                                   struct nft_set_elem *elem, u32 flags)
6452 {
6453         int ret;
6454
6455         if (flags & NFT_SET_ELEM_CATCHALL)
6456                 ret = nft_setelem_catchall_deactivate(net, set, elem);
6457         else
6458                 ret = __nft_setelem_deactivate(net, set, elem);
6459
6460         return ret;
6461 }
6462
6463 static void nft_setelem_catchall_remove(const struct net *net,
6464                                         const struct nft_set *set,
6465                                         const struct nft_set_elem *elem)
6466 {
6467         struct nft_set_elem_catchall *catchall, *next;
6468
6469         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
6470                 if (catchall->elem == elem->priv) {
6471                         list_del_rcu(&catchall->list);
6472                         kfree_rcu(catchall, rcu);
6473                         break;
6474                 }
6475         }
6476 }
6477
6478 static void nft_setelem_remove(const struct net *net,
6479                                const struct nft_set *set,
6480                                const struct nft_set_elem *elem)
6481 {
6482         if (nft_setelem_is_catchall(set, elem))
6483                 nft_setelem_catchall_remove(net, set, elem);
6484         else
6485                 set->ops->remove(net, set, elem);
6486 }
6487
6488 static bool nft_setelem_valid_key_end(const struct nft_set *set,
6489                                       struct nlattr **nla, u32 flags)
6490 {
6491         if ((set->flags & (NFT_SET_CONCAT | NFT_SET_INTERVAL)) ==
6492                           (NFT_SET_CONCAT | NFT_SET_INTERVAL)) {
6493                 if (flags & NFT_SET_ELEM_INTERVAL_END)
6494                         return false;
6495
6496                 if (nla[NFTA_SET_ELEM_KEY_END] &&
6497                     flags & NFT_SET_ELEM_CATCHALL)
6498                         return false;
6499         } else {
6500                 if (nla[NFTA_SET_ELEM_KEY_END])
6501                         return false;
6502         }
6503
6504         return true;
6505 }
6506
6507 static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set,
6508                             const struct nlattr *attr, u32 nlmsg_flags)
6509 {
6510         struct nft_expr *expr_array[NFT_SET_EXPR_MAX] = {};
6511         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
6512         u8 genmask = nft_genmask_next(ctx->net);
6513         u32 flags = 0, size = 0, num_exprs = 0;
6514         struct nft_set_ext_tmpl tmpl;
6515         struct nft_set_ext *ext, *ext2;
6516         struct nft_set_elem elem;
6517         struct nft_set_binding *binding;
6518         struct nft_object *obj = NULL;
6519         struct nft_userdata *udata;
6520         struct nft_data_desc desc;
6521         enum nft_registers dreg;
6522         struct nft_trans *trans;
6523         u64 timeout;
6524         u64 expiration;
6525         int err, i;
6526         u8 ulen;
6527
6528         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
6529                                           nft_set_elem_policy, NULL);
6530         if (err < 0)
6531                 return err;
6532
6533         nft_set_ext_prepare(&tmpl);
6534
6535         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
6536         if (err < 0)
6537                 return err;
6538
6539         if (((flags & NFT_SET_ELEM_CATCHALL) && nla[NFTA_SET_ELEM_KEY]) ||
6540             (!(flags & NFT_SET_ELEM_CATCHALL) && !nla[NFTA_SET_ELEM_KEY]))
6541                 return -EINVAL;
6542
6543         if (flags != 0) {
6544                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
6545                 if (err < 0)
6546                         return err;
6547         }
6548
6549         if (set->flags & NFT_SET_MAP) {
6550                 if (nla[NFTA_SET_ELEM_DATA] == NULL &&
6551                     !(flags & NFT_SET_ELEM_INTERVAL_END))
6552                         return -EINVAL;
6553         } else {
6554                 if (nla[NFTA_SET_ELEM_DATA] != NULL)
6555                         return -EINVAL;
6556         }
6557
6558         if (set->flags & NFT_SET_OBJECT) {
6559                 if (!nla[NFTA_SET_ELEM_OBJREF] &&
6560                     !(flags & NFT_SET_ELEM_INTERVAL_END))
6561                         return -EINVAL;
6562         } else {
6563                 if (nla[NFTA_SET_ELEM_OBJREF])
6564                         return -EINVAL;
6565         }
6566
6567         if (!nft_setelem_valid_key_end(set, nla, flags))
6568                 return -EINVAL;
6569
6570         if ((flags & NFT_SET_ELEM_INTERVAL_END) &&
6571              (nla[NFTA_SET_ELEM_DATA] ||
6572               nla[NFTA_SET_ELEM_OBJREF] ||
6573               nla[NFTA_SET_ELEM_TIMEOUT] ||
6574               nla[NFTA_SET_ELEM_EXPIRATION] ||
6575               nla[NFTA_SET_ELEM_USERDATA] ||
6576               nla[NFTA_SET_ELEM_EXPR] ||
6577               nla[NFTA_SET_ELEM_KEY_END] ||
6578               nla[NFTA_SET_ELEM_EXPRESSIONS]))
6579                 return -EINVAL;
6580
6581         timeout = 0;
6582         if (nla[NFTA_SET_ELEM_TIMEOUT] != NULL) {
6583                 if (!(set->flags & NFT_SET_TIMEOUT))
6584                         return -EINVAL;
6585                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_TIMEOUT],
6586                                             &timeout);
6587                 if (err)
6588                         return err;
6589         } else if (set->flags & NFT_SET_TIMEOUT &&
6590                    !(flags & NFT_SET_ELEM_INTERVAL_END)) {
6591                 timeout = READ_ONCE(set->timeout);
6592         }
6593
6594         expiration = 0;
6595         if (nla[NFTA_SET_ELEM_EXPIRATION] != NULL) {
6596                 if (!(set->flags & NFT_SET_TIMEOUT))
6597                         return -EINVAL;
6598                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_EXPIRATION],
6599                                             &expiration);
6600                 if (err)
6601                         return err;
6602         }
6603
6604         if (nla[NFTA_SET_ELEM_EXPR]) {
6605                 struct nft_expr *expr;
6606
6607                 if (set->num_exprs && set->num_exprs != 1)
6608                         return -EOPNOTSUPP;
6609
6610                 expr = nft_set_elem_expr_alloc(ctx, set,
6611                                                nla[NFTA_SET_ELEM_EXPR]);
6612                 if (IS_ERR(expr))
6613                         return PTR_ERR(expr);
6614
6615                 expr_array[0] = expr;
6616                 num_exprs = 1;
6617
6618                 if (set->num_exprs && set->exprs[0]->ops != expr->ops) {
6619                         err = -EOPNOTSUPP;
6620                         goto err_set_elem_expr;
6621                 }
6622         } else if (nla[NFTA_SET_ELEM_EXPRESSIONS]) {
6623                 struct nft_expr *expr;
6624                 struct nlattr *tmp;
6625                 int left;
6626
6627                 i = 0;
6628                 nla_for_each_nested(tmp, nla[NFTA_SET_ELEM_EXPRESSIONS], left) {
6629                         if (i == NFT_SET_EXPR_MAX ||
6630                             (set->num_exprs && set->num_exprs == i)) {
6631                                 err = -E2BIG;
6632                                 goto err_set_elem_expr;
6633                         }
6634                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
6635                                 err = -EINVAL;
6636                                 goto err_set_elem_expr;
6637                         }
6638                         expr = nft_set_elem_expr_alloc(ctx, set, tmp);
6639                         if (IS_ERR(expr)) {
6640                                 err = PTR_ERR(expr);
6641                                 goto err_set_elem_expr;
6642                         }
6643                         expr_array[i] = expr;
6644                         num_exprs++;
6645
6646                         if (set->num_exprs && expr->ops != set->exprs[i]->ops) {
6647                                 err = -EOPNOTSUPP;
6648                                 goto err_set_elem_expr;
6649                         }
6650                         i++;
6651                 }
6652                 if (set->num_exprs && set->num_exprs != i) {
6653                         err = -EOPNOTSUPP;
6654                         goto err_set_elem_expr;
6655                 }
6656         } else if (set->num_exprs > 0 &&
6657                    !(flags & NFT_SET_ELEM_INTERVAL_END)) {
6658                 err = nft_set_elem_expr_clone(ctx, set, expr_array);
6659                 if (err < 0)
6660                         goto err_set_elem_expr_clone;
6661
6662                 num_exprs = set->num_exprs;
6663         }
6664
6665         if (nla[NFTA_SET_ELEM_KEY]) {
6666                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
6667                                             nla[NFTA_SET_ELEM_KEY]);
6668                 if (err < 0)
6669                         goto err_set_elem_expr;
6670
6671                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
6672                 if (err < 0)
6673                         goto err_parse_key;
6674         }
6675
6676         if (nla[NFTA_SET_ELEM_KEY_END]) {
6677                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
6678                                             nla[NFTA_SET_ELEM_KEY_END]);
6679                 if (err < 0)
6680                         goto err_parse_key;
6681
6682                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
6683                 if (err < 0)
6684                         goto err_parse_key_end;
6685         }
6686
6687         if (timeout > 0) {
6688                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_EXPIRATION);
6689                 if (err < 0)
6690                         goto err_parse_key_end;
6691
6692                 if (timeout != READ_ONCE(set->timeout)) {
6693                         err = nft_set_ext_add(&tmpl, NFT_SET_EXT_TIMEOUT);
6694                         if (err < 0)
6695                                 goto err_parse_key_end;
6696                 }
6697         }
6698
6699         if (num_exprs) {
6700                 for (i = 0; i < num_exprs; i++)
6701                         size += expr_array[i]->ops->size;
6702
6703                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_EXPRESSIONS,
6704                                              sizeof(struct nft_set_elem_expr) + size);
6705                 if (err < 0)
6706                         goto err_parse_key_end;
6707         }
6708
6709         if (nla[NFTA_SET_ELEM_OBJREF] != NULL) {
6710                 obj = nft_obj_lookup(ctx->net, ctx->table,
6711                                      nla[NFTA_SET_ELEM_OBJREF],
6712                                      set->objtype, genmask);
6713                 if (IS_ERR(obj)) {
6714                         err = PTR_ERR(obj);
6715                         obj = NULL;
6716                         goto err_parse_key_end;
6717                 }
6718
6719                 if (!nft_use_inc(&obj->use)) {
6720                         err = -EMFILE;
6721                         obj = NULL;
6722                         goto err_parse_key_end;
6723                 }
6724
6725                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_OBJREF);
6726                 if (err < 0)
6727                         goto err_parse_key_end;
6728         }
6729
6730         if (nla[NFTA_SET_ELEM_DATA] != NULL) {
6731                 err = nft_setelem_parse_data(ctx, set, &desc, &elem.data.val,
6732                                              nla[NFTA_SET_ELEM_DATA]);
6733                 if (err < 0)
6734                         goto err_parse_key_end;
6735
6736                 dreg = nft_type_to_reg(set->dtype);
6737                 list_for_each_entry(binding, &set->bindings, list) {
6738                         struct nft_ctx bind_ctx = {
6739                                 .net    = ctx->net,
6740                                 .family = ctx->family,
6741                                 .table  = ctx->table,
6742                                 .chain  = (struct nft_chain *)binding->chain,
6743                         };
6744
6745                         if (!(binding->flags & NFT_SET_MAP))
6746                                 continue;
6747
6748                         err = nft_validate_register_store(&bind_ctx, dreg,
6749                                                           &elem.data.val,
6750                                                           desc.type, desc.len);
6751                         if (err < 0)
6752                                 goto err_parse_data;
6753
6754                         if (desc.type == NFT_DATA_VERDICT &&
6755                             (elem.data.val.verdict.code == NFT_GOTO ||
6756                              elem.data.val.verdict.code == NFT_JUMP))
6757                                 nft_validate_state_update(ctx->table,
6758                                                           NFT_VALIDATE_NEED);
6759                 }
6760
6761                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_DATA, desc.len);
6762                 if (err < 0)
6763                         goto err_parse_data;
6764         }
6765
6766         /* The full maximum length of userdata can exceed the maximum
6767          * offset value (U8_MAX) for following extensions, therefor it
6768          * must be the last extension added.
6769          */
6770         ulen = 0;
6771         if (nla[NFTA_SET_ELEM_USERDATA] != NULL) {
6772                 ulen = nla_len(nla[NFTA_SET_ELEM_USERDATA]);
6773                 if (ulen > 0) {
6774                         err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_USERDATA,
6775                                                      ulen);
6776                         if (err < 0)
6777                                 goto err_parse_data;
6778                 }
6779         }
6780
6781         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
6782                                       elem.key_end.val.data, elem.data.val.data,
6783                                       timeout, expiration, GFP_KERNEL_ACCOUNT);
6784         if (IS_ERR(elem.priv)) {
6785                 err = PTR_ERR(elem.priv);
6786                 goto err_parse_data;
6787         }
6788
6789         ext = nft_set_elem_ext(set, elem.priv);
6790         if (flags)
6791                 *nft_set_ext_flags(ext) = flags;
6792
6793         if (obj)
6794                 *nft_set_ext_obj(ext) = obj;
6795
6796         if (ulen > 0) {
6797                 if (nft_set_ext_check(&tmpl, NFT_SET_EXT_USERDATA, ulen) < 0) {
6798                         err = -EINVAL;
6799                         goto err_elem_free;
6800                 }
6801                 udata = nft_set_ext_userdata(ext);
6802                 udata->len = ulen - 1;
6803                 nla_memcpy(&udata->data, nla[NFTA_SET_ELEM_USERDATA], ulen);
6804         }
6805         err = nft_set_elem_expr_setup(ctx, &tmpl, ext, expr_array, num_exprs);
6806         if (err < 0)
6807                 goto err_elem_free;
6808
6809         trans = nft_trans_elem_alloc(ctx, NFT_MSG_NEWSETELEM, set);
6810         if (trans == NULL) {
6811                 err = -ENOMEM;
6812                 goto err_elem_free;
6813         }
6814
6815         ext->genmask = nft_genmask_cur(ctx->net);
6816
6817         err = nft_setelem_insert(ctx->net, set, &elem, &ext2, flags);
6818         if (err) {
6819                 if (err == -EEXIST) {
6820                         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) ^
6821                             nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) ||
6822                             nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) ^
6823                             nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF))
6824                                 goto err_element_clash;
6825                         if ((nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
6826                              nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) &&
6827                              memcmp(nft_set_ext_data(ext),
6828                                     nft_set_ext_data(ext2), set->dlen) != 0) ||
6829                             (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
6830                              nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF) &&
6831                              *nft_set_ext_obj(ext) != *nft_set_ext_obj(ext2)))
6832                                 goto err_element_clash;
6833                         else if (!(nlmsg_flags & NLM_F_EXCL))
6834                                 err = 0;
6835                 } else if (err == -ENOTEMPTY) {
6836                         /* ENOTEMPTY reports overlapping between this element
6837                          * and an existing one.
6838                          */
6839                         err = -EEXIST;
6840                 }
6841                 goto err_element_clash;
6842         }
6843
6844         if (!(flags & NFT_SET_ELEM_CATCHALL)) {
6845                 unsigned int max = set->size ? set->size + set->ndeact : UINT_MAX;
6846
6847                 if (!atomic_add_unless(&set->nelems, 1, max)) {
6848                         err = -ENFILE;
6849                         goto err_set_full;
6850                 }
6851         }
6852
6853         nft_trans_elem(trans) = elem;
6854         nft_trans_commit_list_add_tail(ctx->net, trans);
6855         return 0;
6856
6857 err_set_full:
6858         nft_setelem_remove(ctx->net, set, &elem);
6859 err_element_clash:
6860         kfree(trans);
6861 err_elem_free:
6862         nf_tables_set_elem_destroy(ctx, set, elem.priv);
6863 err_parse_data:
6864         if (nla[NFTA_SET_ELEM_DATA] != NULL)
6865                 nft_data_release(&elem.data.val, desc.type);
6866 err_parse_key_end:
6867         if (obj)
6868                 nft_use_dec_restore(&obj->use);
6869
6870         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
6871 err_parse_key:
6872         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
6873 err_set_elem_expr:
6874         for (i = 0; i < num_exprs && expr_array[i]; i++)
6875                 nft_expr_destroy(ctx, expr_array[i]);
6876 err_set_elem_expr_clone:
6877         return err;
6878 }
6879
6880 static int nf_tables_newsetelem(struct sk_buff *skb,
6881                                 const struct nfnl_info *info,
6882                                 const struct nlattr * const nla[])
6883 {
6884         struct netlink_ext_ack *extack = info->extack;
6885         u8 genmask = nft_genmask_next(info->net);
6886         u8 family = info->nfmsg->nfgen_family;
6887         struct net *net = info->net;
6888         const struct nlattr *attr;
6889         struct nft_table *table;
6890         struct nft_set *set;
6891         struct nft_ctx ctx;
6892         int rem, err;
6893
6894         if (nla[NFTA_SET_ELEM_LIST_ELEMENTS] == NULL)
6895                 return -EINVAL;
6896
6897         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
6898                                  genmask, NETLINK_CB(skb).portid);
6899         if (IS_ERR(table)) {
6900                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
6901                 return PTR_ERR(table);
6902         }
6903
6904         set = nft_set_lookup_global(net, table, nla[NFTA_SET_ELEM_LIST_SET],
6905                                     nla[NFTA_SET_ELEM_LIST_SET_ID], genmask);
6906         if (IS_ERR(set))
6907                 return PTR_ERR(set);
6908
6909         if (!list_empty(&set->bindings) &&
6910             (set->flags & (NFT_SET_CONSTANT | NFT_SET_ANONYMOUS)))
6911                 return -EBUSY;
6912
6913         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
6914
6915         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
6916                 err = nft_add_set_elem(&ctx, set, attr, info->nlh->nlmsg_flags);
6917                 if (err < 0) {
6918                         NL_SET_BAD_ATTR(extack, attr);
6919                         return err;
6920                 }
6921         }
6922
6923         if (table->validate_state == NFT_VALIDATE_DO)
6924                 return nft_table_validate(net, table);
6925
6926         return 0;
6927 }
6928
6929 /**
6930  *      nft_data_hold - hold a nft_data item
6931  *
6932  *      @data: struct nft_data to release
6933  *      @type: type of data
6934  *
6935  *      Hold a nft_data item. NFT_DATA_VALUE types can be silently discarded,
6936  *      NFT_DATA_VERDICT bumps the reference to chains in case of NFT_JUMP and
6937  *      NFT_GOTO verdicts. This function must be called on active data objects
6938  *      from the second phase of the commit protocol.
6939  */
6940 void nft_data_hold(const struct nft_data *data, enum nft_data_types type)
6941 {
6942         struct nft_chain *chain;
6943
6944         if (type == NFT_DATA_VERDICT) {
6945                 switch (data->verdict.code) {
6946                 case NFT_JUMP:
6947                 case NFT_GOTO:
6948                         chain = data->verdict.chain;
6949                         nft_use_inc_restore(&chain->use);
6950                         break;
6951                 }
6952         }
6953 }
6954
6955 static void nft_setelem_data_activate(const struct net *net,
6956                                       const struct nft_set *set,
6957                                       struct nft_set_elem *elem)
6958 {
6959         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6960
6961         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
6962                 nft_data_hold(nft_set_ext_data(ext), set->dtype);
6963         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
6964                 nft_use_inc_restore(&(*nft_set_ext_obj(ext))->use);
6965 }
6966
6967 void nft_setelem_data_deactivate(const struct net *net,
6968                                  const struct nft_set *set,
6969                                  struct nft_set_elem *elem)
6970 {
6971         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6972
6973         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
6974                 nft_data_release(nft_set_ext_data(ext), set->dtype);
6975         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
6976                 nft_use_dec(&(*nft_set_ext_obj(ext))->use);
6977 }
6978
6979 static int nft_del_setelem(struct nft_ctx *ctx, struct nft_set *set,
6980                            const struct nlattr *attr)
6981 {
6982         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
6983         struct nft_set_ext_tmpl tmpl;
6984         struct nft_set_elem elem;
6985         struct nft_set_ext *ext;
6986         struct nft_trans *trans;
6987         u32 flags = 0;
6988         int err;
6989
6990         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
6991                                           nft_set_elem_policy, NULL);
6992         if (err < 0)
6993                 return err;
6994
6995         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
6996         if (err < 0)
6997                 return err;
6998
6999         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
7000                 return -EINVAL;
7001
7002         if (!nft_setelem_valid_key_end(set, nla, flags))
7003                 return -EINVAL;
7004
7005         nft_set_ext_prepare(&tmpl);
7006
7007         if (flags != 0) {
7008                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
7009                 if (err < 0)
7010                         return err;
7011         }
7012
7013         if (nla[NFTA_SET_ELEM_KEY]) {
7014                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
7015                                             nla[NFTA_SET_ELEM_KEY]);
7016                 if (err < 0)
7017                         return err;
7018
7019                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
7020                 if (err < 0)
7021                         goto fail_elem;
7022         }
7023
7024         if (nla[NFTA_SET_ELEM_KEY_END]) {
7025                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
7026                                             nla[NFTA_SET_ELEM_KEY_END]);
7027                 if (err < 0)
7028                         goto fail_elem;
7029
7030                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
7031                 if (err < 0)
7032                         goto fail_elem_key_end;
7033         }
7034
7035         err = -ENOMEM;
7036         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
7037                                       elem.key_end.val.data, NULL, 0, 0,
7038                                       GFP_KERNEL_ACCOUNT);
7039         if (IS_ERR(elem.priv)) {
7040                 err = PTR_ERR(elem.priv);
7041                 goto fail_elem_key_end;
7042         }
7043
7044         ext = nft_set_elem_ext(set, elem.priv);
7045         if (flags)
7046                 *nft_set_ext_flags(ext) = flags;
7047
7048         trans = nft_trans_elem_alloc(ctx, NFT_MSG_DELSETELEM, set);
7049         if (trans == NULL)
7050                 goto fail_trans;
7051
7052         err = nft_setelem_deactivate(ctx->net, set, &elem, flags);
7053         if (err < 0)
7054                 goto fail_ops;
7055
7056         nft_setelem_data_deactivate(ctx->net, set, &elem);
7057
7058         nft_trans_elem(trans) = elem;
7059         nft_trans_commit_list_add_tail(ctx->net, trans);
7060         return 0;
7061
7062 fail_ops:
7063         kfree(trans);
7064 fail_trans:
7065         kfree(elem.priv);
7066 fail_elem_key_end:
7067         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
7068 fail_elem:
7069         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
7070         return err;
7071 }
7072
7073 static int nft_setelem_flush(const struct nft_ctx *ctx,
7074                              struct nft_set *set,
7075                              const struct nft_set_iter *iter,
7076                              struct nft_set_elem *elem)
7077 {
7078         struct nft_trans *trans;
7079         int err;
7080
7081         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
7082                                     sizeof(struct nft_trans_elem), GFP_ATOMIC);
7083         if (!trans)
7084                 return -ENOMEM;
7085
7086         if (!set->ops->flush(ctx->net, set, elem->priv)) {
7087                 err = -ENOENT;
7088                 goto err1;
7089         }
7090         set->ndeact++;
7091
7092         nft_setelem_data_deactivate(ctx->net, set, elem);
7093         nft_trans_elem_set(trans) = set;
7094         nft_trans_elem(trans) = *elem;
7095         nft_trans_commit_list_add_tail(ctx->net, trans);
7096
7097         return 0;
7098 err1:
7099         kfree(trans);
7100         return err;
7101 }
7102
7103 static int __nft_set_catchall_flush(const struct nft_ctx *ctx,
7104                                     struct nft_set *set,
7105                                     struct nft_set_elem *elem)
7106 {
7107         struct nft_trans *trans;
7108
7109         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
7110                                     sizeof(struct nft_trans_elem), GFP_KERNEL);
7111         if (!trans)
7112                 return -ENOMEM;
7113
7114         nft_setelem_data_deactivate(ctx->net, set, elem);
7115         nft_trans_elem_set(trans) = set;
7116         nft_trans_elem(trans) = *elem;
7117         nft_trans_commit_list_add_tail(ctx->net, trans);
7118
7119         return 0;
7120 }
7121
7122 static int nft_set_catchall_flush(const struct nft_ctx *ctx,
7123                                   struct nft_set *set)
7124 {
7125         u8 genmask = nft_genmask_next(ctx->net);
7126         struct nft_set_elem_catchall *catchall;
7127         struct nft_set_elem elem;
7128         struct nft_set_ext *ext;
7129         int ret = 0;
7130
7131         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
7132                 ext = nft_set_elem_ext(set, catchall->elem);
7133                 if (!nft_set_elem_active(ext, genmask))
7134                         continue;
7135
7136                 elem.priv = catchall->elem;
7137                 ret = __nft_set_catchall_flush(ctx, set, &elem);
7138                 if (ret < 0)
7139                         break;
7140                 nft_set_elem_change_active(ctx->net, set, ext);
7141         }
7142
7143         return ret;
7144 }
7145
7146 static int nft_set_flush(struct nft_ctx *ctx, struct nft_set *set, u8 genmask)
7147 {
7148         struct nft_set_iter iter = {
7149                 .genmask        = genmask,
7150                 .fn             = nft_setelem_flush,
7151         };
7152
7153         set->ops->walk(ctx, set, &iter);
7154         if (!iter.err)
7155                 iter.err = nft_set_catchall_flush(ctx, set);
7156
7157         return iter.err;
7158 }
7159
7160 static int nf_tables_delsetelem(struct sk_buff *skb,
7161                                 const struct nfnl_info *info,
7162                                 const struct nlattr * const nla[])
7163 {
7164         struct netlink_ext_ack *extack = info->extack;
7165         u8 genmask = nft_genmask_next(info->net);
7166         u8 family = info->nfmsg->nfgen_family;
7167         struct net *net = info->net;
7168         const struct nlattr *attr;
7169         struct nft_table *table;
7170         struct nft_set *set;
7171         struct nft_ctx ctx;
7172         int rem, err = 0;
7173
7174         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
7175                                  genmask, NETLINK_CB(skb).portid);
7176         if (IS_ERR(table)) {
7177                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
7178                 return PTR_ERR(table);
7179         }
7180
7181         set = nft_set_lookup(table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
7182         if (IS_ERR(set))
7183                 return PTR_ERR(set);
7184
7185         if (!list_empty(&set->bindings) &&
7186             (set->flags & (NFT_SET_CONSTANT | NFT_SET_ANONYMOUS)))
7187                 return -EBUSY;
7188
7189         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7190
7191         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
7192                 return nft_set_flush(&ctx, set, genmask);
7193
7194         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
7195                 err = nft_del_setelem(&ctx, set, attr);
7196                 if (err == -ENOENT &&
7197                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYSETELEM)
7198                         continue;
7199
7200                 if (err < 0) {
7201                         NL_SET_BAD_ATTR(extack, attr);
7202                         break;
7203                 }
7204         }
7205         return err;
7206 }
7207
7208 /*
7209  * Stateful objects
7210  */
7211
7212 /**
7213  *      nft_register_obj- register nf_tables stateful object type
7214  *      @obj_type: object type
7215  *
7216  *      Registers the object type for use with nf_tables. Returns zero on
7217  *      success or a negative errno code otherwise.
7218  */
7219 int nft_register_obj(struct nft_object_type *obj_type)
7220 {
7221         if (obj_type->type == NFT_OBJECT_UNSPEC)
7222                 return -EINVAL;
7223
7224         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7225         list_add_rcu(&obj_type->list, &nf_tables_objects);
7226         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7227         return 0;
7228 }
7229 EXPORT_SYMBOL_GPL(nft_register_obj);
7230
7231 /**
7232  *      nft_unregister_obj - unregister nf_tables object type
7233  *      @obj_type: object type
7234  *
7235  *      Unregisters the object type for use with nf_tables.
7236  */
7237 void nft_unregister_obj(struct nft_object_type *obj_type)
7238 {
7239         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7240         list_del_rcu(&obj_type->list);
7241         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7242 }
7243 EXPORT_SYMBOL_GPL(nft_unregister_obj);
7244
7245 struct nft_object *nft_obj_lookup(const struct net *net,
7246                                   const struct nft_table *table,
7247                                   const struct nlattr *nla, u32 objtype,
7248                                   u8 genmask)
7249 {
7250         struct nft_object_hash_key k = { .table = table };
7251         char search[NFT_OBJ_MAXNAMELEN];
7252         struct rhlist_head *tmp, *list;
7253         struct nft_object *obj;
7254
7255         nla_strscpy(search, nla, sizeof(search));
7256         k.name = search;
7257
7258         WARN_ON_ONCE(!rcu_read_lock_held() &&
7259                      !lockdep_commit_lock_is_held(net));
7260
7261         rcu_read_lock();
7262         list = rhltable_lookup(&nft_objname_ht, &k, nft_objname_ht_params);
7263         if (!list)
7264                 goto out;
7265
7266         rhl_for_each_entry_rcu(obj, tmp, list, rhlhead) {
7267                 if (objtype == obj->ops->type->type &&
7268                     nft_active_genmask(obj, genmask)) {
7269                         rcu_read_unlock();
7270                         return obj;
7271                 }
7272         }
7273 out:
7274         rcu_read_unlock();
7275         return ERR_PTR(-ENOENT);
7276 }
7277 EXPORT_SYMBOL_GPL(nft_obj_lookup);
7278
7279 static struct nft_object *nft_obj_lookup_byhandle(const struct nft_table *table,
7280                                                   const struct nlattr *nla,
7281                                                   u32 objtype, u8 genmask)
7282 {
7283         struct nft_object *obj;
7284
7285         list_for_each_entry(obj, &table->objects, list) {
7286                 if (be64_to_cpu(nla_get_be64(nla)) == obj->handle &&
7287                     objtype == obj->ops->type->type &&
7288                     nft_active_genmask(obj, genmask))
7289                         return obj;
7290         }
7291         return ERR_PTR(-ENOENT);
7292 }
7293
7294 static const struct nla_policy nft_obj_policy[NFTA_OBJ_MAX + 1] = {
7295         [NFTA_OBJ_TABLE]        = { .type = NLA_STRING,
7296                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
7297         [NFTA_OBJ_NAME]         = { .type = NLA_STRING,
7298                                     .len = NFT_OBJ_MAXNAMELEN - 1 },
7299         [NFTA_OBJ_TYPE]         = { .type = NLA_U32 },
7300         [NFTA_OBJ_DATA]         = { .type = NLA_NESTED },
7301         [NFTA_OBJ_HANDLE]       = { .type = NLA_U64},
7302         [NFTA_OBJ_USERDATA]     = { .type = NLA_BINARY,
7303                                     .len = NFT_USERDATA_MAXLEN },
7304 };
7305
7306 static struct nft_object *nft_obj_init(const struct nft_ctx *ctx,
7307                                        const struct nft_object_type *type,
7308                                        const struct nlattr *attr)
7309 {
7310         struct nlattr **tb;
7311         const struct nft_object_ops *ops;
7312         struct nft_object *obj;
7313         int err = -ENOMEM;
7314
7315         tb = kmalloc_array(type->maxattr + 1, sizeof(*tb), GFP_KERNEL);
7316         if (!tb)
7317                 goto err1;
7318
7319         if (attr) {
7320                 err = nla_parse_nested_deprecated(tb, type->maxattr, attr,
7321                                                   type->policy, NULL);
7322                 if (err < 0)
7323                         goto err2;
7324         } else {
7325                 memset(tb, 0, sizeof(tb[0]) * (type->maxattr + 1));
7326         }
7327
7328         if (type->select_ops) {
7329                 ops = type->select_ops(ctx, (const struct nlattr * const *)tb);
7330                 if (IS_ERR(ops)) {
7331                         err = PTR_ERR(ops);
7332                         goto err2;
7333                 }
7334         } else {
7335                 ops = type->ops;
7336         }
7337
7338         err = -ENOMEM;
7339         obj = kzalloc(sizeof(*obj) + ops->size, GFP_KERNEL_ACCOUNT);
7340         if (!obj)
7341                 goto err2;
7342
7343         err = ops->init(ctx, (const struct nlattr * const *)tb, obj);
7344         if (err < 0)
7345                 goto err3;
7346
7347         obj->ops = ops;
7348
7349         kfree(tb);
7350         return obj;
7351 err3:
7352         kfree(obj);
7353 err2:
7354         kfree(tb);
7355 err1:
7356         return ERR_PTR(err);
7357 }
7358
7359 static int nft_object_dump(struct sk_buff *skb, unsigned int attr,
7360                            struct nft_object *obj, bool reset)
7361 {
7362         struct nlattr *nest;
7363
7364         nest = nla_nest_start_noflag(skb, attr);
7365         if (!nest)
7366                 goto nla_put_failure;
7367         if (obj->ops->dump(skb, obj, reset) < 0)
7368                 goto nla_put_failure;
7369         nla_nest_end(skb, nest);
7370         return 0;
7371
7372 nla_put_failure:
7373         return -1;
7374 }
7375
7376 static const struct nft_object_type *__nft_obj_type_get(u32 objtype)
7377 {
7378         const struct nft_object_type *type;
7379
7380         list_for_each_entry(type, &nf_tables_objects, list) {
7381                 if (objtype == type->type)
7382                         return type;
7383         }
7384         return NULL;
7385 }
7386
7387 static const struct nft_object_type *
7388 nft_obj_type_get(struct net *net, u32 objtype)
7389 {
7390         const struct nft_object_type *type;
7391
7392         type = __nft_obj_type_get(objtype);
7393         if (type != NULL && try_module_get(type->owner))
7394                 return type;
7395
7396         lockdep_nfnl_nft_mutex_not_held();
7397 #ifdef CONFIG_MODULES
7398         if (type == NULL) {
7399                 if (nft_request_module(net, "nft-obj-%u", objtype) == -EAGAIN)
7400                         return ERR_PTR(-EAGAIN);
7401         }
7402 #endif
7403         return ERR_PTR(-ENOENT);
7404 }
7405
7406 static int nf_tables_updobj(const struct nft_ctx *ctx,
7407                             const struct nft_object_type *type,
7408                             const struct nlattr *attr,
7409                             struct nft_object *obj)
7410 {
7411         struct nft_object *newobj;
7412         struct nft_trans *trans;
7413         int err = -ENOMEM;
7414
7415         if (!try_module_get(type->owner))
7416                 return -ENOENT;
7417
7418         trans = nft_trans_alloc(ctx, NFT_MSG_NEWOBJ,
7419                                 sizeof(struct nft_trans_obj));
7420         if (!trans)
7421                 goto err_trans;
7422
7423         newobj = nft_obj_init(ctx, type, attr);
7424         if (IS_ERR(newobj)) {
7425                 err = PTR_ERR(newobj);
7426                 goto err_free_trans;
7427         }
7428
7429         nft_trans_obj(trans) = obj;
7430         nft_trans_obj_update(trans) = true;
7431         nft_trans_obj_newobj(trans) = newobj;
7432         nft_trans_commit_list_add_tail(ctx->net, trans);
7433
7434         return 0;
7435
7436 err_free_trans:
7437         kfree(trans);
7438 err_trans:
7439         module_put(type->owner);
7440         return err;
7441 }
7442
7443 static int nf_tables_newobj(struct sk_buff *skb, const struct nfnl_info *info,
7444                             const struct nlattr * const nla[])
7445 {
7446         struct netlink_ext_ack *extack = info->extack;
7447         u8 genmask = nft_genmask_next(info->net);
7448         u8 family = info->nfmsg->nfgen_family;
7449         const struct nft_object_type *type;
7450         struct net *net = info->net;
7451         struct nft_table *table;
7452         struct nft_object *obj;
7453         struct nft_ctx ctx;
7454         u32 objtype;
7455         int err;
7456
7457         if (!nla[NFTA_OBJ_TYPE] ||
7458             !nla[NFTA_OBJ_NAME] ||
7459             !nla[NFTA_OBJ_DATA])
7460                 return -EINVAL;
7461
7462         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
7463                                  NETLINK_CB(skb).portid);
7464         if (IS_ERR(table)) {
7465                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7466                 return PTR_ERR(table);
7467         }
7468
7469         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7470         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
7471         if (IS_ERR(obj)) {
7472                 err = PTR_ERR(obj);
7473                 if (err != -ENOENT) {
7474                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7475                         return err;
7476                 }
7477         } else {
7478                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
7479                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7480                         return -EEXIST;
7481                 }
7482                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
7483                         return -EOPNOTSUPP;
7484
7485                 type = __nft_obj_type_get(objtype);
7486                 if (WARN_ON_ONCE(!type))
7487                         return -ENOENT;
7488
7489                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7490
7491                 return nf_tables_updobj(&ctx, type, nla[NFTA_OBJ_DATA], obj);
7492         }
7493
7494         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7495
7496         if (!nft_use_inc(&table->use))
7497                 return -EMFILE;
7498
7499         type = nft_obj_type_get(net, objtype);
7500         if (IS_ERR(type)) {
7501                 err = PTR_ERR(type);
7502                 goto err_type;
7503         }
7504
7505         obj = nft_obj_init(&ctx, type, nla[NFTA_OBJ_DATA]);
7506         if (IS_ERR(obj)) {
7507                 err = PTR_ERR(obj);
7508                 goto err_init;
7509         }
7510         obj->key.table = table;
7511         obj->handle = nf_tables_alloc_handle(table);
7512
7513         obj->key.name = nla_strdup(nla[NFTA_OBJ_NAME], GFP_KERNEL_ACCOUNT);
7514         if (!obj->key.name) {
7515                 err = -ENOMEM;
7516                 goto err_strdup;
7517         }
7518
7519         if (nla[NFTA_OBJ_USERDATA]) {
7520                 obj->udata = nla_memdup(nla[NFTA_OBJ_USERDATA], GFP_KERNEL_ACCOUNT);
7521                 if (obj->udata == NULL)
7522                         goto err_userdata;
7523
7524                 obj->udlen = nla_len(nla[NFTA_OBJ_USERDATA]);
7525         }
7526
7527         err = nft_trans_obj_add(&ctx, NFT_MSG_NEWOBJ, obj);
7528         if (err < 0)
7529                 goto err_trans;
7530
7531         err = rhltable_insert(&nft_objname_ht, &obj->rhlhead,
7532                               nft_objname_ht_params);
7533         if (err < 0)
7534                 goto err_obj_ht;
7535
7536         list_add_tail_rcu(&obj->list, &table->objects);
7537
7538         return 0;
7539 err_obj_ht:
7540         /* queued in transaction log */
7541         INIT_LIST_HEAD(&obj->list);
7542         return err;
7543 err_trans:
7544         kfree(obj->udata);
7545 err_userdata:
7546         kfree(obj->key.name);
7547 err_strdup:
7548         if (obj->ops->destroy)
7549                 obj->ops->destroy(&ctx, obj);
7550         kfree(obj);
7551 err_init:
7552         module_put(type->owner);
7553 err_type:
7554         nft_use_dec_restore(&table->use);
7555
7556         return err;
7557 }
7558
7559 static int nf_tables_fill_obj_info(struct sk_buff *skb, struct net *net,
7560                                    u32 portid, u32 seq, int event, u32 flags,
7561                                    int family, const struct nft_table *table,
7562                                    struct nft_object *obj, bool reset)
7563 {
7564         struct nlmsghdr *nlh;
7565
7566         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
7567         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
7568                            NFNETLINK_V0, nft_base_seq(net));
7569         if (!nlh)
7570                 goto nla_put_failure;
7571
7572         if (nla_put_string(skb, NFTA_OBJ_TABLE, table->name) ||
7573             nla_put_string(skb, NFTA_OBJ_NAME, obj->key.name) ||
7574             nla_put_be64(skb, NFTA_OBJ_HANDLE, cpu_to_be64(obj->handle),
7575                          NFTA_OBJ_PAD))
7576                 goto nla_put_failure;
7577
7578         if (event == NFT_MSG_DELOBJ) {
7579                 nlmsg_end(skb, nlh);
7580                 return 0;
7581         }
7582
7583         if (nla_put_be32(skb, NFTA_OBJ_TYPE, htonl(obj->ops->type->type)) ||
7584             nla_put_be32(skb, NFTA_OBJ_USE, htonl(obj->use)) ||
7585             nft_object_dump(skb, NFTA_OBJ_DATA, obj, reset))
7586                 goto nla_put_failure;
7587
7588         if (obj->udata &&
7589             nla_put(skb, NFTA_OBJ_USERDATA, obj->udlen, obj->udata))
7590                 goto nla_put_failure;
7591
7592         nlmsg_end(skb, nlh);
7593         return 0;
7594
7595 nla_put_failure:
7596         nlmsg_trim(skb, nlh);
7597         return -1;
7598 }
7599
7600 struct nft_obj_filter {
7601         char            *table;
7602         u32             type;
7603 };
7604
7605 static int nf_tables_dump_obj(struct sk_buff *skb, struct netlink_callback *cb)
7606 {
7607         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
7608         const struct nft_table *table;
7609         unsigned int idx = 0, s_idx = cb->args[0];
7610         struct nft_obj_filter *filter = cb->data;
7611         struct net *net = sock_net(skb->sk);
7612         int family = nfmsg->nfgen_family;
7613         struct nftables_pernet *nft_net;
7614         struct nft_object *obj;
7615         bool reset = false;
7616
7617         if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
7618                 reset = true;
7619
7620         rcu_read_lock();
7621         nft_net = nft_pernet(net);
7622         cb->seq = READ_ONCE(nft_net->base_seq);
7623
7624         list_for_each_entry_rcu(table, &nft_net->tables, list) {
7625                 if (family != NFPROTO_UNSPEC && family != table->family)
7626                         continue;
7627
7628                 list_for_each_entry_rcu(obj, &table->objects, list) {
7629                         if (!nft_is_active(net, obj))
7630                                 goto cont;
7631                         if (idx < s_idx)
7632                                 goto cont;
7633                         if (idx > s_idx)
7634                                 memset(&cb->args[1], 0,
7635                                        sizeof(cb->args) - sizeof(cb->args[0]));
7636                         if (filter && filter->table &&
7637                             strcmp(filter->table, table->name))
7638                                 goto cont;
7639                         if (filter &&
7640                             filter->type != NFT_OBJECT_UNSPEC &&
7641                             obj->ops->type->type != filter->type)
7642                                 goto cont;
7643                         if (reset) {
7644                                 char *buf = kasprintf(GFP_ATOMIC,
7645                                                       "%s:%u",
7646                                                       table->name,
7647                                                       nft_net->base_seq);
7648
7649                                 audit_log_nfcfg(buf,
7650                                                 family,
7651                                                 obj->handle,
7652                                                 AUDIT_NFT_OP_OBJ_RESET,
7653                                                 GFP_ATOMIC);
7654                                 kfree(buf);
7655                         }
7656
7657                         if (nf_tables_fill_obj_info(skb, net, NETLINK_CB(cb->skb).portid,
7658                                                     cb->nlh->nlmsg_seq,
7659                                                     NFT_MSG_NEWOBJ,
7660                                                     NLM_F_MULTI | NLM_F_APPEND,
7661                                                     table->family, table,
7662                                                     obj, reset) < 0)
7663                                 goto done;
7664
7665                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
7666 cont:
7667                         idx++;
7668                 }
7669         }
7670 done:
7671         rcu_read_unlock();
7672
7673         cb->args[0] = idx;
7674         return skb->len;
7675 }
7676
7677 static int nf_tables_dump_obj_start(struct netlink_callback *cb)
7678 {
7679         const struct nlattr * const *nla = cb->data;
7680         struct nft_obj_filter *filter = NULL;
7681
7682         if (nla[NFTA_OBJ_TABLE] || nla[NFTA_OBJ_TYPE]) {
7683                 filter = kzalloc(sizeof(*filter), GFP_ATOMIC);
7684                 if (!filter)
7685                         return -ENOMEM;
7686
7687                 if (nla[NFTA_OBJ_TABLE]) {
7688                         filter->table = nla_strdup(nla[NFTA_OBJ_TABLE], GFP_ATOMIC);
7689                         if (!filter->table) {
7690                                 kfree(filter);
7691                                 return -ENOMEM;
7692                         }
7693                 }
7694
7695                 if (nla[NFTA_OBJ_TYPE])
7696                         filter->type = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7697         }
7698
7699         cb->data = filter;
7700         return 0;
7701 }
7702
7703 static int nf_tables_dump_obj_done(struct netlink_callback *cb)
7704 {
7705         struct nft_obj_filter *filter = cb->data;
7706
7707         if (filter) {
7708                 kfree(filter->table);
7709                 kfree(filter);
7710         }
7711
7712         return 0;
7713 }
7714
7715 /* called with rcu_read_lock held */
7716 static int nf_tables_getobj(struct sk_buff *skb, const struct nfnl_info *info,
7717                             const struct nlattr * const nla[])
7718 {
7719         struct netlink_ext_ack *extack = info->extack;
7720         u8 genmask = nft_genmask_cur(info->net);
7721         u8 family = info->nfmsg->nfgen_family;
7722         const struct nft_table *table;
7723         struct net *net = info->net;
7724         struct nft_object *obj;
7725         struct sk_buff *skb2;
7726         bool reset = false;
7727         u32 objtype;
7728         int err;
7729
7730         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
7731                 struct netlink_dump_control c = {
7732                         .start = nf_tables_dump_obj_start,
7733                         .dump = nf_tables_dump_obj,
7734                         .done = nf_tables_dump_obj_done,
7735                         .module = THIS_MODULE,
7736                         .data = (void *)nla,
7737                 };
7738
7739                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
7740         }
7741
7742         if (!nla[NFTA_OBJ_NAME] ||
7743             !nla[NFTA_OBJ_TYPE])
7744                 return -EINVAL;
7745
7746         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask, 0);
7747         if (IS_ERR(table)) {
7748                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7749                 return PTR_ERR(table);
7750         }
7751
7752         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7753         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
7754         if (IS_ERR(obj)) {
7755                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7756                 return PTR_ERR(obj);
7757         }
7758
7759         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
7760         if (!skb2)
7761                 return -ENOMEM;
7762
7763         if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
7764                 reset = true;
7765
7766         if (reset) {
7767                 const struct nftables_pernet *nft_net;
7768                 char *buf;
7769
7770                 nft_net = nft_pernet(net);
7771                 buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, nft_net->base_seq);
7772
7773                 audit_log_nfcfg(buf,
7774                                 family,
7775                                 obj->handle,
7776                                 AUDIT_NFT_OP_OBJ_RESET,
7777                                 GFP_ATOMIC);
7778                 kfree(buf);
7779         }
7780
7781         err = nf_tables_fill_obj_info(skb2, net, NETLINK_CB(skb).portid,
7782                                       info->nlh->nlmsg_seq, NFT_MSG_NEWOBJ, 0,
7783                                       family, table, obj, reset);
7784         if (err < 0)
7785                 goto err_fill_obj_info;
7786
7787         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
7788
7789 err_fill_obj_info:
7790         kfree_skb(skb2);
7791         return err;
7792 }
7793
7794 static void nft_obj_destroy(const struct nft_ctx *ctx, struct nft_object *obj)
7795 {
7796         if (obj->ops->destroy)
7797                 obj->ops->destroy(ctx, obj);
7798
7799         module_put(obj->ops->type->owner);
7800         kfree(obj->key.name);
7801         kfree(obj->udata);
7802         kfree(obj);
7803 }
7804
7805 static int nf_tables_delobj(struct sk_buff *skb, const struct nfnl_info *info,
7806                             const struct nlattr * const nla[])
7807 {
7808         struct netlink_ext_ack *extack = info->extack;
7809         u8 genmask = nft_genmask_next(info->net);
7810         u8 family = info->nfmsg->nfgen_family;
7811         struct net *net = info->net;
7812         const struct nlattr *attr;
7813         struct nft_table *table;
7814         struct nft_object *obj;
7815         struct nft_ctx ctx;
7816         u32 objtype;
7817
7818         if (!nla[NFTA_OBJ_TYPE] ||
7819             (!nla[NFTA_OBJ_NAME] && !nla[NFTA_OBJ_HANDLE]))
7820                 return -EINVAL;
7821
7822         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
7823                                  NETLINK_CB(skb).portid);
7824         if (IS_ERR(table)) {
7825                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7826                 return PTR_ERR(table);
7827         }
7828
7829         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7830         if (nla[NFTA_OBJ_HANDLE]) {
7831                 attr = nla[NFTA_OBJ_HANDLE];
7832                 obj = nft_obj_lookup_byhandle(table, attr, objtype, genmask);
7833         } else {
7834                 attr = nla[NFTA_OBJ_NAME];
7835                 obj = nft_obj_lookup(net, table, attr, objtype, genmask);
7836         }
7837
7838         if (IS_ERR(obj)) {
7839                 if (PTR_ERR(obj) == -ENOENT &&
7840                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYOBJ)
7841                         return 0;
7842
7843                 NL_SET_BAD_ATTR(extack, attr);
7844                 return PTR_ERR(obj);
7845         }
7846         if (obj->use > 0) {
7847                 NL_SET_BAD_ATTR(extack, attr);
7848                 return -EBUSY;
7849         }
7850
7851         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7852
7853         return nft_delobj(&ctx, obj);
7854 }
7855
7856 void nft_obj_notify(struct net *net, const struct nft_table *table,
7857                     struct nft_object *obj, u32 portid, u32 seq, int event,
7858                     u16 flags, int family, int report, gfp_t gfp)
7859 {
7860         struct nftables_pernet *nft_net = nft_pernet(net);
7861         struct sk_buff *skb;
7862         int err;
7863         char *buf = kasprintf(gfp, "%s:%u",
7864                               table->name, nft_net->base_seq);
7865
7866         audit_log_nfcfg(buf,
7867                         family,
7868                         obj->handle,
7869                         event == NFT_MSG_NEWOBJ ?
7870                                  AUDIT_NFT_OP_OBJ_REGISTER :
7871                                  AUDIT_NFT_OP_OBJ_UNREGISTER,
7872                         gfp);
7873         kfree(buf);
7874
7875         if (!report &&
7876             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
7877                 return;
7878
7879         skb = nlmsg_new(NLMSG_GOODSIZE, gfp);
7880         if (skb == NULL)
7881                 goto err;
7882
7883         err = nf_tables_fill_obj_info(skb, net, portid, seq, event,
7884                                       flags & (NLM_F_CREATE | NLM_F_EXCL),
7885                                       family, table, obj, false);
7886         if (err < 0) {
7887                 kfree_skb(skb);
7888                 goto err;
7889         }
7890
7891         nft_notify_enqueue(skb, report, &nft_net->notify_list);
7892         return;
7893 err:
7894         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
7895 }
7896 EXPORT_SYMBOL_GPL(nft_obj_notify);
7897
7898 static void nf_tables_obj_notify(const struct nft_ctx *ctx,
7899                                  struct nft_object *obj, int event)
7900 {
7901         nft_obj_notify(ctx->net, ctx->table, obj, ctx->portid, ctx->seq, event,
7902                        ctx->flags, ctx->family, ctx->report, GFP_KERNEL);
7903 }
7904
7905 /*
7906  * Flow tables
7907  */
7908 void nft_register_flowtable_type(struct nf_flowtable_type *type)
7909 {
7910         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7911         list_add_tail_rcu(&type->list, &nf_tables_flowtables);
7912         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7913 }
7914 EXPORT_SYMBOL_GPL(nft_register_flowtable_type);
7915
7916 void nft_unregister_flowtable_type(struct nf_flowtable_type *type)
7917 {
7918         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7919         list_del_rcu(&type->list);
7920         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7921 }
7922 EXPORT_SYMBOL_GPL(nft_unregister_flowtable_type);
7923
7924 static const struct nla_policy nft_flowtable_policy[NFTA_FLOWTABLE_MAX + 1] = {
7925         [NFTA_FLOWTABLE_TABLE]          = { .type = NLA_STRING,
7926                                             .len = NFT_NAME_MAXLEN - 1 },
7927         [NFTA_FLOWTABLE_NAME]           = { .type = NLA_STRING,
7928                                             .len = NFT_NAME_MAXLEN - 1 },
7929         [NFTA_FLOWTABLE_HOOK]           = { .type = NLA_NESTED },
7930         [NFTA_FLOWTABLE_HANDLE]         = { .type = NLA_U64 },
7931         [NFTA_FLOWTABLE_FLAGS]          = { .type = NLA_U32 },
7932 };
7933
7934 struct nft_flowtable *nft_flowtable_lookup(const struct nft_table *table,
7935                                            const struct nlattr *nla, u8 genmask)
7936 {
7937         struct nft_flowtable *flowtable;
7938
7939         list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
7940                 if (!nla_strcmp(nla, flowtable->name) &&
7941                     nft_active_genmask(flowtable, genmask))
7942                         return flowtable;
7943         }
7944         return ERR_PTR(-ENOENT);
7945 }
7946 EXPORT_SYMBOL_GPL(nft_flowtable_lookup);
7947
7948 void nf_tables_deactivate_flowtable(const struct nft_ctx *ctx,
7949                                     struct nft_flowtable *flowtable,
7950                                     enum nft_trans_phase phase)
7951 {
7952         switch (phase) {
7953         case NFT_TRANS_PREPARE_ERROR:
7954         case NFT_TRANS_PREPARE:
7955         case NFT_TRANS_ABORT:
7956         case NFT_TRANS_RELEASE:
7957                 nft_use_dec(&flowtable->use);
7958                 fallthrough;
7959         default:
7960                 return;
7961         }
7962 }
7963 EXPORT_SYMBOL_GPL(nf_tables_deactivate_flowtable);
7964
7965 static struct nft_flowtable *
7966 nft_flowtable_lookup_byhandle(const struct nft_table *table,
7967                               const struct nlattr *nla, u8 genmask)
7968 {
7969        struct nft_flowtable *flowtable;
7970
7971        list_for_each_entry(flowtable, &table->flowtables, list) {
7972                if (be64_to_cpu(nla_get_be64(nla)) == flowtable->handle &&
7973                    nft_active_genmask(flowtable, genmask))
7974                        return flowtable;
7975        }
7976        return ERR_PTR(-ENOENT);
7977 }
7978
7979 struct nft_flowtable_hook {
7980         u32                     num;
7981         int                     priority;
7982         struct list_head        list;
7983 };
7984
7985 static const struct nla_policy nft_flowtable_hook_policy[NFTA_FLOWTABLE_HOOK_MAX + 1] = {
7986         [NFTA_FLOWTABLE_HOOK_NUM]       = { .type = NLA_U32 },
7987         [NFTA_FLOWTABLE_HOOK_PRIORITY]  = { .type = NLA_U32 },
7988         [NFTA_FLOWTABLE_HOOK_DEVS]      = { .type = NLA_NESTED },
7989 };
7990
7991 static int nft_flowtable_parse_hook(const struct nft_ctx *ctx,
7992                                     const struct nlattr * const nla[],
7993                                     struct nft_flowtable_hook *flowtable_hook,
7994                                     struct nft_flowtable *flowtable,
7995                                     struct netlink_ext_ack *extack, bool add)
7996 {
7997         struct nlattr *tb[NFTA_FLOWTABLE_HOOK_MAX + 1];
7998         struct nft_hook *hook;
7999         int hooknum, priority;
8000         int err;
8001
8002         INIT_LIST_HEAD(&flowtable_hook->list);
8003
8004         err = nla_parse_nested_deprecated(tb, NFTA_FLOWTABLE_HOOK_MAX,
8005                                           nla[NFTA_FLOWTABLE_HOOK],
8006                                           nft_flowtable_hook_policy, NULL);
8007         if (err < 0)
8008                 return err;
8009
8010         if (add) {
8011                 if (!tb[NFTA_FLOWTABLE_HOOK_NUM] ||
8012                     !tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
8013                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8014                         return -ENOENT;
8015                 }
8016
8017                 hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
8018                 if (hooknum != NF_NETDEV_INGRESS)
8019                         return -EOPNOTSUPP;
8020
8021                 priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
8022
8023                 flowtable_hook->priority        = priority;
8024                 flowtable_hook->num             = hooknum;
8025         } else {
8026                 if (tb[NFTA_FLOWTABLE_HOOK_NUM]) {
8027                         hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
8028                         if (hooknum != flowtable->hooknum)
8029                                 return -EOPNOTSUPP;
8030                 }
8031
8032                 if (tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
8033                         priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
8034                         if (priority != flowtable->data.priority)
8035                                 return -EOPNOTSUPP;
8036                 }
8037
8038                 flowtable_hook->priority        = flowtable->data.priority;
8039                 flowtable_hook->num             = flowtable->hooknum;
8040         }
8041
8042         if (tb[NFTA_FLOWTABLE_HOOK_DEVS]) {
8043                 err = nf_tables_parse_netdev_hooks(ctx->net,
8044                                                    tb[NFTA_FLOWTABLE_HOOK_DEVS],
8045                                                    &flowtable_hook->list,
8046                                                    extack);
8047                 if (err < 0)
8048                         return err;
8049         }
8050
8051         list_for_each_entry(hook, &flowtable_hook->list, list) {
8052                 hook->ops.pf            = NFPROTO_NETDEV;
8053                 hook->ops.hooknum       = flowtable_hook->num;
8054                 hook->ops.priority      = flowtable_hook->priority;
8055                 hook->ops.priv          = &flowtable->data;
8056                 hook->ops.hook          = flowtable->data.type->hook;
8057         }
8058
8059         return err;
8060 }
8061
8062 static const struct nf_flowtable_type *__nft_flowtable_type_get(u8 family)
8063 {
8064         const struct nf_flowtable_type *type;
8065
8066         list_for_each_entry(type, &nf_tables_flowtables, list) {
8067                 if (family == type->family)
8068                         return type;
8069         }
8070         return NULL;
8071 }
8072
8073 static const struct nf_flowtable_type *
8074 nft_flowtable_type_get(struct net *net, u8 family)
8075 {
8076         const struct nf_flowtable_type *type;
8077
8078         type = __nft_flowtable_type_get(family);
8079         if (type != NULL && try_module_get(type->owner))
8080                 return type;
8081
8082         lockdep_nfnl_nft_mutex_not_held();
8083 #ifdef CONFIG_MODULES
8084         if (type == NULL) {
8085                 if (nft_request_module(net, "nf-flowtable-%u", family) == -EAGAIN)
8086                         return ERR_PTR(-EAGAIN);
8087         }
8088 #endif
8089         return ERR_PTR(-ENOENT);
8090 }
8091
8092 /* Only called from error and netdev event paths. */
8093 static void nft_unregister_flowtable_hook(struct net *net,
8094                                           struct nft_flowtable *flowtable,
8095                                           struct nft_hook *hook)
8096 {
8097         nf_unregister_net_hook(net, &hook->ops);
8098         flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
8099                                     FLOW_BLOCK_UNBIND);
8100 }
8101
8102 static void __nft_unregister_flowtable_net_hooks(struct net *net,
8103                                                  struct list_head *hook_list,
8104                                                  bool release_netdev)
8105 {
8106         struct nft_hook *hook, *next;
8107
8108         list_for_each_entry_safe(hook, next, hook_list, list) {
8109                 nf_unregister_net_hook(net, &hook->ops);
8110                 if (release_netdev) {
8111                         list_del(&hook->list);
8112                         kfree_rcu(hook, rcu);
8113                 }
8114         }
8115 }
8116
8117 static void nft_unregister_flowtable_net_hooks(struct net *net,
8118                                                struct list_head *hook_list)
8119 {
8120         __nft_unregister_flowtable_net_hooks(net, hook_list, false);
8121 }
8122
8123 static int nft_register_flowtable_net_hooks(struct net *net,
8124                                             struct nft_table *table,
8125                                             struct list_head *hook_list,
8126                                             struct nft_flowtable *flowtable)
8127 {
8128         struct nft_hook *hook, *hook2, *next;
8129         struct nft_flowtable *ft;
8130         int err, i = 0;
8131
8132         list_for_each_entry(hook, hook_list, list) {
8133                 list_for_each_entry(ft, &table->flowtables, list) {
8134                         if (!nft_is_active_next(net, ft))
8135                                 continue;
8136
8137                         list_for_each_entry(hook2, &ft->hook_list, list) {
8138                                 if (hook->ops.dev == hook2->ops.dev &&
8139                                     hook->ops.pf == hook2->ops.pf) {
8140                                         err = -EEXIST;
8141                                         goto err_unregister_net_hooks;
8142                                 }
8143                         }
8144                 }
8145
8146                 err = flowtable->data.type->setup(&flowtable->data,
8147                                                   hook->ops.dev,
8148                                                   FLOW_BLOCK_BIND);
8149                 if (err < 0)
8150                         goto err_unregister_net_hooks;
8151
8152                 err = nf_register_net_hook(net, &hook->ops);
8153                 if (err < 0) {
8154                         flowtable->data.type->setup(&flowtable->data,
8155                                                     hook->ops.dev,
8156                                                     FLOW_BLOCK_UNBIND);
8157                         goto err_unregister_net_hooks;
8158                 }
8159
8160                 i++;
8161         }
8162
8163         return 0;
8164
8165 err_unregister_net_hooks:
8166         list_for_each_entry_safe(hook, next, hook_list, list) {
8167                 if (i-- <= 0)
8168                         break;
8169
8170                 nft_unregister_flowtable_hook(net, flowtable, hook);
8171                 list_del_rcu(&hook->list);
8172                 kfree_rcu(hook, rcu);
8173         }
8174
8175         return err;
8176 }
8177
8178 static void nft_hooks_destroy(struct list_head *hook_list)
8179 {
8180         struct nft_hook *hook, *next;
8181
8182         list_for_each_entry_safe(hook, next, hook_list, list) {
8183                 list_del_rcu(&hook->list);
8184                 kfree_rcu(hook, rcu);
8185         }
8186 }
8187
8188 static int nft_flowtable_update(struct nft_ctx *ctx, const struct nlmsghdr *nlh,
8189                                 struct nft_flowtable *flowtable,
8190                                 struct netlink_ext_ack *extack)
8191 {
8192         const struct nlattr * const *nla = ctx->nla;
8193         struct nft_flowtable_hook flowtable_hook;
8194         struct nft_hook *hook, *next;
8195         struct nft_trans *trans;
8196         bool unregister = false;
8197         u32 flags;
8198         int err;
8199
8200         err = nft_flowtable_parse_hook(ctx, nla, &flowtable_hook, flowtable,
8201                                        extack, false);
8202         if (err < 0)
8203                 return err;
8204
8205         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
8206                 if (nft_hook_list_find(&flowtable->hook_list, hook)) {
8207                         list_del(&hook->list);
8208                         kfree(hook);
8209                 }
8210         }
8211
8212         if (nla[NFTA_FLOWTABLE_FLAGS]) {
8213                 flags = ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
8214                 if (flags & ~NFT_FLOWTABLE_MASK) {
8215                         err = -EOPNOTSUPP;
8216                         goto err_flowtable_update_hook;
8217                 }
8218                 if ((flowtable->data.flags & NFT_FLOWTABLE_HW_OFFLOAD) ^
8219                     (flags & NFT_FLOWTABLE_HW_OFFLOAD)) {
8220                         err = -EOPNOTSUPP;
8221                         goto err_flowtable_update_hook;
8222                 }
8223         } else {
8224                 flags = flowtable->data.flags;
8225         }
8226
8227         err = nft_register_flowtable_net_hooks(ctx->net, ctx->table,
8228                                                &flowtable_hook.list, flowtable);
8229         if (err < 0)
8230                 goto err_flowtable_update_hook;
8231
8232         trans = nft_trans_alloc(ctx, NFT_MSG_NEWFLOWTABLE,
8233                                 sizeof(struct nft_trans_flowtable));
8234         if (!trans) {
8235                 unregister = true;
8236                 err = -ENOMEM;
8237                 goto err_flowtable_update_hook;
8238         }
8239
8240         nft_trans_flowtable_flags(trans) = flags;
8241         nft_trans_flowtable(trans) = flowtable;
8242         nft_trans_flowtable_update(trans) = true;
8243         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
8244         list_splice(&flowtable_hook.list, &nft_trans_flowtable_hooks(trans));
8245
8246         nft_trans_commit_list_add_tail(ctx->net, trans);
8247
8248         return 0;
8249
8250 err_flowtable_update_hook:
8251         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
8252                 if (unregister)
8253                         nft_unregister_flowtable_hook(ctx->net, flowtable, hook);
8254                 list_del_rcu(&hook->list);
8255                 kfree_rcu(hook, rcu);
8256         }
8257
8258         return err;
8259
8260 }
8261
8262 static int nf_tables_newflowtable(struct sk_buff *skb,
8263                                   const struct nfnl_info *info,
8264                                   const struct nlattr * const nla[])
8265 {
8266         struct netlink_ext_ack *extack = info->extack;
8267         struct nft_flowtable_hook flowtable_hook;
8268         u8 genmask = nft_genmask_next(info->net);
8269         u8 family = info->nfmsg->nfgen_family;
8270         const struct nf_flowtable_type *type;
8271         struct nft_flowtable *flowtable;
8272         struct nft_hook *hook, *next;
8273         struct net *net = info->net;
8274         struct nft_table *table;
8275         struct nft_ctx ctx;
8276         int err;
8277
8278         if (!nla[NFTA_FLOWTABLE_TABLE] ||
8279             !nla[NFTA_FLOWTABLE_NAME] ||
8280             !nla[NFTA_FLOWTABLE_HOOK])
8281                 return -EINVAL;
8282
8283         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8284                                  genmask, NETLINK_CB(skb).portid);
8285         if (IS_ERR(table)) {
8286                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
8287                 return PTR_ERR(table);
8288         }
8289
8290         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
8291                                          genmask);
8292         if (IS_ERR(flowtable)) {
8293                 err = PTR_ERR(flowtable);
8294                 if (err != -ENOENT) {
8295                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8296                         return err;
8297                 }
8298         } else {
8299                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
8300                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8301                         return -EEXIST;
8302                 }
8303
8304                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8305
8306                 return nft_flowtable_update(&ctx, info->nlh, flowtable, extack);
8307         }
8308
8309         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8310
8311         if (!nft_use_inc(&table->use))
8312                 return -EMFILE;
8313
8314         flowtable = kzalloc(sizeof(*flowtable), GFP_KERNEL_ACCOUNT);
8315         if (!flowtable) {
8316                 err = -ENOMEM;
8317                 goto flowtable_alloc;
8318         }
8319
8320         flowtable->table = table;
8321         flowtable->handle = nf_tables_alloc_handle(table);
8322         INIT_LIST_HEAD(&flowtable->hook_list);
8323
8324         flowtable->name = nla_strdup(nla[NFTA_FLOWTABLE_NAME], GFP_KERNEL_ACCOUNT);
8325         if (!flowtable->name) {
8326                 err = -ENOMEM;
8327                 goto err1;
8328         }
8329
8330         type = nft_flowtable_type_get(net, family);
8331         if (IS_ERR(type)) {
8332                 err = PTR_ERR(type);
8333                 goto err2;
8334         }
8335
8336         if (nla[NFTA_FLOWTABLE_FLAGS]) {
8337                 flowtable->data.flags =
8338                         ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
8339                 if (flowtable->data.flags & ~NFT_FLOWTABLE_MASK) {
8340                         err = -EOPNOTSUPP;
8341                         goto err3;
8342                 }
8343         }
8344
8345         write_pnet(&flowtable->data.net, net);
8346         flowtable->data.type = type;
8347         err = type->init(&flowtable->data);
8348         if (err < 0)
8349                 goto err3;
8350
8351         err = nft_flowtable_parse_hook(&ctx, nla, &flowtable_hook, flowtable,
8352                                        extack, true);
8353         if (err < 0)
8354                 goto err4;
8355
8356         list_splice(&flowtable_hook.list, &flowtable->hook_list);
8357         flowtable->data.priority = flowtable_hook.priority;
8358         flowtable->hooknum = flowtable_hook.num;
8359
8360         err = nft_register_flowtable_net_hooks(ctx.net, table,
8361                                                &flowtable->hook_list,
8362                                                flowtable);
8363         if (err < 0) {
8364                 nft_hooks_destroy(&flowtable->hook_list);
8365                 goto err4;
8366         }
8367
8368         err = nft_trans_flowtable_add(&ctx, NFT_MSG_NEWFLOWTABLE, flowtable);
8369         if (err < 0)
8370                 goto err5;
8371
8372         list_add_tail_rcu(&flowtable->list, &table->flowtables);
8373
8374         return 0;
8375 err5:
8376         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
8377                 nft_unregister_flowtable_hook(net, flowtable, hook);
8378                 list_del_rcu(&hook->list);
8379                 kfree_rcu(hook, rcu);
8380         }
8381 err4:
8382         flowtable->data.type->free(&flowtable->data);
8383 err3:
8384         module_put(type->owner);
8385 err2:
8386         kfree(flowtable->name);
8387 err1:
8388         kfree(flowtable);
8389 flowtable_alloc:
8390         nft_use_dec_restore(&table->use);
8391
8392         return err;
8393 }
8394
8395 static void nft_flowtable_hook_release(struct nft_flowtable_hook *flowtable_hook)
8396 {
8397         struct nft_hook *this, *next;
8398
8399         list_for_each_entry_safe(this, next, &flowtable_hook->list, list) {
8400                 list_del(&this->list);
8401                 kfree(this);
8402         }
8403 }
8404
8405 static int nft_delflowtable_hook(struct nft_ctx *ctx,
8406                                  struct nft_flowtable *flowtable,
8407                                  struct netlink_ext_ack *extack)
8408 {
8409         const struct nlattr * const *nla = ctx->nla;
8410         struct nft_flowtable_hook flowtable_hook;
8411         LIST_HEAD(flowtable_del_list);
8412         struct nft_hook *this, *hook;
8413         struct nft_trans *trans;
8414         int err;
8415
8416         err = nft_flowtable_parse_hook(ctx, nla, &flowtable_hook, flowtable,
8417                                        extack, false);
8418         if (err < 0)
8419                 return err;
8420
8421         list_for_each_entry(this, &flowtable_hook.list, list) {
8422                 hook = nft_hook_list_find(&flowtable->hook_list, this);
8423                 if (!hook) {
8424                         err = -ENOENT;
8425                         goto err_flowtable_del_hook;
8426                 }
8427                 list_move(&hook->list, &flowtable_del_list);
8428         }
8429
8430         trans = nft_trans_alloc(ctx, NFT_MSG_DELFLOWTABLE,
8431                                 sizeof(struct nft_trans_flowtable));
8432         if (!trans) {
8433                 err = -ENOMEM;
8434                 goto err_flowtable_del_hook;
8435         }
8436
8437         nft_trans_flowtable(trans) = flowtable;
8438         nft_trans_flowtable_update(trans) = true;
8439         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
8440         list_splice(&flowtable_del_list, &nft_trans_flowtable_hooks(trans));
8441         nft_flowtable_hook_release(&flowtable_hook);
8442
8443         nft_trans_commit_list_add_tail(ctx->net, trans);
8444
8445         return 0;
8446
8447 err_flowtable_del_hook:
8448         list_splice(&flowtable_del_list, &flowtable->hook_list);
8449         nft_flowtable_hook_release(&flowtable_hook);
8450
8451         return err;
8452 }
8453
8454 static int nf_tables_delflowtable(struct sk_buff *skb,
8455                                   const struct nfnl_info *info,
8456                                   const struct nlattr * const nla[])
8457 {
8458         struct netlink_ext_ack *extack = info->extack;
8459         u8 genmask = nft_genmask_next(info->net);
8460         u8 family = info->nfmsg->nfgen_family;
8461         struct nft_flowtable *flowtable;
8462         struct net *net = info->net;
8463         const struct nlattr *attr;
8464         struct nft_table *table;
8465         struct nft_ctx ctx;
8466
8467         if (!nla[NFTA_FLOWTABLE_TABLE] ||
8468             (!nla[NFTA_FLOWTABLE_NAME] &&
8469              !nla[NFTA_FLOWTABLE_HANDLE]))
8470                 return -EINVAL;
8471
8472         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8473                                  genmask, NETLINK_CB(skb).portid);
8474         if (IS_ERR(table)) {
8475                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
8476                 return PTR_ERR(table);
8477         }
8478
8479         if (nla[NFTA_FLOWTABLE_HANDLE]) {
8480                 attr = nla[NFTA_FLOWTABLE_HANDLE];
8481                 flowtable = nft_flowtable_lookup_byhandle(table, attr, genmask);
8482         } else {
8483                 attr = nla[NFTA_FLOWTABLE_NAME];
8484                 flowtable = nft_flowtable_lookup(table, attr, genmask);
8485         }
8486
8487         if (IS_ERR(flowtable)) {
8488                 if (PTR_ERR(flowtable) == -ENOENT &&
8489                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYFLOWTABLE)
8490                         return 0;
8491
8492                 NL_SET_BAD_ATTR(extack, attr);
8493                 return PTR_ERR(flowtable);
8494         }
8495
8496         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8497
8498         if (nla[NFTA_FLOWTABLE_HOOK])
8499                 return nft_delflowtable_hook(&ctx, flowtable, extack);
8500
8501         if (flowtable->use > 0) {
8502                 NL_SET_BAD_ATTR(extack, attr);
8503                 return -EBUSY;
8504         }
8505
8506         return nft_delflowtable(&ctx, flowtable);
8507 }
8508
8509 static int nf_tables_fill_flowtable_info(struct sk_buff *skb, struct net *net,
8510                                          u32 portid, u32 seq, int event,
8511                                          u32 flags, int family,
8512                                          struct nft_flowtable *flowtable,
8513                                          struct list_head *hook_list)
8514 {
8515         struct nlattr *nest, *nest_devs;
8516         struct nft_hook *hook;
8517         struct nlmsghdr *nlh;
8518
8519         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
8520         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
8521                            NFNETLINK_V0, nft_base_seq(net));
8522         if (!nlh)
8523                 goto nla_put_failure;
8524
8525         if (nla_put_string(skb, NFTA_FLOWTABLE_TABLE, flowtable->table->name) ||
8526             nla_put_string(skb, NFTA_FLOWTABLE_NAME, flowtable->name) ||
8527             nla_put_be64(skb, NFTA_FLOWTABLE_HANDLE, cpu_to_be64(flowtable->handle),
8528                          NFTA_FLOWTABLE_PAD))
8529                 goto nla_put_failure;
8530
8531         if (event == NFT_MSG_DELFLOWTABLE && !hook_list) {
8532                 nlmsg_end(skb, nlh);
8533                 return 0;
8534         }
8535
8536         if (nla_put_be32(skb, NFTA_FLOWTABLE_USE, htonl(flowtable->use)) ||
8537             nla_put_be32(skb, NFTA_FLOWTABLE_FLAGS, htonl(flowtable->data.flags)))
8538                 goto nla_put_failure;
8539
8540         nest = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK);
8541         if (!nest)
8542                 goto nla_put_failure;
8543         if (nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_NUM, htonl(flowtable->hooknum)) ||
8544             nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_PRIORITY, htonl(flowtable->data.priority)))
8545                 goto nla_put_failure;
8546
8547         nest_devs = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK_DEVS);
8548         if (!nest_devs)
8549                 goto nla_put_failure;
8550
8551         if (!hook_list)
8552                 hook_list = &flowtable->hook_list;
8553
8554         list_for_each_entry_rcu(hook, hook_list, list) {
8555                 if (nla_put_string(skb, NFTA_DEVICE_NAME, hook->ops.dev->name))
8556                         goto nla_put_failure;
8557         }
8558         nla_nest_end(skb, nest_devs);
8559         nla_nest_end(skb, nest);
8560
8561         nlmsg_end(skb, nlh);
8562         return 0;
8563
8564 nla_put_failure:
8565         nlmsg_trim(skb, nlh);
8566         return -1;
8567 }
8568
8569 struct nft_flowtable_filter {
8570         char            *table;
8571 };
8572
8573 static int nf_tables_dump_flowtable(struct sk_buff *skb,
8574                                     struct netlink_callback *cb)
8575 {
8576         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
8577         struct nft_flowtable_filter *filter = cb->data;
8578         unsigned int idx = 0, s_idx = cb->args[0];
8579         struct net *net = sock_net(skb->sk);
8580         int family = nfmsg->nfgen_family;
8581         struct nft_flowtable *flowtable;
8582         struct nftables_pernet *nft_net;
8583         const struct nft_table *table;
8584
8585         rcu_read_lock();
8586         nft_net = nft_pernet(net);
8587         cb->seq = READ_ONCE(nft_net->base_seq);
8588
8589         list_for_each_entry_rcu(table, &nft_net->tables, list) {
8590                 if (family != NFPROTO_UNSPEC && family != table->family)
8591                         continue;
8592
8593                 list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
8594                         if (!nft_is_active(net, flowtable))
8595                                 goto cont;
8596                         if (idx < s_idx)
8597                                 goto cont;
8598                         if (idx > s_idx)
8599                                 memset(&cb->args[1], 0,
8600                                        sizeof(cb->args) - sizeof(cb->args[0]));
8601                         if (filter && filter->table &&
8602                             strcmp(filter->table, table->name))
8603                                 goto cont;
8604
8605                         if (nf_tables_fill_flowtable_info(skb, net, NETLINK_CB(cb->skb).portid,
8606                                                           cb->nlh->nlmsg_seq,
8607                                                           NFT_MSG_NEWFLOWTABLE,
8608                                                           NLM_F_MULTI | NLM_F_APPEND,
8609                                                           table->family,
8610                                                           flowtable, NULL) < 0)
8611                                 goto done;
8612
8613                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
8614 cont:
8615                         idx++;
8616                 }
8617         }
8618 done:
8619         rcu_read_unlock();
8620
8621         cb->args[0] = idx;
8622         return skb->len;
8623 }
8624
8625 static int nf_tables_dump_flowtable_start(struct netlink_callback *cb)
8626 {
8627         const struct nlattr * const *nla = cb->data;
8628         struct nft_flowtable_filter *filter = NULL;
8629
8630         if (nla[NFTA_FLOWTABLE_TABLE]) {
8631                 filter = kzalloc(sizeof(*filter), GFP_ATOMIC);
8632                 if (!filter)
8633                         return -ENOMEM;
8634
8635                 filter->table = nla_strdup(nla[NFTA_FLOWTABLE_TABLE],
8636                                            GFP_ATOMIC);
8637                 if (!filter->table) {
8638                         kfree(filter);
8639                         return -ENOMEM;
8640                 }
8641         }
8642
8643         cb->data = filter;
8644         return 0;
8645 }
8646
8647 static int nf_tables_dump_flowtable_done(struct netlink_callback *cb)
8648 {
8649         struct nft_flowtable_filter *filter = cb->data;
8650
8651         if (!filter)
8652                 return 0;
8653
8654         kfree(filter->table);
8655         kfree(filter);
8656
8657         return 0;
8658 }
8659
8660 /* called with rcu_read_lock held */
8661 static int nf_tables_getflowtable(struct sk_buff *skb,
8662                                   const struct nfnl_info *info,
8663                                   const struct nlattr * const nla[])
8664 {
8665         u8 genmask = nft_genmask_cur(info->net);
8666         u8 family = info->nfmsg->nfgen_family;
8667         struct nft_flowtable *flowtable;
8668         const struct nft_table *table;
8669         struct net *net = info->net;
8670         struct sk_buff *skb2;
8671         int err;
8672
8673         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
8674                 struct netlink_dump_control c = {
8675                         .start = nf_tables_dump_flowtable_start,
8676                         .dump = nf_tables_dump_flowtable,
8677                         .done = nf_tables_dump_flowtable_done,
8678                         .module = THIS_MODULE,
8679                         .data = (void *)nla,
8680                 };
8681
8682                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
8683         }
8684
8685         if (!nla[NFTA_FLOWTABLE_NAME])
8686                 return -EINVAL;
8687
8688         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8689                                  genmask, 0);
8690         if (IS_ERR(table))
8691                 return PTR_ERR(table);
8692
8693         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
8694                                          genmask);
8695         if (IS_ERR(flowtable))
8696                 return PTR_ERR(flowtable);
8697
8698         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
8699         if (!skb2)
8700                 return -ENOMEM;
8701
8702         err = nf_tables_fill_flowtable_info(skb2, net, NETLINK_CB(skb).portid,
8703                                             info->nlh->nlmsg_seq,
8704                                             NFT_MSG_NEWFLOWTABLE, 0, family,
8705                                             flowtable, NULL);
8706         if (err < 0)
8707                 goto err_fill_flowtable_info;
8708
8709         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
8710
8711 err_fill_flowtable_info:
8712         kfree_skb(skb2);
8713         return err;
8714 }
8715
8716 static void nf_tables_flowtable_notify(struct nft_ctx *ctx,
8717                                        struct nft_flowtable *flowtable,
8718                                        struct list_head *hook_list, int event)
8719 {
8720         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
8721         struct sk_buff *skb;
8722         u16 flags = 0;
8723         int err;
8724
8725         if (!ctx->report &&
8726             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
8727                 return;
8728
8729         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
8730         if (skb == NULL)
8731                 goto err;
8732
8733         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
8734                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
8735
8736         err = nf_tables_fill_flowtable_info(skb, ctx->net, ctx->portid,
8737                                             ctx->seq, event, flags,
8738                                             ctx->family, flowtable, hook_list);
8739         if (err < 0) {
8740                 kfree_skb(skb);
8741                 goto err;
8742         }
8743
8744         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
8745         return;
8746 err:
8747         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
8748 }
8749
8750 static void nf_tables_flowtable_destroy(struct nft_flowtable *flowtable)
8751 {
8752         struct nft_hook *hook, *next;
8753
8754         flowtable->data.type->free(&flowtable->data);
8755         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
8756                 flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
8757                                             FLOW_BLOCK_UNBIND);
8758                 list_del_rcu(&hook->list);
8759                 kfree(hook);
8760         }
8761         kfree(flowtable->name);
8762         module_put(flowtable->data.type->owner);
8763         kfree(flowtable);
8764 }
8765
8766 static int nf_tables_fill_gen_info(struct sk_buff *skb, struct net *net,
8767                                    u32 portid, u32 seq)
8768 {
8769         struct nftables_pernet *nft_net = nft_pernet(net);
8770         struct nlmsghdr *nlh;
8771         char buf[TASK_COMM_LEN];
8772         int event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWGEN);
8773
8774         nlh = nfnl_msg_put(skb, portid, seq, event, 0, AF_UNSPEC,
8775                            NFNETLINK_V0, nft_base_seq(net));
8776         if (!nlh)
8777                 goto nla_put_failure;
8778
8779         if (nla_put_be32(skb, NFTA_GEN_ID, htonl(nft_net->base_seq)) ||
8780             nla_put_be32(skb, NFTA_GEN_PROC_PID, htonl(task_pid_nr(current))) ||
8781             nla_put_string(skb, NFTA_GEN_PROC_NAME, get_task_comm(buf, current)))
8782                 goto nla_put_failure;
8783
8784         nlmsg_end(skb, nlh);
8785         return 0;
8786
8787 nla_put_failure:
8788         nlmsg_trim(skb, nlh);
8789         return -EMSGSIZE;
8790 }
8791
8792 static void nft_flowtable_event(unsigned long event, struct net_device *dev,
8793                                 struct nft_flowtable *flowtable)
8794 {
8795         struct nft_hook *hook;
8796
8797         list_for_each_entry(hook, &flowtable->hook_list, list) {
8798                 if (hook->ops.dev != dev)
8799                         continue;
8800
8801                 /* flow_offload_netdev_event() cleans up entries for us. */
8802                 nft_unregister_flowtable_hook(dev_net(dev), flowtable, hook);
8803                 list_del_rcu(&hook->list);
8804                 kfree_rcu(hook, rcu);
8805                 break;
8806         }
8807 }
8808
8809 static int nf_tables_flowtable_event(struct notifier_block *this,
8810                                      unsigned long event, void *ptr)
8811 {
8812         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
8813         struct nft_flowtable *flowtable;
8814         struct nftables_pernet *nft_net;
8815         struct nft_table *table;
8816         struct net *net;
8817
8818         if (event != NETDEV_UNREGISTER)
8819                 return 0;
8820
8821         net = dev_net(dev);
8822         nft_net = nft_pernet(net);
8823         mutex_lock(&nft_net->commit_mutex);
8824         list_for_each_entry(table, &nft_net->tables, list) {
8825                 list_for_each_entry(flowtable, &table->flowtables, list) {
8826                         nft_flowtable_event(event, dev, flowtable);
8827                 }
8828         }
8829         mutex_unlock(&nft_net->commit_mutex);
8830
8831         return NOTIFY_DONE;
8832 }
8833
8834 static struct notifier_block nf_tables_flowtable_notifier = {
8835         .notifier_call  = nf_tables_flowtable_event,
8836 };
8837
8838 static void nf_tables_gen_notify(struct net *net, struct sk_buff *skb,
8839                                  int event)
8840 {
8841         struct nlmsghdr *nlh = nlmsg_hdr(skb);
8842         struct sk_buff *skb2;
8843         int err;
8844
8845         if (!nlmsg_report(nlh) &&
8846             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
8847                 return;
8848
8849         skb2 = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
8850         if (skb2 == NULL)
8851                 goto err;
8852
8853         err = nf_tables_fill_gen_info(skb2, net, NETLINK_CB(skb).portid,
8854                                       nlh->nlmsg_seq);
8855         if (err < 0) {
8856                 kfree_skb(skb2);
8857                 goto err;
8858         }
8859
8860         nfnetlink_send(skb2, net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
8861                        nlmsg_report(nlh), GFP_KERNEL);
8862         return;
8863 err:
8864         nfnetlink_set_err(net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
8865                           -ENOBUFS);
8866 }
8867
8868 static int nf_tables_getgen(struct sk_buff *skb, const struct nfnl_info *info,
8869                             const struct nlattr * const nla[])
8870 {
8871         struct sk_buff *skb2;
8872         int err;
8873
8874         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
8875         if (skb2 == NULL)
8876                 return -ENOMEM;
8877
8878         err = nf_tables_fill_gen_info(skb2, info->net, NETLINK_CB(skb).portid,
8879                                       info->nlh->nlmsg_seq);
8880         if (err < 0)
8881                 goto err_fill_gen_info;
8882
8883         return nfnetlink_unicast(skb2, info->net, NETLINK_CB(skb).portid);
8884
8885 err_fill_gen_info:
8886         kfree_skb(skb2);
8887         return err;
8888 }
8889
8890 static const struct nfnl_callback nf_tables_cb[NFT_MSG_MAX] = {
8891         [NFT_MSG_NEWTABLE] = {
8892                 .call           = nf_tables_newtable,
8893                 .type           = NFNL_CB_BATCH,
8894                 .attr_count     = NFTA_TABLE_MAX,
8895                 .policy         = nft_table_policy,
8896         },
8897         [NFT_MSG_GETTABLE] = {
8898                 .call           = nf_tables_gettable,
8899                 .type           = NFNL_CB_RCU,
8900                 .attr_count     = NFTA_TABLE_MAX,
8901                 .policy         = nft_table_policy,
8902         },
8903         [NFT_MSG_DELTABLE] = {
8904                 .call           = nf_tables_deltable,
8905                 .type           = NFNL_CB_BATCH,
8906                 .attr_count     = NFTA_TABLE_MAX,
8907                 .policy         = nft_table_policy,
8908         },
8909         [NFT_MSG_DESTROYTABLE] = {
8910                 .call           = nf_tables_deltable,
8911                 .type           = NFNL_CB_BATCH,
8912                 .attr_count     = NFTA_TABLE_MAX,
8913                 .policy         = nft_table_policy,
8914         },
8915         [NFT_MSG_NEWCHAIN] = {
8916                 .call           = nf_tables_newchain,
8917                 .type           = NFNL_CB_BATCH,
8918                 .attr_count     = NFTA_CHAIN_MAX,
8919                 .policy         = nft_chain_policy,
8920         },
8921         [NFT_MSG_GETCHAIN] = {
8922                 .call           = nf_tables_getchain,
8923                 .type           = NFNL_CB_RCU,
8924                 .attr_count     = NFTA_CHAIN_MAX,
8925                 .policy         = nft_chain_policy,
8926         },
8927         [NFT_MSG_DELCHAIN] = {
8928                 .call           = nf_tables_delchain,
8929                 .type           = NFNL_CB_BATCH,
8930                 .attr_count     = NFTA_CHAIN_MAX,
8931                 .policy         = nft_chain_policy,
8932         },
8933         [NFT_MSG_DESTROYCHAIN] = {
8934                 .call           = nf_tables_delchain,
8935                 .type           = NFNL_CB_BATCH,
8936                 .attr_count     = NFTA_CHAIN_MAX,
8937                 .policy         = nft_chain_policy,
8938         },
8939         [NFT_MSG_NEWRULE] = {
8940                 .call           = nf_tables_newrule,
8941                 .type           = NFNL_CB_BATCH,
8942                 .attr_count     = NFTA_RULE_MAX,
8943                 .policy         = nft_rule_policy,
8944         },
8945         [NFT_MSG_GETRULE] = {
8946                 .call           = nf_tables_getrule,
8947                 .type           = NFNL_CB_RCU,
8948                 .attr_count     = NFTA_RULE_MAX,
8949                 .policy         = nft_rule_policy,
8950         },
8951         [NFT_MSG_GETRULE_RESET] = {
8952                 .call           = nf_tables_getrule,
8953                 .type           = NFNL_CB_RCU,
8954                 .attr_count     = NFTA_RULE_MAX,
8955                 .policy         = nft_rule_policy,
8956         },
8957         [NFT_MSG_DELRULE] = {
8958                 .call           = nf_tables_delrule,
8959                 .type           = NFNL_CB_BATCH,
8960                 .attr_count     = NFTA_RULE_MAX,
8961                 .policy         = nft_rule_policy,
8962         },
8963         [NFT_MSG_DESTROYRULE] = {
8964                 .call           = nf_tables_delrule,
8965                 .type           = NFNL_CB_BATCH,
8966                 .attr_count     = NFTA_RULE_MAX,
8967                 .policy         = nft_rule_policy,
8968         },
8969         [NFT_MSG_NEWSET] = {
8970                 .call           = nf_tables_newset,
8971                 .type           = NFNL_CB_BATCH,
8972                 .attr_count     = NFTA_SET_MAX,
8973                 .policy         = nft_set_policy,
8974         },
8975         [NFT_MSG_GETSET] = {
8976                 .call           = nf_tables_getset,
8977                 .type           = NFNL_CB_RCU,
8978                 .attr_count     = NFTA_SET_MAX,
8979                 .policy         = nft_set_policy,
8980         },
8981         [NFT_MSG_DELSET] = {
8982                 .call           = nf_tables_delset,
8983                 .type           = NFNL_CB_BATCH,
8984                 .attr_count     = NFTA_SET_MAX,
8985                 .policy         = nft_set_policy,
8986         },
8987         [NFT_MSG_DESTROYSET] = {
8988                 .call           = nf_tables_delset,
8989                 .type           = NFNL_CB_BATCH,
8990                 .attr_count     = NFTA_SET_MAX,
8991                 .policy         = nft_set_policy,
8992         },
8993         [NFT_MSG_NEWSETELEM] = {
8994                 .call           = nf_tables_newsetelem,
8995                 .type           = NFNL_CB_BATCH,
8996                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
8997                 .policy         = nft_set_elem_list_policy,
8998         },
8999         [NFT_MSG_GETSETELEM] = {
9000                 .call           = nf_tables_getsetelem,
9001                 .type           = NFNL_CB_RCU,
9002                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9003                 .policy         = nft_set_elem_list_policy,
9004         },
9005         [NFT_MSG_GETSETELEM_RESET] = {
9006                 .call           = nf_tables_getsetelem,
9007                 .type           = NFNL_CB_RCU,
9008                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9009                 .policy         = nft_set_elem_list_policy,
9010         },
9011         [NFT_MSG_DELSETELEM] = {
9012                 .call           = nf_tables_delsetelem,
9013                 .type           = NFNL_CB_BATCH,
9014                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9015                 .policy         = nft_set_elem_list_policy,
9016         },
9017         [NFT_MSG_DESTROYSETELEM] = {
9018                 .call           = nf_tables_delsetelem,
9019                 .type           = NFNL_CB_BATCH,
9020                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9021                 .policy         = nft_set_elem_list_policy,
9022         },
9023         [NFT_MSG_GETGEN] = {
9024                 .call           = nf_tables_getgen,
9025                 .type           = NFNL_CB_RCU,
9026         },
9027         [NFT_MSG_NEWOBJ] = {
9028                 .call           = nf_tables_newobj,
9029                 .type           = NFNL_CB_BATCH,
9030                 .attr_count     = NFTA_OBJ_MAX,
9031                 .policy         = nft_obj_policy,
9032         },
9033         [NFT_MSG_GETOBJ] = {
9034                 .call           = nf_tables_getobj,
9035                 .type           = NFNL_CB_RCU,
9036                 .attr_count     = NFTA_OBJ_MAX,
9037                 .policy         = nft_obj_policy,
9038         },
9039         [NFT_MSG_DELOBJ] = {
9040                 .call           = nf_tables_delobj,
9041                 .type           = NFNL_CB_BATCH,
9042                 .attr_count     = NFTA_OBJ_MAX,
9043                 .policy         = nft_obj_policy,
9044         },
9045         [NFT_MSG_DESTROYOBJ] = {
9046                 .call           = nf_tables_delobj,
9047                 .type           = NFNL_CB_BATCH,
9048                 .attr_count     = NFTA_OBJ_MAX,
9049                 .policy         = nft_obj_policy,
9050         },
9051         [NFT_MSG_GETOBJ_RESET] = {
9052                 .call           = nf_tables_getobj,
9053                 .type           = NFNL_CB_RCU,
9054                 .attr_count     = NFTA_OBJ_MAX,
9055                 .policy         = nft_obj_policy,
9056         },
9057         [NFT_MSG_NEWFLOWTABLE] = {
9058                 .call           = nf_tables_newflowtable,
9059                 .type           = NFNL_CB_BATCH,
9060                 .attr_count     = NFTA_FLOWTABLE_MAX,
9061                 .policy         = nft_flowtable_policy,
9062         },
9063         [NFT_MSG_GETFLOWTABLE] = {
9064                 .call           = nf_tables_getflowtable,
9065                 .type           = NFNL_CB_RCU,
9066                 .attr_count     = NFTA_FLOWTABLE_MAX,
9067                 .policy         = nft_flowtable_policy,
9068         },
9069         [NFT_MSG_DELFLOWTABLE] = {
9070                 .call           = nf_tables_delflowtable,
9071                 .type           = NFNL_CB_BATCH,
9072                 .attr_count     = NFTA_FLOWTABLE_MAX,
9073                 .policy         = nft_flowtable_policy,
9074         },
9075         [NFT_MSG_DESTROYFLOWTABLE] = {
9076                 .call           = nf_tables_delflowtable,
9077                 .type           = NFNL_CB_BATCH,
9078                 .attr_count     = NFTA_FLOWTABLE_MAX,
9079                 .policy         = nft_flowtable_policy,
9080         },
9081 };
9082
9083 static int nf_tables_validate(struct net *net)
9084 {
9085         struct nftables_pernet *nft_net = nft_pernet(net);
9086         struct nft_table *table;
9087
9088         list_for_each_entry(table, &nft_net->tables, list) {
9089                 switch (table->validate_state) {
9090                 case NFT_VALIDATE_SKIP:
9091                         continue;
9092                 case NFT_VALIDATE_NEED:
9093                         nft_validate_state_update(table, NFT_VALIDATE_DO);
9094                         fallthrough;
9095                 case NFT_VALIDATE_DO:
9096                         if (nft_table_validate(net, table) < 0)
9097                                 return -EAGAIN;
9098
9099                         nft_validate_state_update(table, NFT_VALIDATE_SKIP);
9100                         break;
9101                 }
9102         }
9103
9104         return 0;
9105 }
9106
9107 /* a drop policy has to be deferred until all rules have been activated,
9108  * otherwise a large ruleset that contains a drop-policy base chain will
9109  * cause all packets to get dropped until the full transaction has been
9110  * processed.
9111  *
9112  * We defer the drop policy until the transaction has been finalized.
9113  */
9114 static void nft_chain_commit_drop_policy(struct nft_trans *trans)
9115 {
9116         struct nft_base_chain *basechain;
9117
9118         if (nft_trans_chain_policy(trans) != NF_DROP)
9119                 return;
9120
9121         if (!nft_is_base_chain(trans->ctx.chain))
9122                 return;
9123
9124         basechain = nft_base_chain(trans->ctx.chain);
9125         basechain->policy = NF_DROP;
9126 }
9127
9128 static void nft_chain_commit_update(struct nft_trans *trans)
9129 {
9130         struct nft_base_chain *basechain;
9131
9132         if (nft_trans_chain_name(trans)) {
9133                 rhltable_remove(&trans->ctx.table->chains_ht,
9134                                 &trans->ctx.chain->rhlhead,
9135                                 nft_chain_ht_params);
9136                 swap(trans->ctx.chain->name, nft_trans_chain_name(trans));
9137                 rhltable_insert_key(&trans->ctx.table->chains_ht,
9138                                     trans->ctx.chain->name,
9139                                     &trans->ctx.chain->rhlhead,
9140                                     nft_chain_ht_params);
9141         }
9142
9143         if (!nft_is_base_chain(trans->ctx.chain))
9144                 return;
9145
9146         nft_chain_stats_replace(trans);
9147
9148         basechain = nft_base_chain(trans->ctx.chain);
9149
9150         switch (nft_trans_chain_policy(trans)) {
9151         case NF_DROP:
9152         case NF_ACCEPT:
9153                 basechain->policy = nft_trans_chain_policy(trans);
9154                 break;
9155         }
9156 }
9157
9158 static void nft_obj_commit_update(struct nft_trans *trans)
9159 {
9160         struct nft_object *newobj;
9161         struct nft_object *obj;
9162
9163         obj = nft_trans_obj(trans);
9164         newobj = nft_trans_obj_newobj(trans);
9165
9166         if (obj->ops->update)
9167                 obj->ops->update(obj, newobj);
9168
9169         nft_obj_destroy(&trans->ctx, newobj);
9170 }
9171
9172 static void nft_commit_release(struct nft_trans *trans)
9173 {
9174         switch (trans->msg_type) {
9175         case NFT_MSG_DELTABLE:
9176         case NFT_MSG_DESTROYTABLE:
9177                 nf_tables_table_destroy(&trans->ctx);
9178                 break;
9179         case NFT_MSG_NEWCHAIN:
9180                 free_percpu(nft_trans_chain_stats(trans));
9181                 kfree(nft_trans_chain_name(trans));
9182                 break;
9183         case NFT_MSG_DELCHAIN:
9184         case NFT_MSG_DESTROYCHAIN:
9185                 if (nft_trans_chain_update(trans))
9186                         nft_hooks_destroy(&nft_trans_chain_hooks(trans));
9187                 else
9188                         nf_tables_chain_destroy(&trans->ctx);
9189                 break;
9190         case NFT_MSG_DELRULE:
9191         case NFT_MSG_DESTROYRULE:
9192                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
9193                 break;
9194         case NFT_MSG_DELSET:
9195         case NFT_MSG_DESTROYSET:
9196                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
9197                 break;
9198         case NFT_MSG_DELSETELEM:
9199         case NFT_MSG_DESTROYSETELEM:
9200                 nf_tables_set_elem_destroy(&trans->ctx,
9201                                            nft_trans_elem_set(trans),
9202                                            nft_trans_elem(trans).priv);
9203                 break;
9204         case NFT_MSG_DELOBJ:
9205         case NFT_MSG_DESTROYOBJ:
9206                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
9207                 break;
9208         case NFT_MSG_DELFLOWTABLE:
9209         case NFT_MSG_DESTROYFLOWTABLE:
9210                 if (nft_trans_flowtable_update(trans))
9211                         nft_hooks_destroy(&nft_trans_flowtable_hooks(trans));
9212                 else
9213                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
9214                 break;
9215         }
9216
9217         if (trans->put_net)
9218                 put_net(trans->ctx.net);
9219
9220         kfree(trans);
9221 }
9222
9223 static void nf_tables_trans_destroy_work(struct work_struct *w)
9224 {
9225         struct nft_trans *trans, *next;
9226         LIST_HEAD(head);
9227
9228         spin_lock(&nf_tables_destroy_list_lock);
9229         list_splice_init(&nf_tables_destroy_list, &head);
9230         spin_unlock(&nf_tables_destroy_list_lock);
9231
9232         if (list_empty(&head))
9233                 return;
9234
9235         synchronize_rcu();
9236
9237         list_for_each_entry_safe(trans, next, &head, list) {
9238                 nft_trans_list_del(trans);
9239                 nft_commit_release(trans);
9240         }
9241 }
9242
9243 void nf_tables_trans_destroy_flush_work(void)
9244 {
9245         flush_work(&trans_destroy_work);
9246 }
9247 EXPORT_SYMBOL_GPL(nf_tables_trans_destroy_flush_work);
9248
9249 static bool nft_expr_reduce(struct nft_regs_track *track,
9250                             const struct nft_expr *expr)
9251 {
9252         return false;
9253 }
9254
9255 static int nf_tables_commit_chain_prepare(struct net *net, struct nft_chain *chain)
9256 {
9257         const struct nft_expr *expr, *last;
9258         struct nft_regs_track track = {};
9259         unsigned int size, data_size;
9260         void *data, *data_boundary;
9261         struct nft_rule_dp *prule;
9262         struct nft_rule *rule;
9263
9264         /* already handled or inactive chain? */
9265         if (chain->blob_next || !nft_is_active_next(net, chain))
9266                 return 0;
9267
9268         data_size = 0;
9269         list_for_each_entry(rule, &chain->rules, list) {
9270                 if (nft_is_active_next(net, rule)) {
9271                         data_size += sizeof(*prule) + rule->dlen;
9272                         if (data_size > INT_MAX)
9273                                 return -ENOMEM;
9274                 }
9275         }
9276
9277         chain->blob_next = nf_tables_chain_alloc_rules(chain, data_size);
9278         if (!chain->blob_next)
9279                 return -ENOMEM;
9280
9281         data = (void *)chain->blob_next->data;
9282         data_boundary = data + data_size;
9283         size = 0;
9284
9285         list_for_each_entry(rule, &chain->rules, list) {
9286                 if (!nft_is_active_next(net, rule))
9287                         continue;
9288
9289                 prule = (struct nft_rule_dp *)data;
9290                 data += offsetof(struct nft_rule_dp, data);
9291                 if (WARN_ON_ONCE(data > data_boundary))
9292                         return -ENOMEM;
9293
9294                 size = 0;
9295                 track.last = nft_expr_last(rule);
9296                 nft_rule_for_each_expr(expr, last, rule) {
9297                         track.cur = expr;
9298
9299                         if (nft_expr_reduce(&track, expr)) {
9300                                 expr = track.cur;
9301                                 continue;
9302                         }
9303
9304                         if (WARN_ON_ONCE(data + size + expr->ops->size > data_boundary))
9305                                 return -ENOMEM;
9306
9307                         memcpy(data + size, expr, expr->ops->size);
9308                         size += expr->ops->size;
9309                 }
9310                 if (WARN_ON_ONCE(size >= 1 << 12))
9311                         return -ENOMEM;
9312
9313                 prule->handle = rule->handle;
9314                 prule->dlen = size;
9315                 prule->is_last = 0;
9316
9317                 data += size;
9318                 size = 0;
9319                 chain->blob_next->size += (unsigned long)(data - (void *)prule);
9320         }
9321
9322         if (WARN_ON_ONCE(data > data_boundary))
9323                 return -ENOMEM;
9324
9325         prule = (struct nft_rule_dp *)data;
9326         nft_last_rule(chain, prule);
9327
9328         return 0;
9329 }
9330
9331 static void nf_tables_commit_chain_prepare_cancel(struct net *net)
9332 {
9333         struct nftables_pernet *nft_net = nft_pernet(net);
9334         struct nft_trans *trans, *next;
9335
9336         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
9337                 struct nft_chain *chain = trans->ctx.chain;
9338
9339                 if (trans->msg_type == NFT_MSG_NEWRULE ||
9340                     trans->msg_type == NFT_MSG_DELRULE) {
9341                         kvfree(chain->blob_next);
9342                         chain->blob_next = NULL;
9343                 }
9344         }
9345 }
9346
9347 static void __nf_tables_commit_chain_free_rules(struct rcu_head *h)
9348 {
9349         struct nft_rule_dp_last *l = container_of(h, struct nft_rule_dp_last, h);
9350
9351         kvfree(l->blob);
9352 }
9353
9354 static void nf_tables_commit_chain_free_rules_old(struct nft_rule_blob *blob)
9355 {
9356         struct nft_rule_dp_last *last;
9357
9358         /* last rule trailer is after end marker */
9359         last = (void *)blob + sizeof(*blob) + blob->size;
9360         last->blob = blob;
9361
9362         call_rcu(&last->h, __nf_tables_commit_chain_free_rules);
9363 }
9364
9365 static void nf_tables_commit_chain(struct net *net, struct nft_chain *chain)
9366 {
9367         struct nft_rule_blob *g0, *g1;
9368         bool next_genbit;
9369
9370         next_genbit = nft_gencursor_next(net);
9371
9372         g0 = rcu_dereference_protected(chain->blob_gen_0,
9373                                        lockdep_commit_lock_is_held(net));
9374         g1 = rcu_dereference_protected(chain->blob_gen_1,
9375                                        lockdep_commit_lock_is_held(net));
9376
9377         /* No changes to this chain? */
9378         if (chain->blob_next == NULL) {
9379                 /* chain had no change in last or next generation */
9380                 if (g0 == g1)
9381                         return;
9382                 /*
9383                  * chain had no change in this generation; make sure next
9384                  * one uses same rules as current generation.
9385                  */
9386                 if (next_genbit) {
9387                         rcu_assign_pointer(chain->blob_gen_1, g0);
9388                         nf_tables_commit_chain_free_rules_old(g1);
9389                 } else {
9390                         rcu_assign_pointer(chain->blob_gen_0, g1);
9391                         nf_tables_commit_chain_free_rules_old(g0);
9392                 }
9393
9394                 return;
9395         }
9396
9397         if (next_genbit)
9398                 rcu_assign_pointer(chain->blob_gen_1, chain->blob_next);
9399         else
9400                 rcu_assign_pointer(chain->blob_gen_0, chain->blob_next);
9401
9402         chain->blob_next = NULL;
9403
9404         if (g0 == g1)
9405                 return;
9406
9407         if (next_genbit)
9408                 nf_tables_commit_chain_free_rules_old(g1);
9409         else
9410                 nf_tables_commit_chain_free_rules_old(g0);
9411 }
9412
9413 static void nft_obj_del(struct nft_object *obj)
9414 {
9415         rhltable_remove(&nft_objname_ht, &obj->rhlhead, nft_objname_ht_params);
9416         list_del_rcu(&obj->list);
9417 }
9418
9419 void nft_chain_del(struct nft_chain *chain)
9420 {
9421         struct nft_table *table = chain->table;
9422
9423         WARN_ON_ONCE(rhltable_remove(&table->chains_ht, &chain->rhlhead,
9424                                      nft_chain_ht_params));
9425         list_del_rcu(&chain->list);
9426 }
9427
9428 static void nft_trans_gc_setelem_remove(struct nft_ctx *ctx,
9429                                         struct nft_trans_gc *trans)
9430 {
9431         void **priv = trans->priv;
9432         unsigned int i;
9433
9434         for (i = 0; i < trans->count; i++) {
9435                 struct nft_set_elem elem = {
9436                         .priv = priv[i],
9437                 };
9438
9439                 nft_setelem_data_deactivate(ctx->net, trans->set, &elem);
9440                 nft_setelem_remove(ctx->net, trans->set, &elem);
9441         }
9442 }
9443
9444 void nft_trans_gc_destroy(struct nft_trans_gc *trans)
9445 {
9446         nft_set_put(trans->set);
9447         put_net(trans->net);
9448         kfree(trans);
9449 }
9450
9451 static void nft_trans_gc_trans_free(struct rcu_head *rcu)
9452 {
9453         struct nft_set_elem elem = {};
9454         struct nft_trans_gc *trans;
9455         struct nft_ctx ctx = {};
9456         unsigned int i;
9457
9458         trans = container_of(rcu, struct nft_trans_gc, rcu);
9459         ctx.net = read_pnet(&trans->set->net);
9460
9461         for (i = 0; i < trans->count; i++) {
9462                 elem.priv = trans->priv[i];
9463                 if (!nft_setelem_is_catchall(trans->set, &elem))
9464                         atomic_dec(&trans->set->nelems);
9465
9466                 nf_tables_set_elem_destroy(&ctx, trans->set, elem.priv);
9467         }
9468
9469         nft_trans_gc_destroy(trans);
9470 }
9471
9472 static bool nft_trans_gc_work_done(struct nft_trans_gc *trans)
9473 {
9474         struct nftables_pernet *nft_net;
9475         struct nft_ctx ctx = {};
9476
9477         nft_net = nft_pernet(trans->net);
9478
9479         mutex_lock(&nft_net->commit_mutex);
9480
9481         /* Check for race with transaction, otherwise this batch refers to
9482          * stale objects that might not be there anymore. Skip transaction if
9483          * set has been destroyed from control plane transaction in case gc
9484          * worker loses race.
9485          */
9486         if (READ_ONCE(nft_net->gc_seq) != trans->seq || trans->set->dead) {
9487                 mutex_unlock(&nft_net->commit_mutex);
9488                 return false;
9489         }
9490
9491         ctx.net = trans->net;
9492         ctx.table = trans->set->table;
9493
9494         nft_trans_gc_setelem_remove(&ctx, trans);
9495         mutex_unlock(&nft_net->commit_mutex);
9496
9497         return true;
9498 }
9499
9500 static void nft_trans_gc_work(struct work_struct *work)
9501 {
9502         struct nft_trans_gc *trans, *next;
9503         LIST_HEAD(trans_gc_list);
9504
9505         spin_lock(&nf_tables_gc_list_lock);
9506         list_splice_init(&nf_tables_gc_list, &trans_gc_list);
9507         spin_unlock(&nf_tables_gc_list_lock);
9508
9509         list_for_each_entry_safe(trans, next, &trans_gc_list, list) {
9510                 list_del(&trans->list);
9511                 if (!nft_trans_gc_work_done(trans)) {
9512                         nft_trans_gc_destroy(trans);
9513                         continue;
9514                 }
9515                 call_rcu(&trans->rcu, nft_trans_gc_trans_free);
9516         }
9517 }
9518
9519 struct nft_trans_gc *nft_trans_gc_alloc(struct nft_set *set,
9520                                         unsigned int gc_seq, gfp_t gfp)
9521 {
9522         struct net *net = read_pnet(&set->net);
9523         struct nft_trans_gc *trans;
9524
9525         trans = kzalloc(sizeof(*trans), gfp);
9526         if (!trans)
9527                 return NULL;
9528
9529         trans->net = maybe_get_net(net);
9530         if (!trans->net) {
9531                 kfree(trans);
9532                 return NULL;
9533         }
9534
9535         refcount_inc(&set->refs);
9536         trans->set = set;
9537         trans->seq = gc_seq;
9538
9539         return trans;
9540 }
9541
9542 void nft_trans_gc_elem_add(struct nft_trans_gc *trans, void *priv)
9543 {
9544         trans->priv[trans->count++] = priv;
9545 }
9546
9547 static void nft_trans_gc_queue_work(struct nft_trans_gc *trans)
9548 {
9549         spin_lock(&nf_tables_gc_list_lock);
9550         list_add_tail(&trans->list, &nf_tables_gc_list);
9551         spin_unlock(&nf_tables_gc_list_lock);
9552
9553         schedule_work(&trans_gc_work);
9554 }
9555
9556 static int nft_trans_gc_space(struct nft_trans_gc *trans)
9557 {
9558         return NFT_TRANS_GC_BATCHCOUNT - trans->count;
9559 }
9560
9561 struct nft_trans_gc *nft_trans_gc_queue_async(struct nft_trans_gc *gc,
9562                                               unsigned int gc_seq, gfp_t gfp)
9563 {
9564         if (nft_trans_gc_space(gc))
9565                 return gc;
9566
9567         nft_trans_gc_queue_work(gc);
9568
9569         return nft_trans_gc_alloc(gc->set, gc_seq, gfp);
9570 }
9571
9572 void nft_trans_gc_queue_async_done(struct nft_trans_gc *trans)
9573 {
9574         if (trans->count == 0) {
9575                 nft_trans_gc_destroy(trans);
9576                 return;
9577         }
9578
9579         nft_trans_gc_queue_work(trans);
9580 }
9581
9582 struct nft_trans_gc *nft_trans_gc_queue_sync(struct nft_trans_gc *gc, gfp_t gfp)
9583 {
9584         if (WARN_ON_ONCE(!lockdep_commit_lock_is_held(gc->net)))
9585                 return NULL;
9586
9587         if (nft_trans_gc_space(gc))
9588                 return gc;
9589
9590         call_rcu(&gc->rcu, nft_trans_gc_trans_free);
9591
9592         return nft_trans_gc_alloc(gc->set, 0, gfp);
9593 }
9594
9595 void nft_trans_gc_queue_sync_done(struct nft_trans_gc *trans)
9596 {
9597         WARN_ON_ONCE(!lockdep_commit_lock_is_held(trans->net));
9598
9599         if (trans->count == 0) {
9600                 nft_trans_gc_destroy(trans);
9601                 return;
9602         }
9603
9604         call_rcu(&trans->rcu, nft_trans_gc_trans_free);
9605 }
9606
9607 struct nft_trans_gc *nft_trans_gc_catchall(struct nft_trans_gc *gc,
9608                                            unsigned int gc_seq)
9609 {
9610         struct nft_set_elem_catchall *catchall;
9611         const struct nft_set *set = gc->set;
9612         struct nft_set_ext *ext;
9613
9614         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
9615                 ext = nft_set_elem_ext(set, catchall->elem);
9616
9617                 if (!nft_set_elem_expired(ext))
9618                         continue;
9619                 if (nft_set_elem_is_dead(ext))
9620                         goto dead_elem;
9621
9622                 nft_set_elem_dead(ext);
9623 dead_elem:
9624                 gc = nft_trans_gc_queue_async(gc, gc_seq, GFP_ATOMIC);
9625                 if (!gc)
9626                         return NULL;
9627
9628                 nft_trans_gc_elem_add(gc, catchall->elem);
9629         }
9630
9631         return gc;
9632 }
9633
9634 static void nf_tables_module_autoload_cleanup(struct net *net)
9635 {
9636         struct nftables_pernet *nft_net = nft_pernet(net);
9637         struct nft_module_request *req, *next;
9638
9639         WARN_ON_ONCE(!list_empty(&nft_net->commit_list));
9640         list_for_each_entry_safe(req, next, &nft_net->module_list, list) {
9641                 WARN_ON_ONCE(!req->done);
9642                 list_del(&req->list);
9643                 kfree(req);
9644         }
9645 }
9646
9647 static void nf_tables_commit_release(struct net *net)
9648 {
9649         struct nftables_pernet *nft_net = nft_pernet(net);
9650         struct nft_trans *trans;
9651
9652         /* all side effects have to be made visible.
9653          * For example, if a chain named 'foo' has been deleted, a
9654          * new transaction must not find it anymore.
9655          *
9656          * Memory reclaim happens asynchronously from work queue
9657          * to prevent expensive synchronize_rcu() in commit phase.
9658          */
9659         if (list_empty(&nft_net->commit_list)) {
9660                 nf_tables_module_autoload_cleanup(net);
9661                 mutex_unlock(&nft_net->commit_mutex);
9662                 return;
9663         }
9664
9665         trans = list_last_entry(&nft_net->commit_list,
9666                                 struct nft_trans, list);
9667         get_net(trans->ctx.net);
9668         WARN_ON_ONCE(trans->put_net);
9669
9670         trans->put_net = true;
9671         spin_lock(&nf_tables_destroy_list_lock);
9672         list_splice_tail_init(&nft_net->commit_list, &nf_tables_destroy_list);
9673         spin_unlock(&nf_tables_destroy_list_lock);
9674
9675         nf_tables_module_autoload_cleanup(net);
9676         schedule_work(&trans_destroy_work);
9677
9678         mutex_unlock(&nft_net->commit_mutex);
9679 }
9680
9681 static void nft_commit_notify(struct net *net, u32 portid)
9682 {
9683         struct nftables_pernet *nft_net = nft_pernet(net);
9684         struct sk_buff *batch_skb = NULL, *nskb, *skb;
9685         unsigned char *data;
9686         int len;
9687
9688         list_for_each_entry_safe(skb, nskb, &nft_net->notify_list, list) {
9689                 if (!batch_skb) {
9690 new_batch:
9691                         batch_skb = skb;
9692                         len = NLMSG_GOODSIZE - skb->len;
9693                         list_del(&skb->list);
9694                         continue;
9695                 }
9696                 len -= skb->len;
9697                 if (len > 0 && NFT_CB(skb).report == NFT_CB(batch_skb).report) {
9698                         data = skb_put(batch_skb, skb->len);
9699                         memcpy(data, skb->data, skb->len);
9700                         list_del(&skb->list);
9701                         kfree_skb(skb);
9702                         continue;
9703                 }
9704                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
9705                                NFT_CB(batch_skb).report, GFP_KERNEL);
9706                 goto new_batch;
9707         }
9708
9709         if (batch_skb) {
9710                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
9711                                NFT_CB(batch_skb).report, GFP_KERNEL);
9712         }
9713
9714         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
9715 }
9716
9717 static int nf_tables_commit_audit_alloc(struct list_head *adl,
9718                                         struct nft_table *table)
9719 {
9720         struct nft_audit_data *adp;
9721
9722         list_for_each_entry(adp, adl, list) {
9723                 if (adp->table == table)
9724                         return 0;
9725         }
9726         adp = kzalloc(sizeof(*adp), GFP_KERNEL);
9727         if (!adp)
9728                 return -ENOMEM;
9729         adp->table = table;
9730         list_add(&adp->list, adl);
9731         return 0;
9732 }
9733
9734 static void nf_tables_commit_audit_free(struct list_head *adl)
9735 {
9736         struct nft_audit_data *adp, *adn;
9737
9738         list_for_each_entry_safe(adp, adn, adl, list) {
9739                 list_del(&adp->list);
9740                 kfree(adp);
9741         }
9742 }
9743
9744 static void nf_tables_commit_audit_collect(struct list_head *adl,
9745                                            struct nft_table *table, u32 op)
9746 {
9747         struct nft_audit_data *adp;
9748
9749         list_for_each_entry(adp, adl, list) {
9750                 if (adp->table == table)
9751                         goto found;
9752         }
9753         WARN_ONCE(1, "table=%s not expected in commit list", table->name);
9754         return;
9755 found:
9756         adp->entries++;
9757         if (!adp->op || adp->op > op)
9758                 adp->op = op;
9759 }
9760
9761 #define AUNFTABLENAMELEN (NFT_TABLE_MAXNAMELEN + 22)
9762
9763 static void nf_tables_commit_audit_log(struct list_head *adl, u32 generation)
9764 {
9765         struct nft_audit_data *adp, *adn;
9766         char aubuf[AUNFTABLENAMELEN];
9767
9768         list_for_each_entry_safe(adp, adn, adl, list) {
9769                 snprintf(aubuf, AUNFTABLENAMELEN, "%s:%u", adp->table->name,
9770                          generation);
9771                 audit_log_nfcfg(aubuf, adp->table->family, adp->entries,
9772                                 nft2audit_op[adp->op], GFP_KERNEL);
9773                 list_del(&adp->list);
9774                 kfree(adp);
9775         }
9776 }
9777
9778 static void nft_set_commit_update(struct list_head *set_update_list)
9779 {
9780         struct nft_set *set, *next;
9781
9782         list_for_each_entry_safe(set, next, set_update_list, pending_update) {
9783                 list_del_init(&set->pending_update);
9784
9785                 if (!set->ops->commit)
9786                         continue;
9787
9788                 set->ops->commit(set);
9789         }
9790 }
9791
9792 static unsigned int nft_gc_seq_begin(struct nftables_pernet *nft_net)
9793 {
9794         unsigned int gc_seq;
9795
9796         /* Bump gc counter, it becomes odd, this is the busy mark. */
9797         gc_seq = READ_ONCE(nft_net->gc_seq);
9798         WRITE_ONCE(nft_net->gc_seq, ++gc_seq);
9799
9800         return gc_seq;
9801 }
9802
9803 static void nft_gc_seq_end(struct nftables_pernet *nft_net, unsigned int gc_seq)
9804 {
9805         WRITE_ONCE(nft_net->gc_seq, ++gc_seq);
9806 }
9807
9808 static int nf_tables_commit(struct net *net, struct sk_buff *skb)
9809 {
9810         struct nftables_pernet *nft_net = nft_pernet(net);
9811         struct nft_trans *trans, *next;
9812         unsigned int base_seq, gc_seq;
9813         LIST_HEAD(set_update_list);
9814         struct nft_trans_elem *te;
9815         struct nft_chain *chain;
9816         struct nft_table *table;
9817         LIST_HEAD(adl);
9818         int err;
9819
9820         if (list_empty(&nft_net->commit_list)) {
9821                 mutex_unlock(&nft_net->commit_mutex);
9822                 return 0;
9823         }
9824
9825         list_for_each_entry(trans, &nft_net->binding_list, binding_list) {
9826                 switch (trans->msg_type) {
9827                 case NFT_MSG_NEWSET:
9828                         if (!nft_trans_set_update(trans) &&
9829                             nft_set_is_anonymous(nft_trans_set(trans)) &&
9830                             !nft_trans_set_bound(trans)) {
9831                                 pr_warn_once("nftables ruleset with unbound set\n");
9832                                 return -EINVAL;
9833                         }
9834                         break;
9835                 case NFT_MSG_NEWCHAIN:
9836                         if (!nft_trans_chain_update(trans) &&
9837                             nft_chain_binding(nft_trans_chain(trans)) &&
9838                             !nft_trans_chain_bound(trans)) {
9839                                 pr_warn_once("nftables ruleset with unbound chain\n");
9840                                 return -EINVAL;
9841                         }
9842                         break;
9843                 }
9844         }
9845
9846         /* 0. Validate ruleset, otherwise roll back for error reporting. */
9847         if (nf_tables_validate(net) < 0) {
9848                 nft_net->validate_state = NFT_VALIDATE_DO;
9849                 return -EAGAIN;
9850         }
9851
9852         err = nft_flow_rule_offload_commit(net);
9853         if (err < 0)
9854                 return err;
9855
9856         /* 1.  Allocate space for next generation rules_gen_X[] */
9857         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
9858                 int ret;
9859
9860                 ret = nf_tables_commit_audit_alloc(&adl, trans->ctx.table);
9861                 if (ret) {
9862                         nf_tables_commit_chain_prepare_cancel(net);
9863                         nf_tables_commit_audit_free(&adl);
9864                         return ret;
9865                 }
9866                 if (trans->msg_type == NFT_MSG_NEWRULE ||
9867                     trans->msg_type == NFT_MSG_DELRULE) {
9868                         chain = trans->ctx.chain;
9869
9870                         ret = nf_tables_commit_chain_prepare(net, chain);
9871                         if (ret < 0) {
9872                                 nf_tables_commit_chain_prepare_cancel(net);
9873                                 nf_tables_commit_audit_free(&adl);
9874                                 return ret;
9875                         }
9876                 }
9877         }
9878
9879         /* step 2.  Make rules_gen_X visible to packet path */
9880         list_for_each_entry(table, &nft_net->tables, list) {
9881                 list_for_each_entry(chain, &table->chains, list)
9882                         nf_tables_commit_chain(net, chain);
9883         }
9884
9885         /*
9886          * Bump generation counter, invalidate any dump in progress.
9887          * Cannot fail after this point.
9888          */
9889         base_seq = READ_ONCE(nft_net->base_seq);
9890         while (++base_seq == 0)
9891                 ;
9892
9893         WRITE_ONCE(nft_net->base_seq, base_seq);
9894
9895         gc_seq = nft_gc_seq_begin(nft_net);
9896
9897         /* step 3. Start new generation, rules_gen_X now in use. */
9898         net->nft.gencursor = nft_gencursor_next(net);
9899
9900         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
9901                 nf_tables_commit_audit_collect(&adl, trans->ctx.table,
9902                                                trans->msg_type);
9903                 switch (trans->msg_type) {
9904                 case NFT_MSG_NEWTABLE:
9905                         if (nft_trans_table_update(trans)) {
9906                                 if (!(trans->ctx.table->flags & __NFT_TABLE_F_UPDATE)) {
9907                                         nft_trans_destroy(trans);
9908                                         break;
9909                                 }
9910                                 if (trans->ctx.table->flags & NFT_TABLE_F_DORMANT)
9911                                         nf_tables_table_disable(net, trans->ctx.table);
9912
9913                                 trans->ctx.table->flags &= ~__NFT_TABLE_F_UPDATE;
9914                         } else {
9915                                 nft_clear(net, trans->ctx.table);
9916                         }
9917                         nf_tables_table_notify(&trans->ctx, NFT_MSG_NEWTABLE);
9918                         nft_trans_destroy(trans);
9919                         break;
9920                 case NFT_MSG_DELTABLE:
9921                 case NFT_MSG_DESTROYTABLE:
9922                         list_del_rcu(&trans->ctx.table->list);
9923                         nf_tables_table_notify(&trans->ctx, trans->msg_type);
9924                         break;
9925                 case NFT_MSG_NEWCHAIN:
9926                         if (nft_trans_chain_update(trans)) {
9927                                 nft_chain_commit_update(trans);
9928                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN,
9929                                                        &nft_trans_chain_hooks(trans));
9930                                 list_splice(&nft_trans_chain_hooks(trans),
9931                                             &nft_trans_basechain(trans)->hook_list);
9932                                 /* trans destroyed after rcu grace period */
9933                         } else {
9934                                 nft_chain_commit_drop_policy(trans);
9935                                 nft_clear(net, trans->ctx.chain);
9936                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN, NULL);
9937                                 nft_trans_destroy(trans);
9938                         }
9939                         break;
9940                 case NFT_MSG_DELCHAIN:
9941                 case NFT_MSG_DESTROYCHAIN:
9942                         if (nft_trans_chain_update(trans)) {
9943                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_DELCHAIN,
9944                                                        &nft_trans_chain_hooks(trans));
9945                                 nft_netdev_unregister_hooks(net,
9946                                                             &nft_trans_chain_hooks(trans),
9947                                                             true);
9948                         } else {
9949                                 nft_chain_del(trans->ctx.chain);
9950                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_DELCHAIN,
9951                                                        NULL);
9952                                 nf_tables_unregister_hook(trans->ctx.net,
9953                                                           trans->ctx.table,
9954                                                           trans->ctx.chain);
9955                         }
9956                         break;
9957                 case NFT_MSG_NEWRULE:
9958                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
9959                         nf_tables_rule_notify(&trans->ctx,
9960                                               nft_trans_rule(trans),
9961                                               NFT_MSG_NEWRULE);
9962                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
9963                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
9964
9965                         nft_trans_destroy(trans);
9966                         break;
9967                 case NFT_MSG_DELRULE:
9968                 case NFT_MSG_DESTROYRULE:
9969                         list_del_rcu(&nft_trans_rule(trans)->list);
9970                         nf_tables_rule_notify(&trans->ctx,
9971                                               nft_trans_rule(trans),
9972                                               trans->msg_type);
9973                         nft_rule_expr_deactivate(&trans->ctx,
9974                                                  nft_trans_rule(trans),
9975                                                  NFT_TRANS_COMMIT);
9976
9977                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
9978                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
9979                         break;
9980                 case NFT_MSG_NEWSET:
9981                         if (nft_trans_set_update(trans)) {
9982                                 struct nft_set *set = nft_trans_set(trans);
9983
9984                                 WRITE_ONCE(set->timeout, nft_trans_set_timeout(trans));
9985                                 WRITE_ONCE(set->gc_int, nft_trans_set_gc_int(trans));
9986
9987                                 if (nft_trans_set_size(trans))
9988                                         WRITE_ONCE(set->size, nft_trans_set_size(trans));
9989                         } else {
9990                                 nft_clear(net, nft_trans_set(trans));
9991                                 /* This avoids hitting -EBUSY when deleting the table
9992                                  * from the transaction.
9993                                  */
9994                                 if (nft_set_is_anonymous(nft_trans_set(trans)) &&
9995                                     !list_empty(&nft_trans_set(trans)->bindings))
9996                                         nft_use_dec(&trans->ctx.table->use);
9997                         }
9998                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
9999                                              NFT_MSG_NEWSET, GFP_KERNEL);
10000                         nft_trans_destroy(trans);
10001                         break;
10002                 case NFT_MSG_DELSET:
10003                 case NFT_MSG_DESTROYSET:
10004                         nft_trans_set(trans)->dead = 1;
10005                         list_del_rcu(&nft_trans_set(trans)->list);
10006                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
10007                                              trans->msg_type, GFP_KERNEL);
10008                         break;
10009                 case NFT_MSG_NEWSETELEM:
10010                         te = (struct nft_trans_elem *)trans->data;
10011
10012                         nft_setelem_activate(net, te->set, &te->elem);
10013                         nf_tables_setelem_notify(&trans->ctx, te->set,
10014                                                  &te->elem,
10015                                                  NFT_MSG_NEWSETELEM);
10016                         if (te->set->ops->commit &&
10017                             list_empty(&te->set->pending_update)) {
10018                                 list_add_tail(&te->set->pending_update,
10019                                               &set_update_list);
10020                         }
10021                         nft_trans_destroy(trans);
10022                         break;
10023                 case NFT_MSG_DELSETELEM:
10024                 case NFT_MSG_DESTROYSETELEM:
10025                         te = (struct nft_trans_elem *)trans->data;
10026
10027                         nf_tables_setelem_notify(&trans->ctx, te->set,
10028                                                  &te->elem,
10029                                                  trans->msg_type);
10030                         nft_setelem_remove(net, te->set, &te->elem);
10031                         if (!nft_setelem_is_catchall(te->set, &te->elem)) {
10032                                 atomic_dec(&te->set->nelems);
10033                                 te->set->ndeact--;
10034                         }
10035                         if (te->set->ops->commit &&
10036                             list_empty(&te->set->pending_update)) {
10037                                 list_add_tail(&te->set->pending_update,
10038                                               &set_update_list);
10039                         }
10040                         break;
10041                 case NFT_MSG_NEWOBJ:
10042                         if (nft_trans_obj_update(trans)) {
10043                                 nft_obj_commit_update(trans);
10044                                 nf_tables_obj_notify(&trans->ctx,
10045                                                      nft_trans_obj(trans),
10046                                                      NFT_MSG_NEWOBJ);
10047                         } else {
10048                                 nft_clear(net, nft_trans_obj(trans));
10049                                 nf_tables_obj_notify(&trans->ctx,
10050                                                      nft_trans_obj(trans),
10051                                                      NFT_MSG_NEWOBJ);
10052                                 nft_trans_destroy(trans);
10053                         }
10054                         break;
10055                 case NFT_MSG_DELOBJ:
10056                 case NFT_MSG_DESTROYOBJ:
10057                         nft_obj_del(nft_trans_obj(trans));
10058                         nf_tables_obj_notify(&trans->ctx, nft_trans_obj(trans),
10059                                              trans->msg_type);
10060                         break;
10061                 case NFT_MSG_NEWFLOWTABLE:
10062                         if (nft_trans_flowtable_update(trans)) {
10063                                 nft_trans_flowtable(trans)->data.flags =
10064                                         nft_trans_flowtable_flags(trans);
10065                                 nf_tables_flowtable_notify(&trans->ctx,
10066                                                            nft_trans_flowtable(trans),
10067                                                            &nft_trans_flowtable_hooks(trans),
10068                                                            NFT_MSG_NEWFLOWTABLE);
10069                                 list_splice(&nft_trans_flowtable_hooks(trans),
10070                                             &nft_trans_flowtable(trans)->hook_list);
10071                         } else {
10072                                 nft_clear(net, nft_trans_flowtable(trans));
10073                                 nf_tables_flowtable_notify(&trans->ctx,
10074                                                            nft_trans_flowtable(trans),
10075                                                            NULL,
10076                                                            NFT_MSG_NEWFLOWTABLE);
10077                         }
10078                         nft_trans_destroy(trans);
10079                         break;
10080                 case NFT_MSG_DELFLOWTABLE:
10081                 case NFT_MSG_DESTROYFLOWTABLE:
10082                         if (nft_trans_flowtable_update(trans)) {
10083                                 nf_tables_flowtable_notify(&trans->ctx,
10084                                                            nft_trans_flowtable(trans),
10085                                                            &nft_trans_flowtable_hooks(trans),
10086                                                            trans->msg_type);
10087                                 nft_unregister_flowtable_net_hooks(net,
10088                                                                    &nft_trans_flowtable_hooks(trans));
10089                         } else {
10090                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
10091                                 nf_tables_flowtable_notify(&trans->ctx,
10092                                                            nft_trans_flowtable(trans),
10093                                                            NULL,
10094                                                            trans->msg_type);
10095                                 nft_unregister_flowtable_net_hooks(net,
10096                                                 &nft_trans_flowtable(trans)->hook_list);
10097                         }
10098                         break;
10099                 }
10100         }
10101
10102         nft_set_commit_update(&set_update_list);
10103
10104         nft_commit_notify(net, NETLINK_CB(skb).portid);
10105         nf_tables_gen_notify(net, skb, NFT_MSG_NEWGEN);
10106         nf_tables_commit_audit_log(&adl, nft_net->base_seq);
10107
10108         nft_gc_seq_end(nft_net, gc_seq);
10109         nft_net->validate_state = NFT_VALIDATE_SKIP;
10110         nf_tables_commit_release(net);
10111
10112         return 0;
10113 }
10114
10115 static void nf_tables_module_autoload(struct net *net)
10116 {
10117         struct nftables_pernet *nft_net = nft_pernet(net);
10118         struct nft_module_request *req, *next;
10119         LIST_HEAD(module_list);
10120
10121         list_splice_init(&nft_net->module_list, &module_list);
10122         mutex_unlock(&nft_net->commit_mutex);
10123         list_for_each_entry_safe(req, next, &module_list, list) {
10124                 request_module("%s", req->module);
10125                 req->done = true;
10126         }
10127         mutex_lock(&nft_net->commit_mutex);
10128         list_splice(&module_list, &nft_net->module_list);
10129 }
10130
10131 static void nf_tables_abort_release(struct nft_trans *trans)
10132 {
10133         switch (trans->msg_type) {
10134         case NFT_MSG_NEWTABLE:
10135                 nf_tables_table_destroy(&trans->ctx);
10136                 break;
10137         case NFT_MSG_NEWCHAIN:
10138                 if (nft_trans_chain_update(trans))
10139                         nft_hooks_destroy(&nft_trans_chain_hooks(trans));
10140                 else
10141                         nf_tables_chain_destroy(&trans->ctx);
10142                 break;
10143         case NFT_MSG_NEWRULE:
10144                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
10145                 break;
10146         case NFT_MSG_NEWSET:
10147                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
10148                 break;
10149         case NFT_MSG_NEWSETELEM:
10150                 nft_set_elem_destroy(nft_trans_elem_set(trans),
10151                                      nft_trans_elem(trans).priv, true);
10152                 break;
10153         case NFT_MSG_NEWOBJ:
10154                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
10155                 break;
10156         case NFT_MSG_NEWFLOWTABLE:
10157                 if (nft_trans_flowtable_update(trans))
10158                         nft_hooks_destroy(&nft_trans_flowtable_hooks(trans));
10159                 else
10160                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
10161                 break;
10162         }
10163         kfree(trans);
10164 }
10165
10166 static void nft_set_abort_update(struct list_head *set_update_list)
10167 {
10168         struct nft_set *set, *next;
10169
10170         list_for_each_entry_safe(set, next, set_update_list, pending_update) {
10171                 list_del_init(&set->pending_update);
10172
10173                 if (!set->ops->abort)
10174                         continue;
10175
10176                 set->ops->abort(set);
10177         }
10178 }
10179
10180 static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action)
10181 {
10182         struct nftables_pernet *nft_net = nft_pernet(net);
10183         struct nft_trans *trans, *next;
10184         LIST_HEAD(set_update_list);
10185         struct nft_trans_elem *te;
10186
10187         if (action == NFNL_ABORT_VALIDATE &&
10188             nf_tables_validate(net) < 0)
10189                 return -EAGAIN;
10190
10191         list_for_each_entry_safe_reverse(trans, next, &nft_net->commit_list,
10192                                          list) {
10193                 switch (trans->msg_type) {
10194                 case NFT_MSG_NEWTABLE:
10195                         if (nft_trans_table_update(trans)) {
10196                                 if (!(trans->ctx.table->flags & __NFT_TABLE_F_UPDATE)) {
10197                                         nft_trans_destroy(trans);
10198                                         break;
10199                                 }
10200                                 if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_DORMANT) {
10201                                         nf_tables_table_disable(net, trans->ctx.table);
10202                                         trans->ctx.table->flags |= NFT_TABLE_F_DORMANT;
10203                                 } else if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_AWAKEN) {
10204                                         trans->ctx.table->flags &= ~NFT_TABLE_F_DORMANT;
10205                                 }
10206                                 trans->ctx.table->flags &= ~__NFT_TABLE_F_UPDATE;
10207                                 nft_trans_destroy(trans);
10208                         } else {
10209                                 list_del_rcu(&trans->ctx.table->list);
10210                         }
10211                         break;
10212                 case NFT_MSG_DELTABLE:
10213                 case NFT_MSG_DESTROYTABLE:
10214                         nft_clear(trans->ctx.net, trans->ctx.table);
10215                         nft_trans_destroy(trans);
10216                         break;
10217                 case NFT_MSG_NEWCHAIN:
10218                         if (nft_trans_chain_update(trans)) {
10219                                 nft_netdev_unregister_hooks(net,
10220                                                             &nft_trans_chain_hooks(trans),
10221                                                             true);
10222                                 free_percpu(nft_trans_chain_stats(trans));
10223                                 kfree(nft_trans_chain_name(trans));
10224                                 nft_trans_destroy(trans);
10225                         } else {
10226                                 if (nft_trans_chain_bound(trans)) {
10227                                         nft_trans_destroy(trans);
10228                                         break;
10229                                 }
10230                                 nft_use_dec_restore(&trans->ctx.table->use);
10231                                 nft_chain_del(trans->ctx.chain);
10232                                 nf_tables_unregister_hook(trans->ctx.net,
10233                                                           trans->ctx.table,
10234                                                           trans->ctx.chain);
10235                         }
10236                         break;
10237                 case NFT_MSG_DELCHAIN:
10238                 case NFT_MSG_DESTROYCHAIN:
10239                         if (nft_trans_chain_update(trans)) {
10240                                 list_splice(&nft_trans_chain_hooks(trans),
10241                                             &nft_trans_basechain(trans)->hook_list);
10242                         } else {
10243                                 nft_use_inc_restore(&trans->ctx.table->use);
10244                                 nft_clear(trans->ctx.net, trans->ctx.chain);
10245                         }
10246                         nft_trans_destroy(trans);
10247                         break;
10248                 case NFT_MSG_NEWRULE:
10249                         if (nft_trans_rule_bound(trans)) {
10250                                 nft_trans_destroy(trans);
10251                                 break;
10252                         }
10253                         nft_use_dec_restore(&trans->ctx.chain->use);
10254                         list_del_rcu(&nft_trans_rule(trans)->list);
10255                         nft_rule_expr_deactivate(&trans->ctx,
10256                                                  nft_trans_rule(trans),
10257                                                  NFT_TRANS_ABORT);
10258                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
10259                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10260                         break;
10261                 case NFT_MSG_DELRULE:
10262                 case NFT_MSG_DESTROYRULE:
10263                         nft_use_inc_restore(&trans->ctx.chain->use);
10264                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
10265                         nft_rule_expr_activate(&trans->ctx, nft_trans_rule(trans));
10266                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
10267                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10268
10269                         nft_trans_destroy(trans);
10270                         break;
10271                 case NFT_MSG_NEWSET:
10272                         if (nft_trans_set_update(trans)) {
10273                                 nft_trans_destroy(trans);
10274                                 break;
10275                         }
10276                         nft_use_dec_restore(&trans->ctx.table->use);
10277                         if (nft_trans_set_bound(trans)) {
10278                                 nft_trans_destroy(trans);
10279                                 break;
10280                         }
10281                         list_del_rcu(&nft_trans_set(trans)->list);
10282                         break;
10283                 case NFT_MSG_DELSET:
10284                 case NFT_MSG_DESTROYSET:
10285                         nft_use_inc_restore(&trans->ctx.table->use);
10286                         nft_clear(trans->ctx.net, nft_trans_set(trans));
10287                         if (nft_trans_set(trans)->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
10288                                 nft_map_activate(&trans->ctx, nft_trans_set(trans));
10289
10290                         nft_trans_destroy(trans);
10291                         break;
10292                 case NFT_MSG_NEWSETELEM:
10293                         if (nft_trans_elem_set_bound(trans)) {
10294                                 nft_trans_destroy(trans);
10295                                 break;
10296                         }
10297                         te = (struct nft_trans_elem *)trans->data;
10298                         nft_setelem_remove(net, te->set, &te->elem);
10299                         if (!nft_setelem_is_catchall(te->set, &te->elem))
10300                                 atomic_dec(&te->set->nelems);
10301
10302                         if (te->set->ops->abort &&
10303                             list_empty(&te->set->pending_update)) {
10304                                 list_add_tail(&te->set->pending_update,
10305                                               &set_update_list);
10306                         }
10307                         break;
10308                 case NFT_MSG_DELSETELEM:
10309                 case NFT_MSG_DESTROYSETELEM:
10310                         te = (struct nft_trans_elem *)trans->data;
10311
10312                         nft_setelem_data_activate(net, te->set, &te->elem);
10313                         nft_setelem_activate(net, te->set, &te->elem);
10314                         if (!nft_setelem_is_catchall(te->set, &te->elem))
10315                                 te->set->ndeact--;
10316
10317                         if (te->set->ops->abort &&
10318                             list_empty(&te->set->pending_update)) {
10319                                 list_add_tail(&te->set->pending_update,
10320                                               &set_update_list);
10321                         }
10322                         nft_trans_destroy(trans);
10323                         break;
10324                 case NFT_MSG_NEWOBJ:
10325                         if (nft_trans_obj_update(trans)) {
10326                                 nft_obj_destroy(&trans->ctx, nft_trans_obj_newobj(trans));
10327                                 nft_trans_destroy(trans);
10328                         } else {
10329                                 nft_use_dec_restore(&trans->ctx.table->use);
10330                                 nft_obj_del(nft_trans_obj(trans));
10331                         }
10332                         break;
10333                 case NFT_MSG_DELOBJ:
10334                 case NFT_MSG_DESTROYOBJ:
10335                         nft_use_inc_restore(&trans->ctx.table->use);
10336                         nft_clear(trans->ctx.net, nft_trans_obj(trans));
10337                         nft_trans_destroy(trans);
10338                         break;
10339                 case NFT_MSG_NEWFLOWTABLE:
10340                         if (nft_trans_flowtable_update(trans)) {
10341                                 nft_unregister_flowtable_net_hooks(net,
10342                                                 &nft_trans_flowtable_hooks(trans));
10343                         } else {
10344                                 nft_use_dec_restore(&trans->ctx.table->use);
10345                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
10346                                 nft_unregister_flowtable_net_hooks(net,
10347                                                 &nft_trans_flowtable(trans)->hook_list);
10348                         }
10349                         break;
10350                 case NFT_MSG_DELFLOWTABLE:
10351                 case NFT_MSG_DESTROYFLOWTABLE:
10352                         if (nft_trans_flowtable_update(trans)) {
10353                                 list_splice(&nft_trans_flowtable_hooks(trans),
10354                                             &nft_trans_flowtable(trans)->hook_list);
10355                         } else {
10356                                 nft_use_inc_restore(&trans->ctx.table->use);
10357                                 nft_clear(trans->ctx.net, nft_trans_flowtable(trans));
10358                         }
10359                         nft_trans_destroy(trans);
10360                         break;
10361                 }
10362         }
10363
10364         nft_set_abort_update(&set_update_list);
10365
10366         synchronize_rcu();
10367
10368         list_for_each_entry_safe_reverse(trans, next,
10369                                          &nft_net->commit_list, list) {
10370                 nft_trans_list_del(trans);
10371                 nf_tables_abort_release(trans);
10372         }
10373
10374         if (action == NFNL_ABORT_AUTOLOAD)
10375                 nf_tables_module_autoload(net);
10376         else
10377                 nf_tables_module_autoload_cleanup(net);
10378
10379         return 0;
10380 }
10381
10382 static int nf_tables_abort(struct net *net, struct sk_buff *skb,
10383                            enum nfnl_abort_action action)
10384 {
10385         struct nftables_pernet *nft_net = nft_pernet(net);
10386         unsigned int gc_seq;
10387         int ret;
10388
10389         gc_seq = nft_gc_seq_begin(nft_net);
10390         ret = __nf_tables_abort(net, action);
10391         nft_gc_seq_end(nft_net, gc_seq);
10392         mutex_unlock(&nft_net->commit_mutex);
10393
10394         return ret;
10395 }
10396
10397 static bool nf_tables_valid_genid(struct net *net, u32 genid)
10398 {
10399         struct nftables_pernet *nft_net = nft_pernet(net);
10400         bool genid_ok;
10401
10402         mutex_lock(&nft_net->commit_mutex);
10403
10404         genid_ok = genid == 0 || nft_net->base_seq == genid;
10405         if (!genid_ok)
10406                 mutex_unlock(&nft_net->commit_mutex);
10407
10408         /* else, commit mutex has to be released by commit or abort function */
10409         return genid_ok;
10410 }
10411
10412 static const struct nfnetlink_subsystem nf_tables_subsys = {
10413         .name           = "nf_tables",
10414         .subsys_id      = NFNL_SUBSYS_NFTABLES,
10415         .cb_count       = NFT_MSG_MAX,
10416         .cb             = nf_tables_cb,
10417         .commit         = nf_tables_commit,
10418         .abort          = nf_tables_abort,
10419         .valid_genid    = nf_tables_valid_genid,
10420         .owner          = THIS_MODULE,
10421 };
10422
10423 int nft_chain_validate_dependency(const struct nft_chain *chain,
10424                                   enum nft_chain_types type)
10425 {
10426         const struct nft_base_chain *basechain;
10427
10428         if (nft_is_base_chain(chain)) {
10429                 basechain = nft_base_chain(chain);
10430                 if (basechain->type->type != type)
10431                         return -EOPNOTSUPP;
10432         }
10433         return 0;
10434 }
10435 EXPORT_SYMBOL_GPL(nft_chain_validate_dependency);
10436
10437 int nft_chain_validate_hooks(const struct nft_chain *chain,
10438                              unsigned int hook_flags)
10439 {
10440         struct nft_base_chain *basechain;
10441
10442         if (nft_is_base_chain(chain)) {
10443                 basechain = nft_base_chain(chain);
10444
10445                 if ((1 << basechain->ops.hooknum) & hook_flags)
10446                         return 0;
10447
10448                 return -EOPNOTSUPP;
10449         }
10450
10451         return 0;
10452 }
10453 EXPORT_SYMBOL_GPL(nft_chain_validate_hooks);
10454
10455 /*
10456  * Loop detection - walk through the ruleset beginning at the destination chain
10457  * of a new jump until either the source chain is reached (loop) or all
10458  * reachable chains have been traversed.
10459  *
10460  * The loop check is performed whenever a new jump verdict is added to an
10461  * expression or verdict map or a verdict map is bound to a new chain.
10462  */
10463
10464 static int nf_tables_check_loops(const struct nft_ctx *ctx,
10465                                  const struct nft_chain *chain);
10466
10467 static int nft_check_loops(const struct nft_ctx *ctx,
10468                            const struct nft_set_ext *ext)
10469 {
10470         const struct nft_data *data;
10471         int ret;
10472
10473         data = nft_set_ext_data(ext);
10474         switch (data->verdict.code) {
10475         case NFT_JUMP:
10476         case NFT_GOTO:
10477                 ret = nf_tables_check_loops(ctx, data->verdict.chain);
10478                 break;
10479         default:
10480                 ret = 0;
10481                 break;
10482         }
10483
10484         return ret;
10485 }
10486
10487 static int nf_tables_loop_check_setelem(const struct nft_ctx *ctx,
10488                                         struct nft_set *set,
10489                                         const struct nft_set_iter *iter,
10490                                         struct nft_set_elem *elem)
10491 {
10492         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
10493
10494         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
10495             *nft_set_ext_flags(ext) & NFT_SET_ELEM_INTERVAL_END)
10496                 return 0;
10497
10498         return nft_check_loops(ctx, ext);
10499 }
10500
10501 static int nft_set_catchall_loops(const struct nft_ctx *ctx,
10502                                   struct nft_set *set)
10503 {
10504         u8 genmask = nft_genmask_next(ctx->net);
10505         struct nft_set_elem_catchall *catchall;
10506         struct nft_set_ext *ext;
10507         int ret = 0;
10508
10509         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
10510                 ext = nft_set_elem_ext(set, catchall->elem);
10511                 if (!nft_set_elem_active(ext, genmask))
10512                         continue;
10513
10514                 ret = nft_check_loops(ctx, ext);
10515                 if (ret < 0)
10516                         return ret;
10517         }
10518
10519         return ret;
10520 }
10521
10522 static int nf_tables_check_loops(const struct nft_ctx *ctx,
10523                                  const struct nft_chain *chain)
10524 {
10525         const struct nft_rule *rule;
10526         const struct nft_expr *expr, *last;
10527         struct nft_set *set;
10528         struct nft_set_binding *binding;
10529         struct nft_set_iter iter;
10530
10531         if (ctx->chain == chain)
10532                 return -ELOOP;
10533
10534         if (fatal_signal_pending(current))
10535                 return -EINTR;
10536
10537         list_for_each_entry(rule, &chain->rules, list) {
10538                 nft_rule_for_each_expr(expr, last, rule) {
10539                         struct nft_immediate_expr *priv;
10540                         const struct nft_data *data;
10541                         int err;
10542
10543                         if (strcmp(expr->ops->type->name, "immediate"))
10544                                 continue;
10545
10546                         priv = nft_expr_priv(expr);
10547                         if (priv->dreg != NFT_REG_VERDICT)
10548                                 continue;
10549
10550                         data = &priv->data;
10551                         switch (data->verdict.code) {
10552                         case NFT_JUMP:
10553                         case NFT_GOTO:
10554                                 err = nf_tables_check_loops(ctx,
10555                                                         data->verdict.chain);
10556                                 if (err < 0)
10557                                         return err;
10558                                 break;
10559                         default:
10560                                 break;
10561                         }
10562                 }
10563         }
10564
10565         list_for_each_entry(set, &ctx->table->sets, list) {
10566                 if (!nft_is_active_next(ctx->net, set))
10567                         continue;
10568                 if (!(set->flags & NFT_SET_MAP) ||
10569                     set->dtype != NFT_DATA_VERDICT)
10570                         continue;
10571
10572                 list_for_each_entry(binding, &set->bindings, list) {
10573                         if (!(binding->flags & NFT_SET_MAP) ||
10574                             binding->chain != chain)
10575                                 continue;
10576
10577                         iter.genmask    = nft_genmask_next(ctx->net);
10578                         iter.skip       = 0;
10579                         iter.count      = 0;
10580                         iter.err        = 0;
10581                         iter.fn         = nf_tables_loop_check_setelem;
10582
10583                         set->ops->walk(ctx, set, &iter);
10584                         if (!iter.err)
10585                                 iter.err = nft_set_catchall_loops(ctx, set);
10586
10587                         if (iter.err < 0)
10588                                 return iter.err;
10589                 }
10590         }
10591
10592         return 0;
10593 }
10594
10595 /**
10596  *      nft_parse_u32_check - fetch u32 attribute and check for maximum value
10597  *
10598  *      @attr: netlink attribute to fetch value from
10599  *      @max: maximum value to be stored in dest
10600  *      @dest: pointer to the variable
10601  *
10602  *      Parse, check and store a given u32 netlink attribute into variable.
10603  *      This function returns -ERANGE if the value goes over maximum value.
10604  *      Otherwise a 0 is returned and the attribute value is stored in the
10605  *      destination variable.
10606  */
10607 int nft_parse_u32_check(const struct nlattr *attr, int max, u32 *dest)
10608 {
10609         u32 val;
10610
10611         val = ntohl(nla_get_be32(attr));
10612         if (val > max)
10613                 return -ERANGE;
10614
10615         *dest = val;
10616         return 0;
10617 }
10618 EXPORT_SYMBOL_GPL(nft_parse_u32_check);
10619
10620 static int nft_parse_register(const struct nlattr *attr, u32 *preg)
10621 {
10622         unsigned int reg;
10623
10624         reg = ntohl(nla_get_be32(attr));
10625         switch (reg) {
10626         case NFT_REG_VERDICT...NFT_REG_4:
10627                 *preg = reg * NFT_REG_SIZE / NFT_REG32_SIZE;
10628                 break;
10629         case NFT_REG32_00...NFT_REG32_15:
10630                 *preg = reg + NFT_REG_SIZE / NFT_REG32_SIZE - NFT_REG32_00;
10631                 break;
10632         default:
10633                 return -ERANGE;
10634         }
10635
10636         return 0;
10637 }
10638
10639 /**
10640  *      nft_dump_register - dump a register value to a netlink attribute
10641  *
10642  *      @skb: socket buffer
10643  *      @attr: attribute number
10644  *      @reg: register number
10645  *
10646  *      Construct a netlink attribute containing the register number. For
10647  *      compatibility reasons, register numbers being a multiple of 4 are
10648  *      translated to the corresponding 128 bit register numbers.
10649  */
10650 int nft_dump_register(struct sk_buff *skb, unsigned int attr, unsigned int reg)
10651 {
10652         if (reg % (NFT_REG_SIZE / NFT_REG32_SIZE) == 0)
10653                 reg = reg / (NFT_REG_SIZE / NFT_REG32_SIZE);
10654         else
10655                 reg = reg - NFT_REG_SIZE / NFT_REG32_SIZE + NFT_REG32_00;
10656
10657         return nla_put_be32(skb, attr, htonl(reg));
10658 }
10659 EXPORT_SYMBOL_GPL(nft_dump_register);
10660
10661 static int nft_validate_register_load(enum nft_registers reg, unsigned int len)
10662 {
10663         if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
10664                 return -EINVAL;
10665         if (len == 0)
10666                 return -EINVAL;
10667         if (reg * NFT_REG32_SIZE + len > sizeof_field(struct nft_regs, data))
10668                 return -ERANGE;
10669
10670         return 0;
10671 }
10672
10673 int nft_parse_register_load(const struct nlattr *attr, u8 *sreg, u32 len)
10674 {
10675         u32 reg;
10676         int err;
10677
10678         err = nft_parse_register(attr, &reg);
10679         if (err < 0)
10680                 return err;
10681
10682         err = nft_validate_register_load(reg, len);
10683         if (err < 0)
10684                 return err;
10685
10686         *sreg = reg;
10687         return 0;
10688 }
10689 EXPORT_SYMBOL_GPL(nft_parse_register_load);
10690
10691 static int nft_validate_register_store(const struct nft_ctx *ctx,
10692                                        enum nft_registers reg,
10693                                        const struct nft_data *data,
10694                                        enum nft_data_types type,
10695                                        unsigned int len)
10696 {
10697         int err;
10698
10699         switch (reg) {
10700         case NFT_REG_VERDICT:
10701                 if (type != NFT_DATA_VERDICT)
10702                         return -EINVAL;
10703
10704                 if (data != NULL &&
10705                     (data->verdict.code == NFT_GOTO ||
10706                      data->verdict.code == NFT_JUMP)) {
10707                         err = nf_tables_check_loops(ctx, data->verdict.chain);
10708                         if (err < 0)
10709                                 return err;
10710                 }
10711
10712                 return 0;
10713         default:
10714                 if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
10715                         return -EINVAL;
10716                 if (len == 0)
10717                         return -EINVAL;
10718                 if (reg * NFT_REG32_SIZE + len >
10719                     sizeof_field(struct nft_regs, data))
10720                         return -ERANGE;
10721
10722                 if (data != NULL && type != NFT_DATA_VALUE)
10723                         return -EINVAL;
10724                 return 0;
10725         }
10726 }
10727
10728 int nft_parse_register_store(const struct nft_ctx *ctx,
10729                              const struct nlattr *attr, u8 *dreg,
10730                              const struct nft_data *data,
10731                              enum nft_data_types type, unsigned int len)
10732 {
10733         int err;
10734         u32 reg;
10735
10736         err = nft_parse_register(attr, &reg);
10737         if (err < 0)
10738                 return err;
10739
10740         err = nft_validate_register_store(ctx, reg, data, type, len);
10741         if (err < 0)
10742                 return err;
10743
10744         *dreg = reg;
10745         return 0;
10746 }
10747 EXPORT_SYMBOL_GPL(nft_parse_register_store);
10748
10749 static const struct nla_policy nft_verdict_policy[NFTA_VERDICT_MAX + 1] = {
10750         [NFTA_VERDICT_CODE]     = { .type = NLA_U32 },
10751         [NFTA_VERDICT_CHAIN]    = { .type = NLA_STRING,
10752                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
10753         [NFTA_VERDICT_CHAIN_ID] = { .type = NLA_U32 },
10754 };
10755
10756 static int nft_verdict_init(const struct nft_ctx *ctx, struct nft_data *data,
10757                             struct nft_data_desc *desc, const struct nlattr *nla)
10758 {
10759         u8 genmask = nft_genmask_next(ctx->net);
10760         struct nlattr *tb[NFTA_VERDICT_MAX + 1];
10761         struct nft_chain *chain;
10762         int err;
10763
10764         err = nla_parse_nested_deprecated(tb, NFTA_VERDICT_MAX, nla,
10765                                           nft_verdict_policy, NULL);
10766         if (err < 0)
10767                 return err;
10768
10769         if (!tb[NFTA_VERDICT_CODE])
10770                 return -EINVAL;
10771
10772         /* zero padding hole for memcmp */
10773         memset(data, 0, sizeof(*data));
10774         data->verdict.code = ntohl(nla_get_be32(tb[NFTA_VERDICT_CODE]));
10775
10776         switch (data->verdict.code) {
10777         default:
10778                 switch (data->verdict.code & NF_VERDICT_MASK) {
10779                 case NF_ACCEPT:
10780                 case NF_DROP:
10781                 case NF_QUEUE:
10782                         break;
10783                 default:
10784                         return -EINVAL;
10785                 }
10786                 fallthrough;
10787         case NFT_CONTINUE:
10788         case NFT_BREAK:
10789         case NFT_RETURN:
10790                 break;
10791         case NFT_JUMP:
10792         case NFT_GOTO:
10793                 if (tb[NFTA_VERDICT_CHAIN]) {
10794                         chain = nft_chain_lookup(ctx->net, ctx->table,
10795                                                  tb[NFTA_VERDICT_CHAIN],
10796                                                  genmask);
10797                 } else if (tb[NFTA_VERDICT_CHAIN_ID]) {
10798                         chain = nft_chain_lookup_byid(ctx->net, ctx->table,
10799                                                       tb[NFTA_VERDICT_CHAIN_ID],
10800                                                       genmask);
10801                         if (IS_ERR(chain))
10802                                 return PTR_ERR(chain);
10803                 } else {
10804                         return -EINVAL;
10805                 }
10806
10807                 if (IS_ERR(chain))
10808                         return PTR_ERR(chain);
10809                 if (nft_is_base_chain(chain))
10810                         return -EOPNOTSUPP;
10811                 if (nft_chain_is_bound(chain))
10812                         return -EINVAL;
10813                 if (desc->flags & NFT_DATA_DESC_SETELEM &&
10814                     chain->flags & NFT_CHAIN_BINDING)
10815                         return -EINVAL;
10816                 if (!nft_use_inc(&chain->use))
10817                         return -EMFILE;
10818
10819                 data->verdict.chain = chain;
10820                 break;
10821         }
10822
10823         desc->len = sizeof(data->verdict);
10824
10825         return 0;
10826 }
10827
10828 static void nft_verdict_uninit(const struct nft_data *data)
10829 {
10830         struct nft_chain *chain;
10831
10832         switch (data->verdict.code) {
10833         case NFT_JUMP:
10834         case NFT_GOTO:
10835                 chain = data->verdict.chain;
10836                 nft_use_dec(&chain->use);
10837                 break;
10838         }
10839 }
10840
10841 int nft_verdict_dump(struct sk_buff *skb, int type, const struct nft_verdict *v)
10842 {
10843         struct nlattr *nest;
10844
10845         nest = nla_nest_start_noflag(skb, type);
10846         if (!nest)
10847                 goto nla_put_failure;
10848
10849         if (nla_put_be32(skb, NFTA_VERDICT_CODE, htonl(v->code)))
10850                 goto nla_put_failure;
10851
10852         switch (v->code) {
10853         case NFT_JUMP:
10854         case NFT_GOTO:
10855                 if (nla_put_string(skb, NFTA_VERDICT_CHAIN,
10856                                    v->chain->name))
10857                         goto nla_put_failure;
10858         }
10859         nla_nest_end(skb, nest);
10860         return 0;
10861
10862 nla_put_failure:
10863         return -1;
10864 }
10865
10866 static int nft_value_init(const struct nft_ctx *ctx,
10867                           struct nft_data *data, struct nft_data_desc *desc,
10868                           const struct nlattr *nla)
10869 {
10870         unsigned int len;
10871
10872         len = nla_len(nla);
10873         if (len == 0)
10874                 return -EINVAL;
10875         if (len > desc->size)
10876                 return -EOVERFLOW;
10877         if (desc->len) {
10878                 if (len != desc->len)
10879                         return -EINVAL;
10880         } else {
10881                 desc->len = len;
10882         }
10883
10884         nla_memcpy(data->data, nla, len);
10885
10886         return 0;
10887 }
10888
10889 static int nft_value_dump(struct sk_buff *skb, const struct nft_data *data,
10890                           unsigned int len)
10891 {
10892         return nla_put(skb, NFTA_DATA_VALUE, len, data->data);
10893 }
10894
10895 static const struct nla_policy nft_data_policy[NFTA_DATA_MAX + 1] = {
10896         [NFTA_DATA_VALUE]       = { .type = NLA_BINARY },
10897         [NFTA_DATA_VERDICT]     = { .type = NLA_NESTED },
10898 };
10899
10900 /**
10901  *      nft_data_init - parse nf_tables data netlink attributes
10902  *
10903  *      @ctx: context of the expression using the data
10904  *      @data: destination struct nft_data
10905  *      @desc: data description
10906  *      @nla: netlink attribute containing data
10907  *
10908  *      Parse the netlink data attributes and initialize a struct nft_data.
10909  *      The type and length of data are returned in the data description.
10910  *
10911  *      The caller can indicate that it only wants to accept data of type
10912  *      NFT_DATA_VALUE by passing NULL for the ctx argument.
10913  */
10914 int nft_data_init(const struct nft_ctx *ctx, struct nft_data *data,
10915                   struct nft_data_desc *desc, const struct nlattr *nla)
10916 {
10917         struct nlattr *tb[NFTA_DATA_MAX + 1];
10918         int err;
10919
10920         if (WARN_ON_ONCE(!desc->size))
10921                 return -EINVAL;
10922
10923         err = nla_parse_nested_deprecated(tb, NFTA_DATA_MAX, nla,
10924                                           nft_data_policy, NULL);
10925         if (err < 0)
10926                 return err;
10927
10928         if (tb[NFTA_DATA_VALUE]) {
10929                 if (desc->type != NFT_DATA_VALUE)
10930                         return -EINVAL;
10931
10932                 err = nft_value_init(ctx, data, desc, tb[NFTA_DATA_VALUE]);
10933         } else if (tb[NFTA_DATA_VERDICT] && ctx != NULL) {
10934                 if (desc->type != NFT_DATA_VERDICT)
10935                         return -EINVAL;
10936
10937                 err = nft_verdict_init(ctx, data, desc, tb[NFTA_DATA_VERDICT]);
10938         } else {
10939                 err = -EINVAL;
10940         }
10941
10942         return err;
10943 }
10944 EXPORT_SYMBOL_GPL(nft_data_init);
10945
10946 /**
10947  *      nft_data_release - release a nft_data item
10948  *
10949  *      @data: struct nft_data to release
10950  *      @type: type of data
10951  *
10952  *      Release a nft_data item. NFT_DATA_VALUE types can be silently discarded,
10953  *      all others need to be released by calling this function.
10954  */
10955 void nft_data_release(const struct nft_data *data, enum nft_data_types type)
10956 {
10957         if (type < NFT_DATA_VERDICT)
10958                 return;
10959         switch (type) {
10960         case NFT_DATA_VERDICT:
10961                 return nft_verdict_uninit(data);
10962         default:
10963                 WARN_ON(1);
10964         }
10965 }
10966 EXPORT_SYMBOL_GPL(nft_data_release);
10967
10968 int nft_data_dump(struct sk_buff *skb, int attr, const struct nft_data *data,
10969                   enum nft_data_types type, unsigned int len)
10970 {
10971         struct nlattr *nest;
10972         int err;
10973
10974         nest = nla_nest_start_noflag(skb, attr);
10975         if (nest == NULL)
10976                 return -1;
10977
10978         switch (type) {
10979         case NFT_DATA_VALUE:
10980                 err = nft_value_dump(skb, data, len);
10981                 break;
10982         case NFT_DATA_VERDICT:
10983                 err = nft_verdict_dump(skb, NFTA_DATA_VERDICT, &data->verdict);
10984                 break;
10985         default:
10986                 err = -EINVAL;
10987                 WARN_ON(1);
10988         }
10989
10990         nla_nest_end(skb, nest);
10991         return err;
10992 }
10993 EXPORT_SYMBOL_GPL(nft_data_dump);
10994
10995 int __nft_release_basechain(struct nft_ctx *ctx)
10996 {
10997         struct nft_rule *rule, *nr;
10998
10999         if (WARN_ON(!nft_is_base_chain(ctx->chain)))
11000                 return 0;
11001
11002         nf_tables_unregister_hook(ctx->net, ctx->chain->table, ctx->chain);
11003         list_for_each_entry_safe(rule, nr, &ctx->chain->rules, list) {
11004                 list_del(&rule->list);
11005                 nft_use_dec(&ctx->chain->use);
11006                 nf_tables_rule_release(ctx, rule);
11007         }
11008         nft_chain_del(ctx->chain);
11009         nft_use_dec(&ctx->table->use);
11010         nf_tables_chain_destroy(ctx);
11011
11012         return 0;
11013 }
11014 EXPORT_SYMBOL_GPL(__nft_release_basechain);
11015
11016 static void __nft_release_hook(struct net *net, struct nft_table *table)
11017 {
11018         struct nft_flowtable *flowtable;
11019         struct nft_chain *chain;
11020
11021         list_for_each_entry(chain, &table->chains, list)
11022                 __nf_tables_unregister_hook(net, table, chain, true);
11023         list_for_each_entry(flowtable, &table->flowtables, list)
11024                 __nft_unregister_flowtable_net_hooks(net, &flowtable->hook_list,
11025                                                      true);
11026 }
11027
11028 static void __nft_release_hooks(struct net *net)
11029 {
11030         struct nftables_pernet *nft_net = nft_pernet(net);
11031         struct nft_table *table;
11032
11033         list_for_each_entry(table, &nft_net->tables, list) {
11034                 if (nft_table_has_owner(table))
11035                         continue;
11036
11037                 __nft_release_hook(net, table);
11038         }
11039 }
11040
11041 static void __nft_release_table(struct net *net, struct nft_table *table)
11042 {
11043         struct nft_flowtable *flowtable, *nf;
11044         struct nft_chain *chain, *nc;
11045         struct nft_object *obj, *ne;
11046         struct nft_rule *rule, *nr;
11047         struct nft_set *set, *ns;
11048         struct nft_ctx ctx = {
11049                 .net    = net,
11050                 .family = NFPROTO_NETDEV,
11051         };
11052
11053         ctx.family = table->family;
11054         ctx.table = table;
11055         list_for_each_entry(chain, &table->chains, list) {
11056                 if (nft_chain_is_bound(chain))
11057                         continue;
11058
11059                 ctx.chain = chain;
11060                 list_for_each_entry_safe(rule, nr, &chain->rules, list) {
11061                         list_del(&rule->list);
11062                         nft_use_dec(&chain->use);
11063                         nf_tables_rule_release(&ctx, rule);
11064                 }
11065         }
11066         list_for_each_entry_safe(flowtable, nf, &table->flowtables, list) {
11067                 list_del(&flowtable->list);
11068                 nft_use_dec(&table->use);
11069                 nf_tables_flowtable_destroy(flowtable);
11070         }
11071         list_for_each_entry_safe(set, ns, &table->sets, list) {
11072                 list_del(&set->list);
11073                 nft_use_dec(&table->use);
11074                 if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
11075                         nft_map_deactivate(&ctx, set);
11076
11077                 nft_set_destroy(&ctx, set);
11078         }
11079         list_for_each_entry_safe(obj, ne, &table->objects, list) {
11080                 nft_obj_del(obj);
11081                 nft_use_dec(&table->use);
11082                 nft_obj_destroy(&ctx, obj);
11083         }
11084         list_for_each_entry_safe(chain, nc, &table->chains, list) {
11085                 ctx.chain = chain;
11086                 nft_chain_del(chain);
11087                 nft_use_dec(&table->use);
11088                 nf_tables_chain_destroy(&ctx);
11089         }
11090         nf_tables_table_destroy(&ctx);
11091 }
11092
11093 static void __nft_release_tables(struct net *net)
11094 {
11095         struct nftables_pernet *nft_net = nft_pernet(net);
11096         struct nft_table *table, *nt;
11097
11098         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
11099                 if (nft_table_has_owner(table))
11100                         continue;
11101
11102                 list_del(&table->list);
11103
11104                 __nft_release_table(net, table);
11105         }
11106 }
11107
11108 static int nft_rcv_nl_event(struct notifier_block *this, unsigned long event,
11109                             void *ptr)
11110 {
11111         struct nft_table *table, *to_delete[8];
11112         struct nftables_pernet *nft_net;
11113         struct netlink_notify *n = ptr;
11114         struct net *net = n->net;
11115         unsigned int deleted;
11116         bool restart = false;
11117         unsigned int gc_seq;
11118
11119         if (event != NETLINK_URELEASE || n->protocol != NETLINK_NETFILTER)
11120                 return NOTIFY_DONE;
11121
11122         nft_net = nft_pernet(net);
11123         deleted = 0;
11124         mutex_lock(&nft_net->commit_mutex);
11125
11126         gc_seq = nft_gc_seq_begin(nft_net);
11127
11128         if (!list_empty(&nf_tables_destroy_list))
11129                 nf_tables_trans_destroy_flush_work();
11130 again:
11131         list_for_each_entry(table, &nft_net->tables, list) {
11132                 if (nft_table_has_owner(table) &&
11133                     n->portid == table->nlpid) {
11134                         __nft_release_hook(net, table);
11135                         list_del_rcu(&table->list);
11136                         to_delete[deleted++] = table;
11137                         if (deleted >= ARRAY_SIZE(to_delete))
11138                                 break;
11139                 }
11140         }
11141         if (deleted) {
11142                 restart = deleted >= ARRAY_SIZE(to_delete);
11143                 synchronize_rcu();
11144                 while (deleted)
11145                         __nft_release_table(net, to_delete[--deleted]);
11146
11147                 if (restart)
11148                         goto again;
11149         }
11150         nft_gc_seq_end(nft_net, gc_seq);
11151
11152         mutex_unlock(&nft_net->commit_mutex);
11153
11154         return NOTIFY_DONE;
11155 }
11156
11157 static struct notifier_block nft_nl_notifier = {
11158         .notifier_call  = nft_rcv_nl_event,
11159 };
11160
11161 static int __net_init nf_tables_init_net(struct net *net)
11162 {
11163         struct nftables_pernet *nft_net = nft_pernet(net);
11164
11165         INIT_LIST_HEAD(&nft_net->tables);
11166         INIT_LIST_HEAD(&nft_net->commit_list);
11167         INIT_LIST_HEAD(&nft_net->binding_list);
11168         INIT_LIST_HEAD(&nft_net->module_list);
11169         INIT_LIST_HEAD(&nft_net->notify_list);
11170         mutex_init(&nft_net->commit_mutex);
11171         nft_net->base_seq = 1;
11172         nft_net->gc_seq = 0;
11173         nft_net->validate_state = NFT_VALIDATE_SKIP;
11174
11175         return 0;
11176 }
11177
11178 static void __net_exit nf_tables_pre_exit_net(struct net *net)
11179 {
11180         struct nftables_pernet *nft_net = nft_pernet(net);
11181
11182         mutex_lock(&nft_net->commit_mutex);
11183         __nft_release_hooks(net);
11184         mutex_unlock(&nft_net->commit_mutex);
11185 }
11186
11187 static void __net_exit nf_tables_exit_net(struct net *net)
11188 {
11189         struct nftables_pernet *nft_net = nft_pernet(net);
11190         unsigned int gc_seq;
11191
11192         mutex_lock(&nft_net->commit_mutex);
11193
11194         gc_seq = nft_gc_seq_begin(nft_net);
11195
11196         if (!list_empty(&nft_net->commit_list) ||
11197             !list_empty(&nft_net->module_list))
11198                 __nf_tables_abort(net, NFNL_ABORT_NONE);
11199
11200         __nft_release_tables(net);
11201
11202         nft_gc_seq_end(nft_net, gc_seq);
11203
11204         mutex_unlock(&nft_net->commit_mutex);
11205         WARN_ON_ONCE(!list_empty(&nft_net->tables));
11206         WARN_ON_ONCE(!list_empty(&nft_net->module_list));
11207         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
11208 }
11209
11210 static void nf_tables_exit_batch(struct list_head *net_exit_list)
11211 {
11212         flush_work(&trans_gc_work);
11213 }
11214
11215 static struct pernet_operations nf_tables_net_ops = {
11216         .init           = nf_tables_init_net,
11217         .pre_exit       = nf_tables_pre_exit_net,
11218         .exit           = nf_tables_exit_net,
11219         .exit_batch     = nf_tables_exit_batch,
11220         .id             = &nf_tables_net_id,
11221         .size           = sizeof(struct nftables_pernet),
11222 };
11223
11224 static int __init nf_tables_module_init(void)
11225 {
11226         int err;
11227
11228         err = register_pernet_subsys(&nf_tables_net_ops);
11229         if (err < 0)
11230                 return err;
11231
11232         err = nft_chain_filter_init();
11233         if (err < 0)
11234                 goto err_chain_filter;
11235
11236         err = nf_tables_core_module_init();
11237         if (err < 0)
11238                 goto err_core_module;
11239
11240         err = register_netdevice_notifier(&nf_tables_flowtable_notifier);
11241         if (err < 0)
11242                 goto err_netdev_notifier;
11243
11244         err = rhltable_init(&nft_objname_ht, &nft_objname_ht_params);
11245         if (err < 0)
11246                 goto err_rht_objname;
11247
11248         err = nft_offload_init();
11249         if (err < 0)
11250                 goto err_offload;
11251
11252         err = netlink_register_notifier(&nft_nl_notifier);
11253         if (err < 0)
11254                 goto err_netlink_notifier;
11255
11256         /* must be last */
11257         err = nfnetlink_subsys_register(&nf_tables_subsys);
11258         if (err < 0)
11259                 goto err_nfnl_subsys;
11260
11261         nft_chain_route_init();
11262
11263         return err;
11264
11265 err_nfnl_subsys:
11266         netlink_unregister_notifier(&nft_nl_notifier);
11267 err_netlink_notifier:
11268         nft_offload_exit();
11269 err_offload:
11270         rhltable_destroy(&nft_objname_ht);
11271 err_rht_objname:
11272         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
11273 err_netdev_notifier:
11274         nf_tables_core_module_exit();
11275 err_core_module:
11276         nft_chain_filter_fini();
11277 err_chain_filter:
11278         unregister_pernet_subsys(&nf_tables_net_ops);
11279         return err;
11280 }
11281
11282 static void __exit nf_tables_module_exit(void)
11283 {
11284         nfnetlink_subsys_unregister(&nf_tables_subsys);
11285         netlink_unregister_notifier(&nft_nl_notifier);
11286         nft_offload_exit();
11287         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
11288         nft_chain_filter_fini();
11289         nft_chain_route_fini();
11290         unregister_pernet_subsys(&nf_tables_net_ops);
11291         cancel_work_sync(&trans_gc_work);
11292         cancel_work_sync(&trans_destroy_work);
11293         rcu_barrier();
11294         rhltable_destroy(&nft_objname_ht);
11295         nf_tables_core_module_exit();
11296 }
11297
11298 module_init(nf_tables_module_init);
11299 module_exit(nf_tables_module_exit);
11300
11301 MODULE_LICENSE("GPL");
11302 MODULE_AUTHOR("Patrick McHardy <kaber@trash.net>");
11303 MODULE_ALIAS_NFNL_SUBSYS(NFNL_SUBSYS_NFTABLES);