net/netfilter/nf_tables_api.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2007-2009 Patrick McHardy <kaber@trash.net>
   4  *
   5  * Development of this code funded by Astaro AG (http://www.astaro.com/)
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/init.h>
  10 #include <linux/list.h>
  11 #include <linux/skbuff.h>
  12 #include <linux/netlink.h>
  13 #include <linux/vmalloc.h>
  14 #include <linux/rhashtable.h>
  15 #include <linux/audit.h>
  16 #include <linux/netfilter.h>
  17 #include <linux/netfilter/nfnetlink.h>
  18 #include <linux/netfilter/nf_tables.h>
  19 #include <net/netfilter/nf_flow_table.h>
  20 #include <net/netfilter/nf_tables_core.h>
  21 #include <net/netfilter/nf_tables.h>
  22 #include <net/netfilter/nf_tables_offload.h>
  23 #include <net/net_namespace.h>
  24 #include <net/sock.h>
  25
  26 #define NFT_MODULE_AUTOLOAD_LIMIT (MODULE_NAME_LEN - sizeof("nft-expr-255-"))
  27 #define NFT_SET_MAX_ANONLEN 16
  28
  29 unsigned int nf_tables_net_id __read_mostly;
  30
  31 static LIST_HEAD(nf_tables_expressions);
  32 static LIST_HEAD(nf_tables_objects);
  33 static LIST_HEAD(nf_tables_flowtables);
  34 static LIST_HEAD(nf_tables_destroy_list);
  35 static LIST_HEAD(nf_tables_gc_list);
  36 static DEFINE_SPINLOCK(nf_tables_destroy_list_lock);
  37 static DEFINE_SPINLOCK(nf_tables_gc_list_lock);
  38
  39 enum {
  40         NFT_VALIDATE_SKIP       = 0,
  41         NFT_VALIDATE_NEED,
  42         NFT_VALIDATE_DO,
  43 };
  44
  45 static struct rhltable nft_objname_ht;
  46
  47 static u32 nft_chain_hash(const void *data, u32 len, u32 seed);
  48 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed);
  49 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *, const void *);
  50
  51 static u32 nft_objname_hash(const void *data, u32 len, u32 seed);
  52 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed);
  53 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *, const void *);
  54
  55 static const struct rhashtable_params nft_chain_ht_params = {
  56         .head_offset            = offsetof(struct nft_chain, rhlhead),
  57         .key_offset             = offsetof(struct nft_chain, name),
  58         .hashfn                 = nft_chain_hash,
  59         .obj_hashfn             = nft_chain_hash_obj,
  60         .obj_cmpfn              = nft_chain_hash_cmp,
  61         .automatic_shrinking    = true,
  62 };
  63
  64 static const struct rhashtable_params nft_objname_ht_params = {
  65         .head_offset            = offsetof(struct nft_object, rhlhead),
  66         .key_offset             = offsetof(struct nft_object, key),
  67         .hashfn                 = nft_objname_hash,
  68         .obj_hashfn             = nft_objname_hash_obj,
  69         .obj_cmpfn              = nft_objname_hash_cmp,
  70         .automatic_shrinking    = true,
  71 };
  72
  73 struct nft_audit_data {
  74         struct nft_table *table;
  75         int entries;
  76         int op;
  77         struct list_head list;
  78 };
  79
  80 static const u8 nft2audit_op[NFT_MSG_MAX] = { // enum nf_tables_msg_types
  81         [NFT_MSG_NEWTABLE]      = AUDIT_NFT_OP_TABLE_REGISTER,
  82         [NFT_MSG_GETTABLE]      = AUDIT_NFT_OP_INVALID,
  83         [NFT_MSG_DELTABLE]      = AUDIT_NFT_OP_TABLE_UNREGISTER,
  84         [NFT_MSG_NEWCHAIN]      = AUDIT_NFT_OP_CHAIN_REGISTER,
  85         [NFT_MSG_GETCHAIN]      = AUDIT_NFT_OP_INVALID,
  86         [NFT_MSG_DELCHAIN]      = AUDIT_NFT_OP_CHAIN_UNREGISTER,
  87         [NFT_MSG_NEWRULE]       = AUDIT_NFT_OP_RULE_REGISTER,
  88         [NFT_MSG_GETRULE]       = AUDIT_NFT_OP_INVALID,
  89         [NFT_MSG_DELRULE]       = AUDIT_NFT_OP_RULE_UNREGISTER,
  90         [NFT_MSG_NEWSET]        = AUDIT_NFT_OP_SET_REGISTER,
  91         [NFT_MSG_GETSET]        = AUDIT_NFT_OP_INVALID,
  92         [NFT_MSG_DELSET]        = AUDIT_NFT_OP_SET_UNREGISTER,
  93         [NFT_MSG_NEWSETELEM]    = AUDIT_NFT_OP_SETELEM_REGISTER,
  94         [NFT_MSG_GETSETELEM]    = AUDIT_NFT_OP_INVALID,
  95         [NFT_MSG_DELSETELEM]    = AUDIT_NFT_OP_SETELEM_UNREGISTER,
  96         [NFT_MSG_NEWGEN]        = AUDIT_NFT_OP_GEN_REGISTER,
  97         [NFT_MSG_GETGEN]        = AUDIT_NFT_OP_INVALID,
  98         [NFT_MSG_TRACE]         = AUDIT_NFT_OP_INVALID,
  99         [NFT_MSG_NEWOBJ]        = AUDIT_NFT_OP_OBJ_REGISTER,
 100         [NFT_MSG_GETOBJ]        = AUDIT_NFT_OP_INVALID,
 101         [NFT_MSG_DELOBJ]        = AUDIT_NFT_OP_OBJ_UNREGISTER,
 102         [NFT_MSG_GETOBJ_RESET]  = AUDIT_NFT_OP_OBJ_RESET,
 103         [NFT_MSG_NEWFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_REGISTER,
 104         [NFT_MSG_GETFLOWTABLE]  = AUDIT_NFT_OP_INVALID,
 105         [NFT_MSG_DELFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_UNREGISTER,
 106         [NFT_MSG_GETSETELEM_RESET] = AUDIT_NFT_OP_SETELEM_RESET,
 107 };
 108
 109 static void nft_validate_state_update(struct nft_table *table, u8 new_validate_state)
 110 {
 111         switch (table->validate_state) {
 112         case NFT_VALIDATE_SKIP:
 113                 WARN_ON_ONCE(new_validate_state == NFT_VALIDATE_DO);
 114                 break;
 115         case NFT_VALIDATE_NEED:
 116                 break;
 117         case NFT_VALIDATE_DO:
 118                 if (new_validate_state == NFT_VALIDATE_NEED)
 119                         return;
 120         }
 121
 122         table->validate_state = new_validate_state;
 123 }
 124 static void nf_tables_trans_destroy_work(struct work_struct *w);
 125 static DECLARE_WORK(trans_destroy_work, nf_tables_trans_destroy_work);
 126
 127 static void nft_trans_gc_work(struct work_struct *work);
 128 static DECLARE_WORK(trans_gc_work, nft_trans_gc_work);
 129
 130 static void nft_ctx_init(struct nft_ctx *ctx,
 131                          struct net *net,
 132                          const struct sk_buff *skb,
 133                          const struct nlmsghdr *nlh,
 134                          u8 family,
 135                          struct nft_table *table,
 136                          struct nft_chain *chain,
 137                          const struct nlattr * const *nla)
 138 {
 139         ctx->net        = net;
 140         ctx->family     = family;
 141         ctx->level      = 0;
 142         ctx->table      = table;
 143         ctx->chain      = chain;
 144         ctx->nla        = nla;
 145         ctx->portid     = NETLINK_CB(skb).portid;
 146         ctx->report     = nlmsg_report(nlh);
 147         ctx->flags      = nlh->nlmsg_flags;
 148         ctx->seq        = nlh->nlmsg_seq;
 149 }
 150
 151 static struct nft_trans *nft_trans_alloc_gfp(const struct nft_ctx *ctx,
 152                                              int msg_type, u32 size, gfp_t gfp)
 153 {
 154         struct nft_trans *trans;
 155
 156         trans = kzalloc(sizeof(struct nft_trans) + size, gfp);
 157         if (trans == NULL)
 158                 return NULL;
 159
 160         INIT_LIST_HEAD(&trans->list);
 161         INIT_LIST_HEAD(&trans->binding_list);
 162         trans->msg_type = msg_type;
 163         trans->ctx      = *ctx;
 164
 165         return trans;
 166 }
 167
 168 static struct nft_trans *nft_trans_alloc(const struct nft_ctx *ctx,
 169                                          int msg_type, u32 size)
 170 {
 171         return nft_trans_alloc_gfp(ctx, msg_type, size, GFP_KERNEL);
 172 }
 173
 174 static void nft_trans_list_del(struct nft_trans *trans)
 175 {
 176         list_del(&trans->list);
 177         list_del(&trans->binding_list);
 178 }
 179
 180 static void nft_trans_destroy(struct nft_trans *trans)
 181 {
 182         nft_trans_list_del(trans);
 183         kfree(trans);
 184 }
 185
 186 static void __nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set,
 187                                  bool bind)
 188 {
 189         struct nftables_pernet *nft_net;
 190         struct net *net = ctx->net;
 191         struct nft_trans *trans;
 192
 193         if (!nft_set_is_anonymous(set))
 194                 return;
 195
 196         nft_net = nft_pernet(net);
 197         list_for_each_entry_reverse(trans, &nft_net->commit_list, list) {
 198                 switch (trans->msg_type) {
 199                 case NFT_MSG_NEWSET:
 200                         if (nft_trans_set(trans) == set)
 201                                 nft_trans_set_bound(trans) = bind;
 202                         break;
 203                 case NFT_MSG_NEWSETELEM:
 204                         if (nft_trans_elem_set(trans) == set)
 205                                 nft_trans_elem_set_bound(trans) = bind;
 206                         break;
 207                 }
 208         }
 209 }
 210
 211 static void nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set)
 212 {
 213         return __nft_set_trans_bind(ctx, set, true);
 214 }
 215
 216 static void nft_set_trans_unbind(const struct nft_ctx *ctx, struct nft_set *set)
 217 {
 218         return __nft_set_trans_bind(ctx, set, false);
 219 }
 220
 221 static void __nft_chain_trans_bind(const struct nft_ctx *ctx,
 222                                    struct nft_chain *chain, bool bind)
 223 {
 224         struct nftables_pernet *nft_net;
 225         struct net *net = ctx->net;
 226         struct nft_trans *trans;
 227
 228         if (!nft_chain_binding(chain))
 229                 return;
 230
 231         nft_net = nft_pernet(net);
 232         list_for_each_entry_reverse(trans, &nft_net->commit_list, list) {
 233                 switch (trans->msg_type) {
 234                 case NFT_MSG_NEWCHAIN:
 235                         if (nft_trans_chain(trans) == chain)
 236                                 nft_trans_chain_bound(trans) = bind;
 237                         break;
 238                 case NFT_MSG_NEWRULE:
 239                         if (trans->ctx.chain == chain)
 240                                 nft_trans_rule_bound(trans) = bind;
 241                         break;
 242                 }
 243         }
 244 }
 245
 246 static void nft_chain_trans_bind(const struct nft_ctx *ctx,
 247                                  struct nft_chain *chain)
 248 {
 249         __nft_chain_trans_bind(ctx, chain, true);
 250 }
 251
 252 int nf_tables_bind_chain(const struct nft_ctx *ctx, struct nft_chain *chain)
 253 {
 254         if (!nft_chain_binding(chain))
 255                 return 0;
 256
 257         if (nft_chain_binding(ctx->chain))
 258                 return -EOPNOTSUPP;
 259
 260         if (chain->bound)
 261                 return -EBUSY;
 262
 263         if (!nft_use_inc(&chain->use))
 264                 return -EMFILE;
 265
 266         chain->bound = true;
 267         nft_chain_trans_bind(ctx, chain);
 268
 269         return 0;
 270 }
 271
 272 void nf_tables_unbind_chain(const struct nft_ctx *ctx, struct nft_chain *chain)
 273 {
 274         __nft_chain_trans_bind(ctx, chain, false);
 275 }
 276
 277 static int nft_netdev_register_hooks(struct net *net,
 278                                      struct list_head *hook_list)
 279 {
 280         struct nft_hook *hook;
 281         int err, j;
 282
 283         j = 0;
 284         list_for_each_entry(hook, hook_list, list) {
 285                 err = nf_register_net_hook(net, &hook->ops);
 286                 if (err < 0)
 287                         goto err_register;
 288
 289                 j++;
 290         }
 291         return 0;
 292
 293 err_register:
 294         list_for_each_entry(hook, hook_list, list) {
 295                 if (j-- <= 0)
 296                         break;
 297
 298                 nf_unregister_net_hook(net, &hook->ops);
 299         }
 300         return err;
 301 }
 302
 303 static void nft_netdev_unregister_hooks(struct net *net,
 304                                         struct list_head *hook_list,
 305                                         bool release_netdev)
 306 {
 307         struct nft_hook *hook, *next;
 308
 309         list_for_each_entry_safe(hook, next, hook_list, list) {
 310                 nf_unregister_net_hook(net, &hook->ops);
 311                 if (release_netdev) {
 312                         list_del(&hook->list);
 313                         kfree_rcu(hook, rcu);
 314                 }
 315         }
 316 }
 317
 318 static int nf_tables_register_hook(struct net *net,
 319                                    const struct nft_table *table,
 320                                    struct nft_chain *chain)
 321 {
 322         struct nft_base_chain *basechain;
 323         const struct nf_hook_ops *ops;
 324
 325         if (table->flags & NFT_TABLE_F_DORMANT ||
 326             !nft_is_base_chain(chain))
 327                 return 0;
 328
 329         basechain = nft_base_chain(chain);
 330         ops = &basechain->ops;
 331
 332         if (basechain->type->ops_register)
 333                 return basechain->type->ops_register(net, ops);
 334
 335         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 336                 return nft_netdev_register_hooks(net, &basechain->hook_list);
 337
 338         return nf_register_net_hook(net, &basechain->ops);
 339 }
 340
 341 static void __nf_tables_unregister_hook(struct net *net,
 342                                         const struct nft_table *table,
 343                                         struct nft_chain *chain,
 344                                         bool release_netdev)
 345 {
 346         struct nft_base_chain *basechain;
 347         const struct nf_hook_ops *ops;
 348
 349         if (table->flags & NFT_TABLE_F_DORMANT ||
 350             !nft_is_base_chain(chain))
 351                 return;
 352         basechain = nft_base_chain(chain);
 353         ops = &basechain->ops;
 354
 355         if (basechain->type->ops_unregister)
 356                 return basechain->type->ops_unregister(net, ops);
 357
 358         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 359                 nft_netdev_unregister_hooks(net, &basechain->hook_list,
 360                                             release_netdev);
 361         else
 362                 nf_unregister_net_hook(net, &basechain->ops);
 363 }
 364
 365 static void nf_tables_unregister_hook(struct net *net,
 366                                       const struct nft_table *table,
 367                                       struct nft_chain *chain)
 368 {
 369         return __nf_tables_unregister_hook(net, table, chain, false);
 370 }
 371
 372 static void nft_trans_commit_list_add_tail(struct net *net, struct nft_trans *trans)
 373 {
 374         struct nftables_pernet *nft_net = nft_pernet(net);
 375
 376         switch (trans->msg_type) {
 377         case NFT_MSG_NEWSET:
 378                 if (!nft_trans_set_update(trans) &&
 379                     nft_set_is_anonymous(nft_trans_set(trans)))
 380                         list_add_tail(&trans->binding_list, &nft_net->binding_list);
 381                 break;
 382         case NFT_MSG_NEWCHAIN:
 383                 if (!nft_trans_chain_update(trans) &&
 384                     nft_chain_binding(nft_trans_chain(trans)))
 385                         list_add_tail(&trans->binding_list, &nft_net->binding_list);
 386                 break;
 387         }
 388
 389         list_add_tail(&trans->list, &nft_net->commit_list);
 390 }
 391
 392 static int nft_trans_table_add(struct nft_ctx *ctx, int msg_type)
 393 {
 394         struct nft_trans *trans;
 395
 396         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_table));
 397         if (trans == NULL)
 398                 return -ENOMEM;
 399
 400         if (msg_type == NFT_MSG_NEWTABLE)
 401                 nft_activate_next(ctx->net, ctx->table);
 402
 403         nft_trans_commit_list_add_tail(ctx->net, trans);
 404         return 0;
 405 }
 406
 407 static int nft_deltable(struct nft_ctx *ctx)
 408 {
 409         int err;
 410
 411         err = nft_trans_table_add(ctx, NFT_MSG_DELTABLE);
 412         if (err < 0)
 413                 return err;
 414
 415         nft_deactivate_next(ctx->net, ctx->table);
 416         return err;
 417 }
 418
 419 static struct nft_trans *nft_trans_chain_add(struct nft_ctx *ctx, int msg_type)
 420 {
 421         struct nft_trans *trans;
 422
 423         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_chain));
 424         if (trans == NULL)
 425                 return ERR_PTR(-ENOMEM);
 426
 427         if (msg_type == NFT_MSG_NEWCHAIN) {
 428                 nft_activate_next(ctx->net, ctx->chain);
 429
 430                 if (ctx->nla[NFTA_CHAIN_ID]) {
 431                         nft_trans_chain_id(trans) =
 432                                 ntohl(nla_get_be32(ctx->nla[NFTA_CHAIN_ID]));
 433                 }
 434         }
 435         nft_trans_chain(trans) = ctx->chain;
 436         nft_trans_commit_list_add_tail(ctx->net, trans);
 437
 438         return trans;
 439 }
 440
 441 static int nft_delchain(struct nft_ctx *ctx)
 442 {
 443         struct nft_trans *trans;
 444
 445         trans = nft_trans_chain_add(ctx, NFT_MSG_DELCHAIN);
 446         if (IS_ERR(trans))
 447                 return PTR_ERR(trans);
 448
 449         nft_use_dec(&ctx->table->use);
 450         nft_deactivate_next(ctx->net, ctx->chain);
 451
 452         return 0;
 453 }
 454
 455 void nft_rule_expr_activate(const struct nft_ctx *ctx, struct nft_rule *rule)
 456 {
 457         struct nft_expr *expr;
 458
 459         expr = nft_expr_first(rule);
 460         while (nft_expr_more(rule, expr)) {
 461                 if (expr->ops->activate)
 462                         expr->ops->activate(ctx, expr);
 463
 464                 expr = nft_expr_next(expr);
 465         }
 466 }
 467
 468 void nft_rule_expr_deactivate(const struct nft_ctx *ctx, struct nft_rule *rule,
 469                               enum nft_trans_phase phase)
 470 {
 471         struct nft_expr *expr;
 472
 473         expr = nft_expr_first(rule);
 474         while (nft_expr_more(rule, expr)) {
 475                 if (expr->ops->deactivate)
 476                         expr->ops->deactivate(ctx, expr, phase);
 477
 478                 expr = nft_expr_next(expr);
 479         }
 480 }
 481
 482 static int
 483 nf_tables_delrule_deactivate(struct nft_ctx *ctx, struct nft_rule *rule)
 484 {
 485         /* You cannot delete the same rule twice */
 486         if (nft_is_active_next(ctx->net, rule)) {
 487                 nft_deactivate_next(ctx->net, rule);
 488                 nft_use_dec(&ctx->chain->use);
 489                 return 0;
 490         }
 491         return -ENOENT;
 492 }
 493
 494 static struct nft_trans *nft_trans_rule_add(struct nft_ctx *ctx, int msg_type,
 495                                             struct nft_rule *rule)
 496 {
 497         struct nft_trans *trans;
 498
 499         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_rule));
 500         if (trans == NULL)
 501                 return NULL;
 502
 503         if (msg_type == NFT_MSG_NEWRULE && ctx->nla[NFTA_RULE_ID] != NULL) {
 504                 nft_trans_rule_id(trans) =
 505                         ntohl(nla_get_be32(ctx->nla[NFTA_RULE_ID]));
 506         }
 507         nft_trans_rule(trans) = rule;
 508         nft_trans_commit_list_add_tail(ctx->net, trans);
 509
 510         return trans;
 511 }
 512
 513 static int nft_delrule(struct nft_ctx *ctx, struct nft_rule *rule)
 514 {
 515         struct nft_flow_rule *flow;
 516         struct nft_trans *trans;
 517         int err;
 518
 519         trans = nft_trans_rule_add(ctx, NFT_MSG_DELRULE, rule);
 520         if (trans == NULL)
 521                 return -ENOMEM;
 522
 523         if (ctx->chain->flags & NFT_CHAIN_HW_OFFLOAD) {
 524                 flow = nft_flow_rule_create(ctx->net, rule);
 525                 if (IS_ERR(flow)) {
 526                         nft_trans_destroy(trans);
 527                         return PTR_ERR(flow);
 528                 }
 529
 530                 nft_trans_flow_rule(trans) = flow;
 531         }
 532
 533         err = nf_tables_delrule_deactivate(ctx, rule);
 534         if (err < 0) {
 535                 nft_trans_destroy(trans);
 536                 return err;
 537         }
 538         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_PREPARE);
 539
 540         return 0;
 541 }
 542
 543 static int nft_delrule_by_chain(struct nft_ctx *ctx)
 544 {
 545         struct nft_rule *rule;
 546         int err;
 547
 548         list_for_each_entry(rule, &ctx->chain->rules, list) {
 549                 if (!nft_is_active_next(ctx->net, rule))
 550                         continue;
 551
 552                 err = nft_delrule(ctx, rule);
 553                 if (err < 0)
 554                         return err;
 555         }
 556         return 0;
 557 }
 558
 559 static int __nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
 560                                struct nft_set *set,
 561                                const struct nft_set_desc *desc)
 562 {
 563         struct nft_trans *trans;
 564
 565         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_set));
 566         if (trans == NULL)
 567                 return -ENOMEM;
 568
 569         if (msg_type == NFT_MSG_NEWSET && ctx->nla[NFTA_SET_ID] && !desc) {
 570                 nft_trans_set_id(trans) =
 571                         ntohl(nla_get_be32(ctx->nla[NFTA_SET_ID]));
 572                 nft_activate_next(ctx->net, set);
 573         }
 574         nft_trans_set(trans) = set;
 575         if (desc) {
 576                 nft_trans_set_update(trans) = true;
 577                 nft_trans_set_gc_int(trans) = desc->gc_int;
 578                 nft_trans_set_timeout(trans) = desc->timeout;
 579                 nft_trans_set_size(trans) = desc->size;
 580         }
 581         nft_trans_commit_list_add_tail(ctx->net, trans);
 582
 583         return 0;
 584 }
 585
 586 static int nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
 587                              struct nft_set *set)
 588 {
 589         return __nft_trans_set_add(ctx, msg_type, set, NULL);
 590 }
 591
 592 static int nft_mapelem_deactivate(const struct nft_ctx *ctx,
 593                                   struct nft_set *set,
 594                                   const struct nft_set_iter *iter,
 595                                   struct nft_set_elem *elem)
 596 {
 597         nft_setelem_data_deactivate(ctx->net, set, elem);
 598
 599         return 0;
 600 }
 601
 602 struct nft_set_elem_catchall {
 603         struct list_head        list;
 604         struct rcu_head         rcu;
 605         void                    *elem;
 606 };
 607
 608 static void nft_map_catchall_deactivate(const struct nft_ctx *ctx,
 609                                         struct nft_set *set)
 610 {
 611         u8 genmask = nft_genmask_next(ctx->net);
 612         struct nft_set_elem_catchall *catchall;
 613         struct nft_set_elem elem;
 614         struct nft_set_ext *ext;
 615
 616         list_for_each_entry(catchall, &set->catchall_list, list) {
 617                 ext = nft_set_elem_ext(set, catchall->elem);
 618                 if (!nft_set_elem_active(ext, genmask))
 619                         continue;
 620
 621                 elem.priv = catchall->elem;
 622                 nft_setelem_data_deactivate(ctx->net, set, &elem);
 623                 break;
 624         }
 625 }
 626
 627 static void nft_map_deactivate(const struct nft_ctx *ctx, struct nft_set *set)
 628 {
 629         struct nft_set_iter iter = {
 630                 .genmask        = nft_genmask_next(ctx->net),
 631                 .fn             = nft_mapelem_deactivate,
 632         };
 633
 634         set->ops->walk(ctx, set, &iter);
 635         WARN_ON_ONCE(iter.err);
 636
 637         nft_map_catchall_deactivate(ctx, set);
 638 }
 639
 640 static int nft_delset(const struct nft_ctx *ctx, struct nft_set *set)
 641 {
 642         int err;
 643
 644         err = nft_trans_set_add(ctx, NFT_MSG_DELSET, set);
 645         if (err < 0)
 646                 return err;
 647
 648         if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
 649                 nft_map_deactivate(ctx, set);
 650
 651         nft_deactivate_next(ctx->net, set);
 652         nft_use_dec(&ctx->table->use);
 653
 654         return err;
 655 }
 656
 657 static int nft_trans_obj_add(struct nft_ctx *ctx, int msg_type,
 658                              struct nft_object *obj)
 659 {
 660         struct nft_trans *trans;
 661
 662         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_obj));
 663         if (trans == NULL)
 664                 return -ENOMEM;
 665
 666         if (msg_type == NFT_MSG_NEWOBJ)
 667                 nft_activate_next(ctx->net, obj);
 668
 669         nft_trans_obj(trans) = obj;
 670         nft_trans_commit_list_add_tail(ctx->net, trans);
 671
 672         return 0;
 673 }
 674
 675 static int nft_delobj(struct nft_ctx *ctx, struct nft_object *obj)
 676 {
 677         int err;
 678
 679         err = nft_trans_obj_add(ctx, NFT_MSG_DELOBJ, obj);
 680         if (err < 0)
 681                 return err;
 682
 683         nft_deactivate_next(ctx->net, obj);
 684         nft_use_dec(&ctx->table->use);
 685
 686         return err;
 687 }
 688
 689 static int nft_trans_flowtable_add(struct nft_ctx *ctx, int msg_type,
 690                                    struct nft_flowtable *flowtable)
 691 {
 692         struct nft_trans *trans;
 693
 694         trans = nft_trans_alloc(ctx, msg_type,
 695                                 sizeof(struct nft_trans_flowtable));
 696         if (trans == NULL)
 697                 return -ENOMEM;
 698
 699         if (msg_type == NFT_MSG_NEWFLOWTABLE)
 700                 nft_activate_next(ctx->net, flowtable);
 701
 702         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
 703         nft_trans_flowtable(trans) = flowtable;
 704         nft_trans_commit_list_add_tail(ctx->net, trans);
 705
 706         return 0;
 707 }
 708
 709 static int nft_delflowtable(struct nft_ctx *ctx,
 710                             struct nft_flowtable *flowtable)
 711 {
 712         int err;
 713
 714         err = nft_trans_flowtable_add(ctx, NFT_MSG_DELFLOWTABLE, flowtable);
 715         if (err < 0)
 716                 return err;
 717
 718         nft_deactivate_next(ctx->net, flowtable);
 719         nft_use_dec(&ctx->table->use);
 720
 721         return err;
 722 }
 723
 724 static void __nft_reg_track_clobber(struct nft_regs_track *track, u8 dreg)
 725 {
 726         int i;
 727
 728         for (i = track->regs[dreg].num_reg; i > 0; i--)
 729                 __nft_reg_track_cancel(track, dreg - i);
 730 }
 731
 732 static void __nft_reg_track_update(struct nft_regs_track *track,
 733                                    const struct nft_expr *expr,
 734                                    u8 dreg, u8 num_reg)
 735 {
 736         track->regs[dreg].selector = expr;
 737         track->regs[dreg].bitwise = NULL;
 738         track->regs[dreg].num_reg = num_reg;
 739 }
 740
 741 void nft_reg_track_update(struct nft_regs_track *track,
 742                           const struct nft_expr *expr, u8 dreg, u8 len)
 743 {
 744         unsigned int regcount;
 745         int i;
 746
 747         __nft_reg_track_clobber(track, dreg);
 748
 749         regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
 750         for (i = 0; i < regcount; i++, dreg++)
 751                 __nft_reg_track_update(track, expr, dreg, i);
 752 }
 753 EXPORT_SYMBOL_GPL(nft_reg_track_update);
 754
 755 void nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg, u8 len)
 756 {
 757         unsigned int regcount;
 758         int i;
 759
 760         __nft_reg_track_clobber(track, dreg);
 761
 762         regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
 763         for (i = 0; i < regcount; i++, dreg++)
 764                 __nft_reg_track_cancel(track, dreg);
 765 }
 766 EXPORT_SYMBOL_GPL(nft_reg_track_cancel);
 767
 768 void __nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg)
 769 {
 770         track->regs[dreg].selector = NULL;
 771         track->regs[dreg].bitwise = NULL;
 772         track->regs[dreg].num_reg = 0;
 773 }
 774 EXPORT_SYMBOL_GPL(__nft_reg_track_cancel);
 775
 776 /*
 777  * Tables
 778  */
 779
 780 static struct nft_table *nft_table_lookup(const struct net *net,
 781                                           const struct nlattr *nla,
 782                                           u8 family, u8 genmask, u32 nlpid)
 783 {
 784         struct nftables_pernet *nft_net;
 785         struct nft_table *table;
 786
 787         if (nla == NULL)
 788                 return ERR_PTR(-EINVAL);
 789
 790         nft_net = nft_pernet(net);
 791         list_for_each_entry_rcu(table, &nft_net->tables, list,
 792                                 lockdep_is_held(&nft_net->commit_mutex)) {
 793                 if (!nla_strcmp(nla, table->name) &&
 794                     table->family == family &&
 795                     nft_active_genmask(table, genmask)) {
 796                         if (nft_table_has_owner(table) &&
 797                             nlpid && table->nlpid != nlpid)
 798                                 return ERR_PTR(-EPERM);
 799
 800                         return table;
 801                 }
 802         }
 803
 804         return ERR_PTR(-ENOENT);
 805 }
 806
 807 static struct nft_table *nft_table_lookup_byhandle(const struct net *net,
 808                                                    const struct nlattr *nla,
 809                                                    int family, u8 genmask, u32 nlpid)
 810 {
 811         struct nftables_pernet *nft_net;
 812         struct nft_table *table;
 813
 814         nft_net = nft_pernet(net);
 815         list_for_each_entry(table, &nft_net->tables, list) {
 816                 if (be64_to_cpu(nla_get_be64(nla)) == table->handle &&
 817                     table->family == family &&
 818                     nft_active_genmask(table, genmask)) {
 819                         if (nft_table_has_owner(table) &&
 820                             nlpid && table->nlpid != nlpid)
 821                                 return ERR_PTR(-EPERM);
 822
 823                         return table;
 824                 }
 825         }
 826
 827         return ERR_PTR(-ENOENT);
 828 }
 829
 830 static inline u64 nf_tables_alloc_handle(struct nft_table *table)
 831 {
 832         return ++table->hgenerator;
 833 }
 834
 835 static const struct nft_chain_type *chain_type[NFPROTO_NUMPROTO][NFT_CHAIN_T_MAX];
 836
 837 static const struct nft_chain_type *
 838 __nft_chain_type_get(u8 family, enum nft_chain_types type)
 839 {
 840         if (family >= NFPROTO_NUMPROTO ||
 841             type >= NFT_CHAIN_T_MAX)
 842                 return NULL;
 843
 844         return chain_type[family][type];
 845 }
 846
 847 static const struct nft_chain_type *
 848 __nf_tables_chain_type_lookup(const struct nlattr *nla, u8 family)
 849 {
 850         const struct nft_chain_type *type;
 851         int i;
 852
 853         for (i = 0; i < NFT_CHAIN_T_MAX; i++) {
 854                 type = __nft_chain_type_get(family, i);
 855                 if (!type)
 856                         continue;
 857                 if (!nla_strcmp(nla, type->name))
 858                         return type;
 859         }
 860         return NULL;
 861 }
 862
 863 struct nft_module_request {
 864         struct list_head        list;
 865         char                    module[MODULE_NAME_LEN];
 866         bool                    done;
 867 };
 868
 869 #ifdef CONFIG_MODULES
 870 __printf(2, 3) int nft_request_module(struct net *net, const char *fmt,
 871                                       ...)
 872 {
 873         char module_name[MODULE_NAME_LEN];
 874         struct nftables_pernet *nft_net;
 875         struct nft_module_request *req;
 876         va_list args;
 877         int ret;
 878
 879         va_start(args, fmt);
 880         ret = vsnprintf(module_name, MODULE_NAME_LEN, fmt, args);
 881         va_end(args);
 882         if (ret >= MODULE_NAME_LEN)
 883                 return 0;
 884
 885         nft_net = nft_pernet(net);
 886         list_for_each_entry(req, &nft_net->module_list, list) {
 887                 if (!strcmp(req->module, module_name)) {
 888                         if (req->done)
 889                                 return 0;
 890
 891                         /* A request to load this module already exists. */
 892                         return -EAGAIN;
 893                 }
 894         }
 895
 896         req = kmalloc(sizeof(*req), GFP_KERNEL);
 897         if (!req)
 898                 return -ENOMEM;
 899
 900         req->done = false;
 901         strscpy(req->module, module_name, MODULE_NAME_LEN);
 902         list_add_tail(&req->list, &nft_net->module_list);
 903
 904         return -EAGAIN;
 905 }
 906 EXPORT_SYMBOL_GPL(nft_request_module);
 907 #endif
 908
 909 static void lockdep_nfnl_nft_mutex_not_held(void)
 910 {
 911 #ifdef CONFIG_PROVE_LOCKING
 912         if (debug_locks)
 913                 WARN_ON_ONCE(lockdep_nfnl_is_held(NFNL_SUBSYS_NFTABLES));
 914 #endif
 915 }
 916
 917 static const struct nft_chain_type *
 918 nf_tables_chain_type_lookup(struct net *net, const struct nlattr *nla,
 919                             u8 family, bool autoload)
 920 {
 921         const struct nft_chain_type *type;
 922
 923         type = __nf_tables_chain_type_lookup(nla, family);
 924         if (type != NULL)
 925                 return type;
 926
 927         lockdep_nfnl_nft_mutex_not_held();
 928 #ifdef CONFIG_MODULES
 929         if (autoload) {
 930                 if (nft_request_module(net, "nft-chain-%u-%.*s", family,
 931                                        nla_len(nla),
 932                                        (const char *)nla_data(nla)) == -EAGAIN)
 933                         return ERR_PTR(-EAGAIN);
 934         }
 935 #endif
 936         return ERR_PTR(-ENOENT);
 937 }
 938
 939 static __be16 nft_base_seq(const struct net *net)
 940 {
 941         struct nftables_pernet *nft_net = nft_pernet(net);
 942
 943         return htons(nft_net->base_seq & 0xffff);
 944 }
 945
 946 static const struct nla_policy nft_table_policy[NFTA_TABLE_MAX + 1] = {
 947         [NFTA_TABLE_NAME]       = { .type = NLA_STRING,
 948                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
 949         [NFTA_TABLE_FLAGS]      = { .type = NLA_U32 },
 950         [NFTA_TABLE_HANDLE]     = { .type = NLA_U64 },
 951         [NFTA_TABLE_USERDATA]   = { .type = NLA_BINARY,
 952                                     .len = NFT_USERDATA_MAXLEN }
 953 };
 954
 955 static int nf_tables_fill_table_info(struct sk_buff *skb, struct net *net,
 956                                      u32 portid, u32 seq, int event, u32 flags,
 957                                      int family, const struct nft_table *table)
 958 {
 959         struct nlmsghdr *nlh;
 960
 961         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
 962         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
 963                            NFNETLINK_V0, nft_base_seq(net));
 964         if (!nlh)
 965                 goto nla_put_failure;
 966
 967         if (nla_put_string(skb, NFTA_TABLE_NAME, table->name) ||
 968             nla_put_be32(skb, NFTA_TABLE_USE, htonl(table->use)) ||
 969             nla_put_be64(skb, NFTA_TABLE_HANDLE, cpu_to_be64(table->handle),
 970                          NFTA_TABLE_PAD))
 971                 goto nla_put_failure;
 972
 973         if (event == NFT_MSG_DELTABLE) {
 974                 nlmsg_end(skb, nlh);
 975                 return 0;
 976         }
 977
 978         if (nla_put_be32(skb, NFTA_TABLE_FLAGS,
 979                          htonl(table->flags & NFT_TABLE_F_MASK)))
 980                 goto nla_put_failure;
 981
 982         if (nft_table_has_owner(table) &&
 983             nla_put_be32(skb, NFTA_TABLE_OWNER, htonl(table->nlpid)))
 984                 goto nla_put_failure;
 985
 986         if (table->udata) {
 987                 if (nla_put(skb, NFTA_TABLE_USERDATA, table->udlen, table->udata))
 988                         goto nla_put_failure;
 989         }
 990
 991         nlmsg_end(skb, nlh);
 992         return 0;
 993
 994 nla_put_failure:
 995         nlmsg_trim(skb, nlh);
 996         return -1;
 997 }
 998
 999 struct nftnl_skb_parms {
1000         bool report;
1001 };
1002 #define NFT_CB(skb)     (*(struct nftnl_skb_parms*)&((skb)->cb))
1003
1004 static void nft_notify_enqueue(struct sk_buff *skb, bool report,
1005                                struct list_head *notify_list)
1006 {
1007         NFT_CB(skb).report = report;
1008         list_add_tail(&skb->list, notify_list);
1009 }
1010
1011 static void nf_tables_table_notify(const struct nft_ctx *ctx, int event)
1012 {
1013         struct nftables_pernet *nft_net;
1014         struct sk_buff *skb;
1015         u16 flags = 0;
1016         int err;
1017
1018         if (!ctx->report &&
1019             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
1020                 return;
1021
1022         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
1023         if (skb == NULL)
1024                 goto err;
1025
1026         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
1027                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
1028
1029         err = nf_tables_fill_table_info(skb, ctx->net, ctx->portid, ctx->seq,
1030                                         event, flags, ctx->family, ctx->table);
1031         if (err < 0) {
1032                 kfree_skb(skb);
1033                 goto err;
1034         }
1035
1036         nft_net = nft_pernet(ctx->net);
1037         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
1038         return;
1039 err:
1040         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
1041 }
1042
1043 static int nf_tables_dump_tables(struct sk_buff *skb,
1044                                  struct netlink_callback *cb)
1045 {
1046         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
1047         struct nftables_pernet *nft_net;
1048         const struct nft_table *table;
1049         unsigned int idx = 0, s_idx = cb->args[0];
1050         struct net *net = sock_net(skb->sk);
1051         int family = nfmsg->nfgen_family;
1052
1053         rcu_read_lock();
1054         nft_net = nft_pernet(net);
1055         cb->seq = READ_ONCE(nft_net->base_seq);
1056
1057         list_for_each_entry_rcu(table, &nft_net->tables, list) {
1058                 if (family != NFPROTO_UNSPEC && family != table->family)
1059                         continue;
1060
1061                 if (idx < s_idx)
1062                         goto cont;
1063                 if (idx > s_idx)
1064                         memset(&cb->args[1], 0,
1065                                sizeof(cb->args) - sizeof(cb->args[0]));
1066                 if (!nft_is_active(net, table))
1067                         continue;
1068                 if (nf_tables_fill_table_info(skb, net,
1069                                               NETLINK_CB(cb->skb).portid,
1070                                               cb->nlh->nlmsg_seq,
1071                                               NFT_MSG_NEWTABLE, NLM_F_MULTI,
1072                                               table->family, table) < 0)
1073                         goto done;
1074
1075                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
1076 cont:
1077                 idx++;
1078         }
1079 done:
1080         rcu_read_unlock();
1081         cb->args[0] = idx;
1082         return skb->len;
1083 }
1084
1085 static int nft_netlink_dump_start_rcu(struct sock *nlsk, struct sk_buff *skb,
1086                                       const struct nlmsghdr *nlh,
1087                                       struct netlink_dump_control *c)
1088 {
1089         int err;
1090
1091         if (!try_module_get(THIS_MODULE))
1092                 return -EINVAL;
1093
1094         rcu_read_unlock();
1095         err = netlink_dump_start(nlsk, skb, nlh, c);
1096         rcu_read_lock();
1097         module_put(THIS_MODULE);
1098
1099         return err;
1100 }
1101
1102 /* called with rcu_read_lock held */
1103 static int nf_tables_gettable(struct sk_buff *skb, const struct nfnl_info *info,
1104                               const struct nlattr * const nla[])
1105 {
1106         struct netlink_ext_ack *extack = info->extack;
1107         u8 genmask = nft_genmask_cur(info->net);
1108         u8 family = info->nfmsg->nfgen_family;
1109         const struct nft_table *table;
1110         struct net *net = info->net;
1111         struct sk_buff *skb2;
1112         int err;
1113
1114         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
1115                 struct netlink_dump_control c = {
1116                         .dump = nf_tables_dump_tables,
1117                         .module = THIS_MODULE,
1118                 };
1119
1120                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
1121         }
1122
1123         table = nft_table_lookup(net, nla[NFTA_TABLE_NAME], family, genmask, 0);
1124         if (IS_ERR(table)) {
1125                 NL_SET_BAD_ATTR(extack, nla[NFTA_TABLE_NAME]);
1126                 return PTR_ERR(table);
1127         }
1128
1129         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
1130         if (!skb2)
1131                 return -ENOMEM;
1132
1133         err = nf_tables_fill_table_info(skb2, net, NETLINK_CB(skb).portid,
1134                                         info->nlh->nlmsg_seq, NFT_MSG_NEWTABLE,
1135                                         0, family, table);
1136         if (err < 0)
1137                 goto err_fill_table_info;
1138
1139         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
1140
1141 err_fill_table_info:
1142         kfree_skb(skb2);
1143         return err;
1144 }
1145
1146 static void nft_table_disable(struct net *net, struct nft_table *table, u32 cnt)
1147 {
1148         struct nft_chain *chain;
1149         u32 i = 0;
1150
1151         list_for_each_entry(chain, &table->chains, list) {
1152                 if (!nft_is_active_next(net, chain))
1153                         continue;
1154                 if (!nft_is_base_chain(chain))
1155                         continue;
1156
1157                 if (cnt && i++ == cnt)
1158                         break;
1159
1160                 nf_tables_unregister_hook(net, table, chain);
1161         }
1162 }
1163
1164 static int nf_tables_table_enable(struct net *net, struct nft_table *table)
1165 {
1166         struct nft_chain *chain;
1167         int err, i = 0;
1168
1169         list_for_each_entry(chain, &table->chains, list) {
1170                 if (!nft_is_active_next(net, chain))
1171                         continue;
1172                 if (!nft_is_base_chain(chain))
1173                         continue;
1174
1175                 err = nf_tables_register_hook(net, table, chain);
1176                 if (err < 0)
1177                         goto err_register_hooks;
1178
1179                 i++;
1180         }
1181         return 0;
1182
1183 err_register_hooks:
1184         if (i)
1185                 nft_table_disable(net, table, i);
1186         return err;
1187 }
1188
1189 static void nf_tables_table_disable(struct net *net, struct nft_table *table)
1190 {
1191         table->flags &= ~NFT_TABLE_F_DORMANT;
1192         nft_table_disable(net, table, 0);
1193         table->flags |= NFT_TABLE_F_DORMANT;
1194 }
1195
1196 #define __NFT_TABLE_F_INTERNAL          (NFT_TABLE_F_MASK + 1)
1197 #define __NFT_TABLE_F_WAS_DORMANT       (__NFT_TABLE_F_INTERNAL << 0)
1198 #define __NFT_TABLE_F_WAS_AWAKEN        (__NFT_TABLE_F_INTERNAL << 1)
1199 #define __NFT_TABLE_F_UPDATE            (__NFT_TABLE_F_WAS_DORMANT | \
1200                                          __NFT_TABLE_F_WAS_AWAKEN)
1201
1202 static int nf_tables_updtable(struct nft_ctx *ctx)
1203 {
1204         struct nft_trans *trans;
1205         u32 flags;
1206         int ret;
1207
1208         if (!ctx->nla[NFTA_TABLE_FLAGS])
1209                 return 0;
1210
1211         flags = ntohl(nla_get_be32(ctx->nla[NFTA_TABLE_FLAGS]));
1212         if (flags & ~NFT_TABLE_F_MASK)
1213                 return -EOPNOTSUPP;
1214
1215         if (flags == ctx->table->flags)
1216                 return 0;
1217
1218         if ((nft_table_has_owner(ctx->table) &&
1219              !(flags & NFT_TABLE_F_OWNER)) ||
1220             (!nft_table_has_owner(ctx->table) &&
1221              flags & NFT_TABLE_F_OWNER))
1222                 return -EOPNOTSUPP;
1223
1224         /* No dormant off/on/off/on games in single transaction */
1225         if (ctx->table->flags & __NFT_TABLE_F_UPDATE)
1226                 return -EINVAL;
1227
1228         trans = nft_trans_alloc(ctx, NFT_MSG_NEWTABLE,
1229                                 sizeof(struct nft_trans_table));
1230         if (trans == NULL)
1231                 return -ENOMEM;
1232
1233         if ((flags & NFT_TABLE_F_DORMANT) &&
1234             !(ctx->table->flags & NFT_TABLE_F_DORMANT)) {
1235                 ctx->table->flags |= NFT_TABLE_F_DORMANT;
1236                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE))
1237                         ctx->table->flags |= __NFT_TABLE_F_WAS_AWAKEN;
1238         } else if (!(flags & NFT_TABLE_F_DORMANT) &&
1239                    ctx->table->flags & NFT_TABLE_F_DORMANT) {
1240                 ctx->table->flags &= ~NFT_TABLE_F_DORMANT;
1241                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE)) {
1242                         ret = nf_tables_table_enable(ctx->net, ctx->table);
1243                         if (ret < 0)
1244                                 goto err_register_hooks;
1245
1246                         ctx->table->flags |= __NFT_TABLE_F_WAS_DORMANT;
1247                 }
1248         }
1249
1250         nft_trans_table_update(trans) = true;
1251         nft_trans_commit_list_add_tail(ctx->net, trans);
1252
1253         return 0;
1254
1255 err_register_hooks:
1256         nft_trans_destroy(trans);
1257         return ret;
1258 }
1259
1260 static u32 nft_chain_hash(const void *data, u32 len, u32 seed)
1261 {
1262         const char *name = data;
1263
1264         return jhash(name, strlen(name), seed);
1265 }
1266
1267 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed)
1268 {
1269         const struct nft_chain *chain = data;
1270
1271         return nft_chain_hash(chain->name, 0, seed);
1272 }
1273
1274 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *arg,
1275                               const void *ptr)
1276 {
1277         const struct nft_chain *chain = ptr;
1278         const char *name = arg->key;
1279
1280         return strcmp(chain->name, name);
1281 }
1282
1283 static u32 nft_objname_hash(const void *data, u32 len, u32 seed)
1284 {
1285         const struct nft_object_hash_key *k = data;
1286
1287         seed ^= hash_ptr(k->table, 32);
1288
1289         return jhash(k->name, strlen(k->name), seed);
1290 }
1291
1292 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed)
1293 {
1294         const struct nft_object *obj = data;
1295
1296         return nft_objname_hash(&obj->key, 0, seed);
1297 }
1298
1299 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *arg,
1300                                 const void *ptr)
1301 {
1302         const struct nft_object_hash_key *k = arg->key;
1303         const struct nft_object *obj = ptr;
1304
1305         if (obj->key.table != k->table)
1306                 return -1;
1307
1308         return strcmp(obj->key.name, k->name);
1309 }
1310
1311 static bool nft_supported_family(u8 family)
1312 {
1313         return false
1314 #ifdef CONFIG_NF_TABLES_INET
1315                 || family == NFPROTO_INET
1316 #endif
1317 #ifdef CONFIG_NF_TABLES_IPV4
1318                 || family == NFPROTO_IPV4
1319 #endif
1320 #ifdef CONFIG_NF_TABLES_ARP
1321                 || family == NFPROTO_ARP
1322 #endif
1323 #ifdef CONFIG_NF_TABLES_NETDEV
1324                 || family == NFPROTO_NETDEV
1325 #endif
1326 #if IS_ENABLED(CONFIG_NF_TABLES_BRIDGE)
1327                 || family == NFPROTO_BRIDGE
1328 #endif
1329 #ifdef CONFIG_NF_TABLES_IPV6
1330                 || family == NFPROTO_IPV6
1331 #endif
1332                 ;
1333 }
1334
1335 static int nf_tables_newtable(struct sk_buff *skb, const struct nfnl_info *info,
1336                               const struct nlattr * const nla[])
1337 {
1338         struct nftables_pernet *nft_net = nft_pernet(info->net);
1339         struct netlink_ext_ack *extack = info->extack;
1340         u8 genmask = nft_genmask_next(info->net);
1341         u8 family = info->nfmsg->nfgen_family;
1342         struct net *net = info->net;
1343         const struct nlattr *attr;
1344         struct nft_table *table;
1345         struct nft_ctx ctx;
1346         u32 flags = 0;
1347         int err;
1348
1349         if (!nft_supported_family(family))
1350                 return -EOPNOTSUPP;
1351
1352         lockdep_assert_held(&nft_net->commit_mutex);
1353         attr = nla[NFTA_TABLE_NAME];
1354         table = nft_table_lookup(net, attr, family, genmask,
1355                                  NETLINK_CB(skb).portid);
1356         if (IS_ERR(table)) {
1357                 if (PTR_ERR(table) != -ENOENT)
1358                         return PTR_ERR(table);
1359         } else {
1360                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
1361                         NL_SET_BAD_ATTR(extack, attr);
1362                         return -EEXIST;
1363                 }
1364                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
1365                         return -EOPNOTSUPP;
1366
1367                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1368
1369                 return nf_tables_updtable(&ctx);
1370         }
1371
1372         if (nla[NFTA_TABLE_FLAGS]) {
1373                 flags = ntohl(nla_get_be32(nla[NFTA_TABLE_FLAGS]));
1374                 if (flags & ~NFT_TABLE_F_MASK)
1375                         return -EOPNOTSUPP;
1376         }
1377
1378         err = -ENOMEM;
1379         table = kzalloc(sizeof(*table), GFP_KERNEL_ACCOUNT);
1380         if (table == NULL)
1381                 goto err_kzalloc;
1382
1383         table->validate_state = nft_net->validate_state;
1384         table->name = nla_strdup(attr, GFP_KERNEL_ACCOUNT);
1385         if (table->name == NULL)
1386                 goto err_strdup;
1387
1388         if (nla[NFTA_TABLE_USERDATA]) {
1389                 table->udata = nla_memdup(nla[NFTA_TABLE_USERDATA], GFP_KERNEL_ACCOUNT);
1390                 if (table->udata == NULL)
1391                         goto err_table_udata;
1392
1393                 table->udlen = nla_len(nla[NFTA_TABLE_USERDATA]);
1394         }
1395
1396         err = rhltable_init(&table->chains_ht, &nft_chain_ht_params);
1397         if (err)
1398                 goto err_chain_ht;
1399
1400         INIT_LIST_HEAD(&table->chains);
1401         INIT_LIST_HEAD(&table->sets);
1402         INIT_LIST_HEAD(&table->objects);
1403         INIT_LIST_HEAD(&table->flowtables);
1404         table->family = family;
1405         table->flags = flags;
1406         table->handle = ++nft_net->table_handle;
1407         if (table->flags & NFT_TABLE_F_OWNER)
1408                 table->nlpid = NETLINK_CB(skb).portid;
1409
1410         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1411         err = nft_trans_table_add(&ctx, NFT_MSG_NEWTABLE);
1412         if (err < 0)
1413                 goto err_trans;
1414
1415         list_add_tail_rcu(&table->list, &nft_net->tables);
1416         return 0;
1417 err_trans:
1418         rhltable_destroy(&table->chains_ht);
1419 err_chain_ht:
1420         kfree(table->udata);
1421 err_table_udata:
1422         kfree(table->name);
1423 err_strdup:
1424         kfree(table);
1425 err_kzalloc:
1426         return err;
1427 }
1428
1429 static int nft_flush_table(struct nft_ctx *ctx)
1430 {
1431         struct nft_flowtable *flowtable, *nft;
1432         struct nft_chain *chain, *nc;
1433         struct nft_object *obj, *ne;
1434         struct nft_set *set, *ns;
1435         int err;
1436
1437         list_for_each_entry(chain, &ctx->table->chains, list) {
1438                 if (!nft_is_active_next(ctx->net, chain))
1439                         continue;
1440
1441                 if (nft_chain_binding(chain))
1442                         continue;
1443
1444                 ctx->chain = chain;
1445
1446                 err = nft_delrule_by_chain(ctx);
1447                 if (err < 0)
1448                         goto out;
1449         }
1450
1451         list_for_each_entry_safe(set, ns, &ctx->table->sets, list) {
1452                 if (!nft_is_active_next(ctx->net, set))
1453                         continue;
1454
1455                 if (nft_set_is_anonymous(set))
1456                         continue;
1457
1458                 err = nft_delset(ctx, set);
1459                 if (err < 0)
1460                         goto out;
1461         }
1462
1463         list_for_each_entry_safe(flowtable, nft, &ctx->table->flowtables, list) {
1464                 if (!nft_is_active_next(ctx->net, flowtable))
1465                         continue;
1466
1467                 err = nft_delflowtable(ctx, flowtable);
1468                 if (err < 0)
1469                         goto out;
1470         }
1471
1472         list_for_each_entry_safe(obj, ne, &ctx->table->objects, list) {
1473                 if (!nft_is_active_next(ctx->net, obj))
1474                         continue;
1475
1476                 err = nft_delobj(ctx, obj);
1477                 if (err < 0)
1478                         goto out;
1479         }
1480
1481         list_for_each_entry_safe(chain, nc, &ctx->table->chains, list) {
1482                 if (!nft_is_active_next(ctx->net, chain))
1483                         continue;
1484
1485                 if (nft_chain_binding(chain))
1486                         continue;
1487
1488                 ctx->chain = chain;
1489
1490                 err = nft_delchain(ctx);
1491                 if (err < 0)
1492                         goto out;
1493         }
1494
1495         err = nft_deltable(ctx);
1496 out:
1497         return err;
1498 }
1499
1500 static int nft_flush(struct nft_ctx *ctx, int family)
1501 {
1502         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
1503         const struct nlattr * const *nla = ctx->nla;
1504         struct nft_table *table, *nt;
1505         int err = 0;
1506
1507         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
1508                 if (family != AF_UNSPEC && table->family != family)
1509                         continue;
1510
1511                 ctx->family = table->family;
1512
1513                 if (!nft_is_active_next(ctx->net, table))
1514                         continue;
1515
1516                 if (nft_table_has_owner(table) && table->nlpid != ctx->portid)
1517                         continue;
1518
1519                 if (nla[NFTA_TABLE_NAME] &&
1520                     nla_strcmp(nla[NFTA_TABLE_NAME], table->name) != 0)
1521                         continue;
1522
1523                 ctx->table = table;
1524
1525                 err = nft_flush_table(ctx);
1526                 if (err < 0)
1527                         goto out;
1528         }
1529 out:
1530         return err;
1531 }
1532
1533 static int nf_tables_deltable(struct sk_buff *skb, const struct nfnl_info *info,
1534                               const struct nlattr * const nla[])
1535 {
1536         struct netlink_ext_ack *extack = info->extack;
1537         u8 genmask = nft_genmask_next(info->net);
1538         u8 family = info->nfmsg->nfgen_family;
1539         struct net *net = info->net;
1540         const struct nlattr *attr;
1541         struct nft_table *table;
1542         struct nft_ctx ctx;
1543
1544         nft_ctx_init(&ctx, net, skb, info->nlh, 0, NULL, NULL, nla);
1545         if (family == AF_UNSPEC ||
1546             (!nla[NFTA_TABLE_NAME] && !nla[NFTA_TABLE_HANDLE]))
1547                 return nft_flush(&ctx, family);
1548
1549         if (nla[NFTA_TABLE_HANDLE]) {
1550                 attr = nla[NFTA_TABLE_HANDLE];
1551                 table = nft_table_lookup_byhandle(net, attr, family, genmask,
1552                                                   NETLINK_CB(skb).portid);
1553         } else {
1554                 attr = nla[NFTA_TABLE_NAME];
1555                 table = nft_table_lookup(net, attr, family, genmask,
1556                                          NETLINK_CB(skb).portid);
1557         }
1558
1559         if (IS_ERR(table)) {
1560                 if (PTR_ERR(table) == -ENOENT &&
1561                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYTABLE)
1562                         return 0;
1563
1564                 NL_SET_BAD_ATTR(extack, attr);
1565                 return PTR_ERR(table);
1566         }
1567
1568         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
1569             table->use > 0)
1570                 return -EBUSY;
1571
1572         ctx.family = family;
1573         ctx.table = table;
1574
1575         return nft_flush_table(&ctx);
1576 }
1577
1578 static void nf_tables_table_destroy(struct nft_ctx *ctx)
1579 {
1580         if (WARN_ON(ctx->table->use > 0))
1581                 return;
1582
1583         rhltable_destroy(&ctx->table->chains_ht);
1584         kfree(ctx->table->name);
1585         kfree(ctx->table->udata);
1586         kfree(ctx->table);
1587 }
1588
1589 void nft_register_chain_type(const struct nft_chain_type *ctype)
1590 {
1591         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1592         if (WARN_ON(__nft_chain_type_get(ctype->family, ctype->type))) {
1593                 nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1594                 return;
1595         }
1596         chain_type[ctype->family][ctype->type] = ctype;
1597         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1598 }
1599 EXPORT_SYMBOL_GPL(nft_register_chain_type);
1600
1601 void nft_unregister_chain_type(const struct nft_chain_type *ctype)
1602 {
1603         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1604         chain_type[ctype->family][ctype->type] = NULL;
1605         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1606 }
1607 EXPORT_SYMBOL_GPL(nft_unregister_chain_type);
1608
1609 /*
1610  * Chains
1611  */
1612
1613 static struct nft_chain *
1614 nft_chain_lookup_byhandle(const struct nft_table *table, u64 handle, u8 genmask)
1615 {
1616         struct nft_chain *chain;
1617
1618         list_for_each_entry(chain, &table->chains, list) {
1619                 if (chain->handle == handle &&
1620                     nft_active_genmask(chain, genmask))
1621                         return chain;
1622         }
1623
1624         return ERR_PTR(-ENOENT);
1625 }
1626
1627 static bool lockdep_commit_lock_is_held(const struct net *net)
1628 {
1629 #ifdef CONFIG_PROVE_LOCKING
1630         struct nftables_pernet *nft_net = nft_pernet(net);
1631
1632         return lockdep_is_held(&nft_net->commit_mutex);
1633 #else
1634         return true;
1635 #endif
1636 }
1637
1638 static struct nft_chain *nft_chain_lookup(struct net *net,
1639                                           struct nft_table *table,
1640                                           const struct nlattr *nla, u8 genmask)
1641 {
1642         char search[NFT_CHAIN_MAXNAMELEN + 1];
1643         struct rhlist_head *tmp, *list;
1644         struct nft_chain *chain;
1645
1646         if (nla == NULL)
1647                 return ERR_PTR(-EINVAL);
1648
1649         nla_strscpy(search, nla, sizeof(search));
1650
1651         WARN_ON(!rcu_read_lock_held() &&
1652                 !lockdep_commit_lock_is_held(net));
1653
1654         chain = ERR_PTR(-ENOENT);
1655         rcu_read_lock();
1656         list = rhltable_lookup(&table->chains_ht, search, nft_chain_ht_params);
1657         if (!list)
1658                 goto out_unlock;
1659
1660         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
1661                 if (nft_active_genmask(chain, genmask))
1662                         goto out_unlock;
1663         }
1664         chain = ERR_PTR(-ENOENT);
1665 out_unlock:
1666         rcu_read_unlock();
1667         return chain;
1668 }
1669
1670 static const struct nla_policy nft_chain_policy[NFTA_CHAIN_MAX + 1] = {
1671         [NFTA_CHAIN_TABLE]      = { .type = NLA_STRING,
1672                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
1673         [NFTA_CHAIN_HANDLE]     = { .type = NLA_U64 },
1674         [NFTA_CHAIN_NAME]       = { .type = NLA_STRING,
1675                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
1676         [NFTA_CHAIN_HOOK]       = { .type = NLA_NESTED },
1677         [NFTA_CHAIN_POLICY]     = { .type = NLA_U32 },
1678         [NFTA_CHAIN_TYPE]       = { .type = NLA_STRING,
1679                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
1680         [NFTA_CHAIN_COUNTERS]   = { .type = NLA_NESTED },
1681         [NFTA_CHAIN_FLAGS]      = { .type = NLA_U32 },
1682         [NFTA_CHAIN_ID]         = { .type = NLA_U32 },
1683         [NFTA_CHAIN_USERDATA]   = { .type = NLA_BINARY,
1684                                     .len = NFT_USERDATA_MAXLEN },
1685 };
1686
1687 static const struct nla_policy nft_hook_policy[NFTA_HOOK_MAX + 1] = {
1688         [NFTA_HOOK_HOOKNUM]     = { .type = NLA_U32 },
1689         [NFTA_HOOK_PRIORITY]    = { .type = NLA_U32 },
1690         [NFTA_HOOK_DEV]         = { .type = NLA_STRING,
1691                                     .len = IFNAMSIZ - 1 },
1692 };
1693
1694 static int nft_dump_stats(struct sk_buff *skb, struct nft_stats __percpu *stats)
1695 {
1696         struct nft_stats *cpu_stats, total;
1697         struct nlattr *nest;
1698         unsigned int seq;
1699         u64 pkts, bytes;
1700         int cpu;
1701
1702         if (!stats)
1703                 return 0;
1704
1705         memset(&total, 0, sizeof(total));
1706         for_each_possible_cpu(cpu) {
1707                 cpu_stats = per_cpu_ptr(stats, cpu);
1708                 do {
1709                         seq = u64_stats_fetch_begin(&cpu_stats->syncp);
1710                         pkts = cpu_stats->pkts;
1711                         bytes = cpu_stats->bytes;
1712                 } while (u64_stats_fetch_retry(&cpu_stats->syncp, seq));
1713                 total.pkts += pkts;
1714                 total.bytes += bytes;
1715         }
1716         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_COUNTERS);
1717         if (nest == NULL)
1718                 goto nla_put_failure;
1719
1720         if (nla_put_be64(skb, NFTA_COUNTER_PACKETS, cpu_to_be64(total.pkts),
1721                          NFTA_COUNTER_PAD) ||
1722             nla_put_be64(skb, NFTA_COUNTER_BYTES, cpu_to_be64(total.bytes),
1723                          NFTA_COUNTER_PAD))
1724                 goto nla_put_failure;
1725
1726         nla_nest_end(skb, nest);
1727         return 0;
1728
1729 nla_put_failure:
1730         return -ENOSPC;
1731 }
1732
1733 static int nft_dump_basechain_hook(struct sk_buff *skb, int family,
1734                                    const struct nft_base_chain *basechain,
1735                                    const struct list_head *hook_list)
1736 {
1737         const struct nf_hook_ops *ops = &basechain->ops;
1738         struct nft_hook *hook, *first = NULL;
1739         struct nlattr *nest, *nest_devs;
1740         int n = 0;
1741
1742         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_HOOK);
1743         if (nest == NULL)
1744                 goto nla_put_failure;
1745         if (nla_put_be32(skb, NFTA_HOOK_HOOKNUM, htonl(ops->hooknum)))
1746                 goto nla_put_failure;
1747         if (nla_put_be32(skb, NFTA_HOOK_PRIORITY, htonl(ops->priority)))
1748                 goto nla_put_failure;
1749
1750         if (nft_base_chain_netdev(family, ops->hooknum)) {
1751                 nest_devs = nla_nest_start_noflag(skb, NFTA_HOOK_DEVS);
1752                 if (!nest_devs)
1753                         goto nla_put_failure;
1754
1755                 if (!hook_list)
1756                         hook_list = &basechain->hook_list;
1757
1758                 list_for_each_entry(hook, hook_list, list) {
1759                         if (!first)
1760                                 first = hook;
1761
1762                         if (nla_put_string(skb, NFTA_DEVICE_NAME,
1763                                            hook->ops.dev->name))
1764                                 goto nla_put_failure;
1765                         n++;
1766                 }
1767                 nla_nest_end(skb, nest_devs);
1768
1769                 if (n == 1 &&
1770                     nla_put_string(skb, NFTA_HOOK_DEV, first->ops.dev->name))
1771                         goto nla_put_failure;
1772         }
1773         nla_nest_end(skb, nest);
1774
1775         return 0;
1776 nla_put_failure:
1777         return -1;
1778 }
1779
1780 static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net,
1781                                      u32 portid, u32 seq, int event, u32 flags,
1782                                      int family, const struct nft_table *table,
1783                                      const struct nft_chain *chain,
1784                                      const struct list_head *hook_list)
1785 {
1786         struct nlmsghdr *nlh;
1787
1788         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
1789         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
1790                            NFNETLINK_V0, nft_base_seq(net));
1791         if (!nlh)
1792                 goto nla_put_failure;
1793
1794         if (nla_put_string(skb, NFTA_CHAIN_TABLE, table->name) ||
1795             nla_put_string(skb, NFTA_CHAIN_NAME, chain->name) ||
1796             nla_put_be64(skb, NFTA_CHAIN_HANDLE, cpu_to_be64(chain->handle),
1797                          NFTA_CHAIN_PAD))
1798                 goto nla_put_failure;
1799
1800         if (event == NFT_MSG_DELCHAIN && !hook_list) {
1801                 nlmsg_end(skb, nlh);
1802                 return 0;
1803         }
1804
1805         if (nft_is_base_chain(chain)) {
1806                 const struct nft_base_chain *basechain = nft_base_chain(chain);
1807                 struct nft_stats __percpu *stats;
1808
1809                 if (nft_dump_basechain_hook(skb, family, basechain, hook_list))
1810                         goto nla_put_failure;
1811
1812                 if (nla_put_be32(skb, NFTA_CHAIN_POLICY,
1813                                  htonl(basechain->policy)))
1814                         goto nla_put_failure;
1815
1816                 if (nla_put_string(skb, NFTA_CHAIN_TYPE, basechain->type->name))
1817                         goto nla_put_failure;
1818
1819                 stats = rcu_dereference_check(basechain->stats,
1820                                               lockdep_commit_lock_is_held(net));
1821                 if (nft_dump_stats(skb, stats))
1822                         goto nla_put_failure;
1823         }
1824
1825         if (chain->flags &&
1826             nla_put_be32(skb, NFTA_CHAIN_FLAGS, htonl(chain->flags)))
1827                 goto nla_put_failure;
1828
1829         if (nla_put_be32(skb, NFTA_CHAIN_USE, htonl(chain->use)))
1830                 goto nla_put_failure;
1831
1832         if (chain->udata &&
1833             nla_put(skb, NFTA_CHAIN_USERDATA, chain->udlen, chain->udata))
1834                 goto nla_put_failure;
1835
1836         nlmsg_end(skb, nlh);
1837         return 0;
1838
1839 nla_put_failure:
1840         nlmsg_trim(skb, nlh);
1841         return -1;
1842 }
1843
1844 static void nf_tables_chain_notify(const struct nft_ctx *ctx, int event,
1845                                    const struct list_head *hook_list)
1846 {
1847         struct nftables_pernet *nft_net;
1848         struct sk_buff *skb;
1849         u16 flags = 0;
1850         int err;
1851
1852         if (!ctx->report &&
1853             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
1854                 return;
1855
1856         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
1857         if (skb == NULL)
1858                 goto err;
1859
1860         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
1861                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
1862
1863         err = nf_tables_fill_chain_info(skb, ctx->net, ctx->portid, ctx->seq,
1864                                         event, flags, ctx->family, ctx->table,
1865                                         ctx->chain, hook_list);
1866         if (err < 0) {
1867                 kfree_skb(skb);
1868                 goto err;
1869         }
1870
1871         nft_net = nft_pernet(ctx->net);
1872         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
1873         return;
1874 err:
1875         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
1876 }
1877
1878 static int nf_tables_dump_chains(struct sk_buff *skb,
1879                                  struct netlink_callback *cb)
1880 {
1881         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
1882         unsigned int idx = 0, s_idx = cb->args[0];
1883         struct net *net = sock_net(skb->sk);
1884         int family = nfmsg->nfgen_family;
1885         struct nftables_pernet *nft_net;
1886         const struct nft_table *table;
1887         const struct nft_chain *chain;
1888
1889         rcu_read_lock();
1890         nft_net = nft_pernet(net);
1891         cb->seq = READ_ONCE(nft_net->base_seq);
1892
1893         list_for_each_entry_rcu(table, &nft_net->tables, list) {
1894                 if (family != NFPROTO_UNSPEC && family != table->family)
1895                         continue;
1896
1897                 list_for_each_entry_rcu(chain, &table->chains, list) {
1898                         if (idx < s_idx)
1899                                 goto cont;
1900                         if (idx > s_idx)
1901                                 memset(&cb->args[1], 0,
1902                                        sizeof(cb->args) - sizeof(cb->args[0]));
1903                         if (!nft_is_active(net, chain))
1904                                 continue;
1905                         if (nf_tables_fill_chain_info(skb, net,
1906                                                       NETLINK_CB(cb->skb).portid,
1907                                                       cb->nlh->nlmsg_seq,
1908                                                       NFT_MSG_NEWCHAIN,
1909                                                       NLM_F_MULTI,
1910                                                       table->family, table,
1911                                                       chain, NULL) < 0)
1912                                 goto done;
1913
1914                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
1915 cont:
1916                         idx++;
1917                 }
1918         }
1919 done:
1920         rcu_read_unlock();
1921         cb->args[0] = idx;
1922         return skb->len;
1923 }
1924
1925 /* called with rcu_read_lock held */
1926 static int nf_tables_getchain(struct sk_buff *skb, const struct nfnl_info *info,
1927                               const struct nlattr * const nla[])
1928 {
1929         struct netlink_ext_ack *extack = info->extack;
1930         u8 genmask = nft_genmask_cur(info->net);
1931         u8 family = info->nfmsg->nfgen_family;
1932         const struct nft_chain *chain;
1933         struct net *net = info->net;
1934         struct nft_table *table;
1935         struct sk_buff *skb2;
1936         int err;
1937
1938         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
1939                 struct netlink_dump_control c = {
1940                         .dump = nf_tables_dump_chains,
1941                         .module = THIS_MODULE,
1942                 };
1943
1944                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
1945         }
1946
1947         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask, 0);
1948         if (IS_ERR(table)) {
1949                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
1950                 return PTR_ERR(table);
1951         }
1952
1953         chain = nft_chain_lookup(net, table, nla[NFTA_CHAIN_NAME], genmask);
1954         if (IS_ERR(chain)) {
1955                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
1956                 return PTR_ERR(chain);
1957         }
1958
1959         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
1960         if (!skb2)
1961                 return -ENOMEM;
1962
1963         err = nf_tables_fill_chain_info(skb2, net, NETLINK_CB(skb).portid,
1964                                         info->nlh->nlmsg_seq, NFT_MSG_NEWCHAIN,
1965                                         0, family, table, chain, NULL);
1966         if (err < 0)
1967                 goto err_fill_chain_info;
1968
1969         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
1970
1971 err_fill_chain_info:
1972         kfree_skb(skb2);
1973         return err;
1974 }
1975
1976 static const struct nla_policy nft_counter_policy[NFTA_COUNTER_MAX + 1] = {
1977         [NFTA_COUNTER_PACKETS]  = { .type = NLA_U64 },
1978         [NFTA_COUNTER_BYTES]    = { .type = NLA_U64 },
1979 };
1980
1981 static struct nft_stats __percpu *nft_stats_alloc(const struct nlattr *attr)
1982 {
1983         struct nlattr *tb[NFTA_COUNTER_MAX+1];
1984         struct nft_stats __percpu *newstats;
1985         struct nft_stats *stats;
1986         int err;
1987
1988         err = nla_parse_nested_deprecated(tb, NFTA_COUNTER_MAX, attr,
1989                                           nft_counter_policy, NULL);
1990         if (err < 0)
1991                 return ERR_PTR(err);
1992
1993         if (!tb[NFTA_COUNTER_BYTES] || !tb[NFTA_COUNTER_PACKETS])
1994                 return ERR_PTR(-EINVAL);
1995
1996         newstats = netdev_alloc_pcpu_stats(struct nft_stats);
1997         if (newstats == NULL)
1998                 return ERR_PTR(-ENOMEM);
1999
2000         /* Restore old counters on this cpu, no problem. Per-cpu statistics
2001          * are not exposed to userspace.
2002          */
2003         preempt_disable();
2004         stats = this_cpu_ptr(newstats);
2005         stats->bytes = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_BYTES]));
2006         stats->pkts = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_PACKETS]));
2007         preempt_enable();
2008
2009         return newstats;
2010 }
2011
2012 static void nft_chain_stats_replace(struct nft_trans *trans)
2013 {
2014         struct nft_base_chain *chain = nft_base_chain(trans->ctx.chain);
2015
2016         if (!nft_trans_chain_stats(trans))
2017                 return;
2018
2019         nft_trans_chain_stats(trans) =
2020                 rcu_replace_pointer(chain->stats, nft_trans_chain_stats(trans),
2021                                     lockdep_commit_lock_is_held(trans->ctx.net));
2022
2023         if (!nft_trans_chain_stats(trans))
2024                 static_branch_inc(&nft_counters_enabled);
2025 }
2026
2027 static void nf_tables_chain_free_chain_rules(struct nft_chain *chain)
2028 {
2029         struct nft_rule_blob *g0 = rcu_dereference_raw(chain->blob_gen_0);
2030         struct nft_rule_blob *g1 = rcu_dereference_raw(chain->blob_gen_1);
2031
2032         if (g0 != g1)
2033                 kvfree(g1);
2034         kvfree(g0);
2035
2036         /* should be NULL either via abort or via successful commit */
2037         WARN_ON_ONCE(chain->blob_next);
2038         kvfree(chain->blob_next);
2039 }
2040
2041 void nf_tables_chain_destroy(struct nft_ctx *ctx)
2042 {
2043         struct nft_chain *chain = ctx->chain;
2044         struct nft_hook *hook, *next;
2045
2046         if (WARN_ON(chain->use > 0))
2047                 return;
2048
2049         /* no concurrent access possible anymore */
2050         nf_tables_chain_free_chain_rules(chain);
2051
2052         if (nft_is_base_chain(chain)) {
2053                 struct nft_base_chain *basechain = nft_base_chain(chain);
2054
2055                 if (nft_base_chain_netdev(ctx->family, basechain->ops.hooknum)) {
2056                         list_for_each_entry_safe(hook, next,
2057                                                  &basechain->hook_list, list) {
2058                                 list_del_rcu(&hook->list);
2059                                 kfree_rcu(hook, rcu);
2060                         }
2061                 }
2062                 module_put(basechain->type->owner);
2063                 if (rcu_access_pointer(basechain->stats)) {
2064                         static_branch_dec(&nft_counters_enabled);
2065                         free_percpu(rcu_dereference_raw(basechain->stats));
2066                 }
2067                 kfree(chain->name);
2068                 kfree(chain->udata);
2069                 kfree(basechain);
2070         } else {
2071                 kfree(chain->name);
2072                 kfree(chain->udata);
2073                 kfree(chain);
2074         }
2075 }
2076
2077 static struct nft_hook *nft_netdev_hook_alloc(struct net *net,
2078                                               const struct nlattr *attr)
2079 {
2080         struct net_device *dev;
2081         char ifname[IFNAMSIZ];
2082         struct nft_hook *hook;
2083         int err;
2084
2085         hook = kmalloc(sizeof(struct nft_hook), GFP_KERNEL_ACCOUNT);
2086         if (!hook) {
2087                 err = -ENOMEM;
2088                 goto err_hook_alloc;
2089         }
2090
2091         nla_strscpy(ifname, attr, IFNAMSIZ);
2092         /* nf_tables_netdev_event() is called under rtnl_mutex, this is
2093          * indirectly serializing all the other holders of the commit_mutex with
2094          * the rtnl_mutex.
2095          */
2096         dev = __dev_get_by_name(net, ifname);
2097         if (!dev) {
2098                 err = -ENOENT;
2099                 goto err_hook_dev;
2100         }
2101         hook->ops.dev = dev;
2102
2103         return hook;
2104
2105 err_hook_dev:
2106         kfree(hook);
2107 err_hook_alloc:
2108         return ERR_PTR(err);
2109 }
2110
2111 static struct nft_hook *nft_hook_list_find(struct list_head *hook_list,
2112                                            const struct nft_hook *this)
2113 {
2114         struct nft_hook *hook;
2115
2116         list_for_each_entry(hook, hook_list, list) {
2117                 if (this->ops.dev == hook->ops.dev)
2118                         return hook;
2119         }
2120
2121         return NULL;
2122 }
2123
2124 static int nf_tables_parse_netdev_hooks(struct net *net,
2125                                         const struct nlattr *attr,
2126                                         struct list_head *hook_list,
2127                                         struct netlink_ext_ack *extack)
2128 {
2129         struct nft_hook *hook, *next;
2130         const struct nlattr *tmp;
2131         int rem, n = 0, err;
2132
2133         nla_for_each_nested(tmp, attr, rem) {
2134                 if (nla_type(tmp) != NFTA_DEVICE_NAME) {
2135                         err = -EINVAL;
2136                         goto err_hook;
2137                 }
2138
2139                 hook = nft_netdev_hook_alloc(net, tmp);
2140                 if (IS_ERR(hook)) {
2141                         NL_SET_BAD_ATTR(extack, tmp);
2142                         err = PTR_ERR(hook);
2143                         goto err_hook;
2144                 }
2145                 if (nft_hook_list_find(hook_list, hook)) {
2146                         NL_SET_BAD_ATTR(extack, tmp);
2147                         kfree(hook);
2148                         err = -EEXIST;
2149                         goto err_hook;
2150                 }
2151                 list_add_tail(&hook->list, hook_list);
2152                 n++;
2153
2154                 if (n == NFT_NETDEVICE_MAX) {
2155                         err = -EFBIG;
2156                         goto err_hook;
2157                 }
2158         }
2159
2160         return 0;
2161
2162 err_hook:
2163         list_for_each_entry_safe(hook, next, hook_list, list) {
2164                 list_del(&hook->list);
2165                 kfree(hook);
2166         }
2167         return err;
2168 }
2169
2170 struct nft_chain_hook {
2171         u32                             num;
2172         s32                             priority;
2173         const struct nft_chain_type     *type;
2174         struct list_head                list;
2175 };
2176
2177 static int nft_chain_parse_netdev(struct net *net, struct nlattr *tb[],
2178                                   struct list_head *hook_list,
2179                                   struct netlink_ext_ack *extack, u32 flags)
2180 {
2181         struct nft_hook *hook;
2182         int err;
2183
2184         if (tb[NFTA_HOOK_DEV]) {
2185                 hook = nft_netdev_hook_alloc(net, tb[NFTA_HOOK_DEV]);
2186                 if (IS_ERR(hook)) {
2187                         NL_SET_BAD_ATTR(extack, tb[NFTA_HOOK_DEV]);
2188                         return PTR_ERR(hook);
2189                 }
2190
2191                 list_add_tail(&hook->list, hook_list);
2192         } else if (tb[NFTA_HOOK_DEVS]) {
2193                 err = nf_tables_parse_netdev_hooks(net, tb[NFTA_HOOK_DEVS],
2194                                                    hook_list, extack);
2195                 if (err < 0)
2196                         return err;
2197
2198         }
2199
2200         if (flags & NFT_CHAIN_HW_OFFLOAD &&
2201             list_empty(hook_list))
2202                 return -EINVAL;
2203
2204         return 0;
2205 }
2206
2207 static int nft_chain_parse_hook(struct net *net,
2208                                 struct nft_base_chain *basechain,
2209                                 const struct nlattr * const nla[],
2210                                 struct nft_chain_hook *hook, u8 family,
2211                                 u32 flags, struct netlink_ext_ack *extack)
2212 {
2213         struct nftables_pernet *nft_net = nft_pernet(net);
2214         struct nlattr *ha[NFTA_HOOK_MAX + 1];
2215         const struct nft_chain_type *type;
2216         int err;
2217
2218         lockdep_assert_held(&nft_net->commit_mutex);
2219         lockdep_nfnl_nft_mutex_not_held();
2220
2221         err = nla_parse_nested_deprecated(ha, NFTA_HOOK_MAX,
2222                                           nla[NFTA_CHAIN_HOOK],
2223                                           nft_hook_policy, NULL);
2224         if (err < 0)
2225                 return err;
2226
2227         if (!basechain) {
2228                 if (!ha[NFTA_HOOK_HOOKNUM] ||
2229                     !ha[NFTA_HOOK_PRIORITY]) {
2230                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2231                         return -ENOENT;
2232                 }
2233
2234                 hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
2235                 hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
2236
2237                 type = __nft_chain_type_get(family, NFT_CHAIN_T_DEFAULT);
2238                 if (!type)
2239                         return -EOPNOTSUPP;
2240
2241                 if (nla[NFTA_CHAIN_TYPE]) {
2242                         type = nf_tables_chain_type_lookup(net, nla[NFTA_CHAIN_TYPE],
2243                                                            family, true);
2244                         if (IS_ERR(type)) {
2245                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2246                                 return PTR_ERR(type);
2247                         }
2248                 }
2249                 if (hook->num >= NFT_MAX_HOOKS || !(type->hook_mask & (1 << hook->num)))
2250                         return -EOPNOTSUPP;
2251
2252                 if (type->type == NFT_CHAIN_T_NAT &&
2253                     hook->priority <= NF_IP_PRI_CONNTRACK)
2254                         return -EOPNOTSUPP;
2255         } else {
2256                 if (ha[NFTA_HOOK_HOOKNUM]) {
2257                         hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
2258                         if (hook->num != basechain->ops.hooknum)
2259                                 return -EOPNOTSUPP;
2260                 }
2261                 if (ha[NFTA_HOOK_PRIORITY]) {
2262                         hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
2263                         if (hook->priority != basechain->ops.priority)
2264                                 return -EOPNOTSUPP;
2265                 }
2266
2267                 if (nla[NFTA_CHAIN_TYPE]) {
2268                         type = __nf_tables_chain_type_lookup(nla[NFTA_CHAIN_TYPE],
2269                                                              family);
2270                         if (!type) {
2271                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2272                                 return -ENOENT;
2273                         }
2274                 } else {
2275                         type = basechain->type;
2276                 }
2277         }
2278
2279         if (!try_module_get(type->owner)) {
2280                 if (nla[NFTA_CHAIN_TYPE])
2281                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2282                 return -ENOENT;
2283         }
2284
2285         hook->type = type;
2286
2287         INIT_LIST_HEAD(&hook->list);
2288         if (nft_base_chain_netdev(family, hook->num)) {
2289                 err = nft_chain_parse_netdev(net, ha, &hook->list, extack, flags);
2290                 if (err < 0) {
2291                         module_put(type->owner);
2292                         return err;
2293                 }
2294         } else if (ha[NFTA_HOOK_DEV] || ha[NFTA_HOOK_DEVS]) {
2295                 module_put(type->owner);
2296                 return -EOPNOTSUPP;
2297         }
2298
2299         return 0;
2300 }
2301
2302 static void nft_chain_release_hook(struct nft_chain_hook *hook)
2303 {
2304         struct nft_hook *h, *next;
2305
2306         list_for_each_entry_safe(h, next, &hook->list, list) {
2307                 list_del(&h->list);
2308                 kfree(h);
2309         }
2310         module_put(hook->type->owner);
2311 }
2312
2313 static void nft_last_rule(const struct nft_chain *chain, const void *ptr)
2314 {
2315         struct nft_rule_dp_last *lrule;
2316
2317         BUILD_BUG_ON(offsetof(struct nft_rule_dp_last, end) != 0);
2318
2319         lrule = (struct nft_rule_dp_last *)ptr;
2320         lrule->end.is_last = 1;
2321         lrule->chain = chain;
2322         /* blob size does not include the trailer rule */
2323 }
2324
2325 static struct nft_rule_blob *nf_tables_chain_alloc_rules(const struct nft_chain *chain,
2326                                                          unsigned int size)
2327 {
2328         struct nft_rule_blob *blob;
2329
2330         if (size > INT_MAX)
2331                 return NULL;
2332
2333         size += sizeof(struct nft_rule_blob) + sizeof(struct nft_rule_dp_last);
2334
2335         blob = kvmalloc(size, GFP_KERNEL_ACCOUNT);
2336         if (!blob)
2337                 return NULL;
2338
2339         blob->size = 0;
2340         nft_last_rule(chain, blob->data);
2341
2342         return blob;
2343 }
2344
2345 static void nft_basechain_hook_init(struct nf_hook_ops *ops, u8 family,
2346                                     const struct nft_chain_hook *hook,
2347                                     struct nft_chain *chain)
2348 {
2349         ops->pf                 = family;
2350         ops->hooknum            = hook->num;
2351         ops->priority           = hook->priority;
2352         ops->priv               = chain;
2353         ops->hook               = hook->type->hooks[ops->hooknum];
2354         ops->hook_ops_type      = NF_HOOK_OP_NF_TABLES;
2355 }
2356
2357 static int nft_basechain_init(struct nft_base_chain *basechain, u8 family,
2358                               struct nft_chain_hook *hook, u32 flags)
2359 {
2360         struct nft_chain *chain;
2361         struct nft_hook *h;
2362
2363         basechain->type = hook->type;
2364         INIT_LIST_HEAD(&basechain->hook_list);
2365         chain = &basechain->chain;
2366
2367         if (nft_base_chain_netdev(family, hook->num)) {
2368                 list_splice_init(&hook->list, &basechain->hook_list);
2369                 list_for_each_entry(h, &basechain->hook_list, list)
2370                         nft_basechain_hook_init(&h->ops, family, hook, chain);
2371         }
2372         nft_basechain_hook_init(&basechain->ops, family, hook, chain);
2373
2374         chain->flags |= NFT_CHAIN_BASE | flags;
2375         basechain->policy = NF_ACCEPT;
2376         if (chain->flags & NFT_CHAIN_HW_OFFLOAD &&
2377             !nft_chain_offload_support(basechain)) {
2378                 list_splice_init(&basechain->hook_list, &hook->list);
2379                 return -EOPNOTSUPP;
2380         }
2381
2382         flow_block_init(&basechain->flow_block);
2383
2384         return 0;
2385 }
2386
2387 int nft_chain_add(struct nft_table *table, struct nft_chain *chain)
2388 {
2389         int err;
2390
2391         err = rhltable_insert_key(&table->chains_ht, chain->name,
2392                                   &chain->rhlhead, nft_chain_ht_params);
2393         if (err)
2394                 return err;
2395
2396         list_add_tail_rcu(&chain->list, &table->chains);
2397
2398         return 0;
2399 }
2400
2401 static u64 chain_id;
2402
2403 static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 genmask,
2404                               u8 policy, u32 flags,
2405                               struct netlink_ext_ack *extack)
2406 {
2407         const struct nlattr * const *nla = ctx->nla;
2408         struct nft_table *table = ctx->table;
2409         struct nft_base_chain *basechain;
2410         struct net *net = ctx->net;
2411         char name[NFT_NAME_MAXLEN];
2412         struct nft_rule_blob *blob;
2413         struct nft_trans *trans;
2414         struct nft_chain *chain;
2415         int err;
2416
2417         if (nla[NFTA_CHAIN_HOOK]) {
2418                 struct nft_stats __percpu *stats = NULL;
2419                 struct nft_chain_hook hook = {};
2420
2421                 if (flags & NFT_CHAIN_BINDING)
2422                         return -EOPNOTSUPP;
2423
2424                 err = nft_chain_parse_hook(net, NULL, nla, &hook, family, flags,
2425                                            extack);
2426                 if (err < 0)
2427                         return err;
2428
2429                 basechain = kzalloc(sizeof(*basechain), GFP_KERNEL_ACCOUNT);
2430                 if (basechain == NULL) {
2431                         nft_chain_release_hook(&hook);
2432                         return -ENOMEM;
2433                 }
2434                 chain = &basechain->chain;
2435
2436                 if (nla[NFTA_CHAIN_COUNTERS]) {
2437                         stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2438                         if (IS_ERR(stats)) {
2439                                 nft_chain_release_hook(&hook);
2440                                 kfree(basechain);
2441                                 return PTR_ERR(stats);
2442                         }
2443                         rcu_assign_pointer(basechain->stats, stats);
2444                 }
2445
2446                 err = nft_basechain_init(basechain, family, &hook, flags);
2447                 if (err < 0) {
2448                         nft_chain_release_hook(&hook);
2449                         kfree(basechain);
2450                         free_percpu(stats);
2451                         return err;
2452                 }
2453                 if (stats)
2454                         static_branch_inc(&nft_counters_enabled);
2455         } else {
2456                 if (flags & NFT_CHAIN_BASE)
2457                         return -EINVAL;
2458                 if (flags & NFT_CHAIN_HW_OFFLOAD)
2459                         return -EOPNOTSUPP;
2460
2461                 chain = kzalloc(sizeof(*chain), GFP_KERNEL_ACCOUNT);
2462                 if (chain == NULL)
2463                         return -ENOMEM;
2464
2465                 chain->flags = flags;
2466         }
2467         ctx->chain = chain;
2468
2469         INIT_LIST_HEAD(&chain->rules);
2470         chain->handle = nf_tables_alloc_handle(table);
2471         chain->table = table;
2472
2473         if (nla[NFTA_CHAIN_NAME]) {
2474                 chain->name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
2475         } else {
2476                 if (!(flags & NFT_CHAIN_BINDING)) {
2477                         err = -EINVAL;
2478                         goto err_destroy_chain;
2479                 }
2480
2481                 snprintf(name, sizeof(name), "__chain%llu", ++chain_id);
2482                 chain->name = kstrdup(name, GFP_KERNEL_ACCOUNT);
2483         }
2484
2485         if (!chain->name) {
2486                 err = -ENOMEM;
2487                 goto err_destroy_chain;
2488         }
2489
2490         if (nla[NFTA_CHAIN_USERDATA]) {
2491                 chain->udata = nla_memdup(nla[NFTA_CHAIN_USERDATA], GFP_KERNEL_ACCOUNT);
2492                 if (chain->udata == NULL) {
2493                         err = -ENOMEM;
2494                         goto err_destroy_chain;
2495                 }
2496                 chain->udlen = nla_len(nla[NFTA_CHAIN_USERDATA]);
2497         }
2498
2499         blob = nf_tables_chain_alloc_rules(chain, 0);
2500         if (!blob) {
2501                 err = -ENOMEM;
2502                 goto err_destroy_chain;
2503         }
2504
2505         RCU_INIT_POINTER(chain->blob_gen_0, blob);
2506         RCU_INIT_POINTER(chain->blob_gen_1, blob);
2507
2508         err = nf_tables_register_hook(net, table, chain);
2509         if (err < 0)
2510                 goto err_destroy_chain;
2511
2512         if (!nft_use_inc(&table->use)) {
2513                 err = -EMFILE;
2514                 goto err_use;
2515         }
2516
2517         trans = nft_trans_chain_add(ctx, NFT_MSG_NEWCHAIN);
2518         if (IS_ERR(trans)) {
2519                 err = PTR_ERR(trans);
2520                 goto err_unregister_hook;
2521         }
2522
2523         nft_trans_chain_policy(trans) = NFT_CHAIN_POLICY_UNSET;
2524         if (nft_is_base_chain(chain))
2525                 nft_trans_chain_policy(trans) = policy;
2526
2527         err = nft_chain_add(table, chain);
2528         if (err < 0) {
2529                 nft_trans_destroy(trans);
2530                 goto err_unregister_hook;
2531         }
2532
2533         return 0;
2534
2535 err_unregister_hook:
2536         nft_use_dec_restore(&table->use);
2537 err_use:
2538         nf_tables_unregister_hook(net, table, chain);
2539 err_destroy_chain:
2540         nf_tables_chain_destroy(ctx);
2541
2542         return err;
2543 }
2544
2545 static int nf_tables_updchain(struct nft_ctx *ctx, u8 genmask, u8 policy,
2546                               u32 flags, const struct nlattr *attr,
2547                               struct netlink_ext_ack *extack)
2548 {
2549         const struct nlattr * const *nla = ctx->nla;
2550         struct nft_base_chain *basechain = NULL;
2551         struct nft_table *table = ctx->table;
2552         struct nft_chain *chain = ctx->chain;
2553         struct nft_chain_hook hook = {};
2554         struct nft_stats *stats = NULL;
2555         struct nft_hook *h, *next;
2556         struct nf_hook_ops *ops;
2557         struct nft_trans *trans;
2558         bool unregister = false;
2559         int err;
2560
2561         if (chain->flags ^ flags)
2562                 return -EOPNOTSUPP;
2563
2564         INIT_LIST_HEAD(&hook.list);
2565
2566         if (nla[NFTA_CHAIN_HOOK]) {
2567                 if (!nft_is_base_chain(chain)) {
2568                         NL_SET_BAD_ATTR(extack, attr);
2569                         return -EEXIST;
2570                 }
2571
2572                 basechain = nft_base_chain(chain);
2573                 err = nft_chain_parse_hook(ctx->net, basechain, nla, &hook,
2574                                            ctx->family, flags, extack);
2575                 if (err < 0)
2576                         return err;
2577
2578                 if (basechain->type != hook.type) {
2579                         nft_chain_release_hook(&hook);
2580                         NL_SET_BAD_ATTR(extack, attr);
2581                         return -EEXIST;
2582                 }
2583
2584                 if (nft_base_chain_netdev(ctx->family, basechain->ops.hooknum)) {
2585                         list_for_each_entry_safe(h, next, &hook.list, list) {
2586                                 h->ops.pf       = basechain->ops.pf;
2587                                 h->ops.hooknum  = basechain->ops.hooknum;
2588                                 h->ops.priority = basechain->ops.priority;
2589                                 h->ops.priv     = basechain->ops.priv;
2590                                 h->ops.hook     = basechain->ops.hook;
2591
2592                                 if (nft_hook_list_find(&basechain->hook_list, h)) {
2593                                         list_del(&h->list);
2594                                         kfree(h);
2595                                 }
2596                         }
2597                 } else {
2598                         ops = &basechain->ops;
2599                         if (ops->hooknum != hook.num ||
2600                             ops->priority != hook.priority) {
2601                                 nft_chain_release_hook(&hook);
2602                                 NL_SET_BAD_ATTR(extack, attr);
2603                                 return -EEXIST;
2604                         }
2605                 }
2606         }
2607
2608         if (nla[NFTA_CHAIN_HANDLE] &&
2609             nla[NFTA_CHAIN_NAME]) {
2610                 struct nft_chain *chain2;
2611
2612                 chain2 = nft_chain_lookup(ctx->net, table,
2613                                           nla[NFTA_CHAIN_NAME], genmask);
2614                 if (!IS_ERR(chain2)) {
2615                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2616                         err = -EEXIST;
2617                         goto err_hooks;
2618                 }
2619         }
2620
2621         if (nla[NFTA_CHAIN_COUNTERS]) {
2622                 if (!nft_is_base_chain(chain)) {
2623                         err = -EOPNOTSUPP;
2624                         goto err_hooks;
2625                 }
2626
2627                 stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2628                 if (IS_ERR(stats)) {
2629                         err = PTR_ERR(stats);
2630                         goto err_hooks;
2631                 }
2632         }
2633
2634         if (!(table->flags & NFT_TABLE_F_DORMANT) &&
2635             nft_is_base_chain(chain) &&
2636             !list_empty(&hook.list)) {
2637                 basechain = nft_base_chain(chain);
2638                 ops = &basechain->ops;
2639
2640                 if (nft_base_chain_netdev(table->family, basechain->ops.hooknum)) {
2641                         err = nft_netdev_register_hooks(ctx->net, &hook.list);
2642                         if (err < 0)
2643                                 goto err_hooks;
2644                 }
2645         }
2646
2647         unregister = true;
2648         err = -ENOMEM;
2649         trans = nft_trans_alloc(ctx, NFT_MSG_NEWCHAIN,
2650                                 sizeof(struct nft_trans_chain));
2651         if (trans == NULL)
2652                 goto err_trans;
2653
2654         nft_trans_chain_stats(trans) = stats;
2655         nft_trans_chain_update(trans) = true;
2656
2657         if (nla[NFTA_CHAIN_POLICY])
2658                 nft_trans_chain_policy(trans) = policy;
2659         else
2660                 nft_trans_chain_policy(trans) = -1;
2661
2662         if (nla[NFTA_CHAIN_HANDLE] &&
2663             nla[NFTA_CHAIN_NAME]) {
2664                 struct nftables_pernet *nft_net = nft_pernet(ctx->net);
2665                 struct nft_trans *tmp;
2666                 char *name;
2667
2668                 err = -ENOMEM;
2669                 name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
2670                 if (!name)
2671                         goto err_trans;
2672
2673                 err = -EEXIST;
2674                 list_for_each_entry(tmp, &nft_net->commit_list, list) {
2675                         if (tmp->msg_type == NFT_MSG_NEWCHAIN &&
2676                             tmp->ctx.table == table &&
2677                             nft_trans_chain_update(tmp) &&
2678                             nft_trans_chain_name(tmp) &&
2679                             strcmp(name, nft_trans_chain_name(tmp)) == 0) {
2680                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2681                                 kfree(name);
2682                                 goto err_trans;
2683                         }
2684                 }
2685
2686                 nft_trans_chain_name(trans) = name;
2687         }
2688
2689         nft_trans_basechain(trans) = basechain;
2690         INIT_LIST_HEAD(&nft_trans_chain_hooks(trans));
2691         list_splice(&hook.list, &nft_trans_chain_hooks(trans));
2692         if (nla[NFTA_CHAIN_HOOK])
2693                 module_put(hook.type->owner);
2694
2695         nft_trans_commit_list_add_tail(ctx->net, trans);
2696
2697         return 0;
2698
2699 err_trans:
2700         free_percpu(stats);
2701         kfree(trans);
2702 err_hooks:
2703         if (nla[NFTA_CHAIN_HOOK]) {
2704                 list_for_each_entry_safe(h, next, &hook.list, list) {
2705                         if (unregister)
2706                                 nf_unregister_net_hook(ctx->net, &h->ops);
2707                         list_del(&h->list);
2708                         kfree_rcu(h, rcu);
2709                 }
2710                 module_put(hook.type->owner);
2711         }
2712
2713         return err;
2714 }
2715
2716 static struct nft_chain *nft_chain_lookup_byid(const struct net *net,
2717                                                const struct nft_table *table,
2718                                                const struct nlattr *nla, u8 genmask)
2719 {
2720         struct nftables_pernet *nft_net = nft_pernet(net);
2721         u32 id = ntohl(nla_get_be32(nla));
2722         struct nft_trans *trans;
2723
2724         list_for_each_entry(trans, &nft_net->commit_list, list) {
2725                 struct nft_chain *chain = trans->ctx.chain;
2726
2727                 if (trans->msg_type == NFT_MSG_NEWCHAIN &&
2728                     chain->table == table &&
2729                     id == nft_trans_chain_id(trans) &&
2730                     nft_active_genmask(chain, genmask))
2731                         return chain;
2732         }
2733         return ERR_PTR(-ENOENT);
2734 }
2735
2736 static int nf_tables_newchain(struct sk_buff *skb, const struct nfnl_info *info,
2737                               const struct nlattr * const nla[])
2738 {
2739         struct nftables_pernet *nft_net = nft_pernet(info->net);
2740         struct netlink_ext_ack *extack = info->extack;
2741         u8 genmask = nft_genmask_next(info->net);
2742         u8 family = info->nfmsg->nfgen_family;
2743         struct nft_chain *chain = NULL;
2744         struct net *net = info->net;
2745         const struct nlattr *attr;
2746         struct nft_table *table;
2747         u8 policy = NF_ACCEPT;
2748         struct nft_ctx ctx;
2749         u64 handle = 0;
2750         u32 flags = 0;
2751
2752         lockdep_assert_held(&nft_net->commit_mutex);
2753
2754         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
2755                                  NETLINK_CB(skb).portid);
2756         if (IS_ERR(table)) {
2757                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2758                 return PTR_ERR(table);
2759         }
2760
2761         chain = NULL;
2762         attr = nla[NFTA_CHAIN_NAME];
2763
2764         if (nla[NFTA_CHAIN_HANDLE]) {
2765                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_CHAIN_HANDLE]));
2766                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2767                 if (IS_ERR(chain)) {
2768                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_HANDLE]);
2769                         return PTR_ERR(chain);
2770                 }
2771                 attr = nla[NFTA_CHAIN_HANDLE];
2772         } else if (nla[NFTA_CHAIN_NAME]) {
2773                 chain = nft_chain_lookup(net, table, attr, genmask);
2774                 if (IS_ERR(chain)) {
2775                         if (PTR_ERR(chain) != -ENOENT) {
2776                                 NL_SET_BAD_ATTR(extack, attr);
2777                                 return PTR_ERR(chain);
2778                         }
2779                         chain = NULL;
2780                 }
2781         } else if (!nla[NFTA_CHAIN_ID]) {
2782                 return -EINVAL;
2783         }
2784
2785         if (nla[NFTA_CHAIN_POLICY]) {
2786                 if (chain != NULL &&
2787                     !nft_is_base_chain(chain)) {
2788                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2789                         return -EOPNOTSUPP;
2790                 }
2791
2792                 if (chain == NULL &&
2793                     nla[NFTA_CHAIN_HOOK] == NULL) {
2794                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2795                         return -EOPNOTSUPP;
2796                 }
2797
2798                 policy = ntohl(nla_get_be32(nla[NFTA_CHAIN_POLICY]));
2799                 switch (policy) {
2800                 case NF_DROP:
2801                 case NF_ACCEPT:
2802                         break;
2803                 default:
2804                         return -EINVAL;
2805                 }
2806         }
2807
2808         if (nla[NFTA_CHAIN_FLAGS])
2809                 flags = ntohl(nla_get_be32(nla[NFTA_CHAIN_FLAGS]));
2810         else if (chain)
2811                 flags = chain->flags;
2812
2813         if (flags & ~NFT_CHAIN_FLAGS)
2814                 return -EOPNOTSUPP;
2815
2816         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
2817
2818         if (chain != NULL) {
2819                 if (chain->flags & NFT_CHAIN_BINDING)
2820                         return -EINVAL;
2821
2822                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
2823                         NL_SET_BAD_ATTR(extack, attr);
2824                         return -EEXIST;
2825                 }
2826                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
2827                         return -EOPNOTSUPP;
2828
2829                 flags |= chain->flags & NFT_CHAIN_BASE;
2830                 return nf_tables_updchain(&ctx, genmask, policy, flags, attr,
2831                                           extack);
2832         }
2833
2834         return nf_tables_addchain(&ctx, family, genmask, policy, flags, extack);
2835 }
2836
2837 static int nft_delchain_hook(struct nft_ctx *ctx,
2838                              struct nft_base_chain *basechain,
2839                              struct netlink_ext_ack *extack)
2840 {
2841         const struct nft_chain *chain = &basechain->chain;
2842         const struct nlattr * const *nla = ctx->nla;
2843         struct nft_chain_hook chain_hook = {};
2844         struct nft_hook *this, *hook;
2845         LIST_HEAD(chain_del_list);
2846         struct nft_trans *trans;
2847         int err;
2848
2849         err = nft_chain_parse_hook(ctx->net, basechain, nla, &chain_hook,
2850                                    ctx->family, chain->flags, extack);
2851         if (err < 0)
2852                 return err;
2853
2854         list_for_each_entry(this, &chain_hook.list, list) {
2855                 hook = nft_hook_list_find(&basechain->hook_list, this);
2856                 if (!hook) {
2857                         err = -ENOENT;
2858                         goto err_chain_del_hook;
2859                 }
2860                 list_move(&hook->list, &chain_del_list);
2861         }
2862
2863         trans = nft_trans_alloc(ctx, NFT_MSG_DELCHAIN,
2864                                 sizeof(struct nft_trans_chain));
2865         if (!trans) {
2866                 err = -ENOMEM;
2867                 goto err_chain_del_hook;
2868         }
2869
2870         nft_trans_basechain(trans) = basechain;
2871         nft_trans_chain_update(trans) = true;
2872         INIT_LIST_HEAD(&nft_trans_chain_hooks(trans));
2873         list_splice(&chain_del_list, &nft_trans_chain_hooks(trans));
2874         nft_chain_release_hook(&chain_hook);
2875
2876         nft_trans_commit_list_add_tail(ctx->net, trans);
2877
2878         return 0;
2879
2880 err_chain_del_hook:
2881         list_splice(&chain_del_list, &basechain->hook_list);
2882         nft_chain_release_hook(&chain_hook);
2883
2884         return err;
2885 }
2886
2887 static int nf_tables_delchain(struct sk_buff *skb, const struct nfnl_info *info,
2888                               const struct nlattr * const nla[])
2889 {
2890         struct netlink_ext_ack *extack = info->extack;
2891         u8 genmask = nft_genmask_next(info->net);
2892         u8 family = info->nfmsg->nfgen_family;
2893         struct net *net = info->net;
2894         const struct nlattr *attr;
2895         struct nft_table *table;
2896         struct nft_chain *chain;
2897         struct nft_rule *rule;
2898         struct nft_ctx ctx;
2899         u64 handle;
2900         u32 use;
2901         int err;
2902
2903         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
2904                                  NETLINK_CB(skb).portid);
2905         if (IS_ERR(table)) {
2906                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2907                 return PTR_ERR(table);
2908         }
2909
2910         if (nla[NFTA_CHAIN_HANDLE]) {
2911                 attr = nla[NFTA_CHAIN_HANDLE];
2912                 handle = be64_to_cpu(nla_get_be64(attr));
2913                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2914         } else {
2915                 attr = nla[NFTA_CHAIN_NAME];
2916                 chain = nft_chain_lookup(net, table, attr, genmask);
2917         }
2918         if (IS_ERR(chain)) {
2919                 if (PTR_ERR(chain) == -ENOENT &&
2920                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYCHAIN)
2921                         return 0;
2922
2923                 NL_SET_BAD_ATTR(extack, attr);
2924                 return PTR_ERR(chain);
2925         }
2926
2927         if (nft_chain_binding(chain))
2928                 return -EOPNOTSUPP;
2929
2930         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
2931
2932         if (nla[NFTA_CHAIN_HOOK]) {
2933                 if (chain->flags & NFT_CHAIN_HW_OFFLOAD)
2934                         return -EOPNOTSUPP;
2935
2936                 if (nft_is_base_chain(chain)) {
2937                         struct nft_base_chain *basechain = nft_base_chain(chain);
2938
2939                         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
2940                                 return nft_delchain_hook(&ctx, basechain, extack);
2941                 }
2942         }
2943
2944         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
2945             chain->use > 0)
2946                 return -EBUSY;
2947
2948         use = chain->use;
2949         list_for_each_entry(rule, &chain->rules, list) {
2950                 if (!nft_is_active_next(net, rule))
2951                         continue;
2952                 use--;
2953
2954                 err = nft_delrule(&ctx, rule);
2955                 if (err < 0)
2956                         return err;
2957         }
2958
2959         /* There are rules and elements that are still holding references to us,
2960          * we cannot do a recursive removal in this case.
2961          */
2962         if (use > 0) {
2963                 NL_SET_BAD_ATTR(extack, attr);
2964                 return -EBUSY;
2965         }
2966
2967         return nft_delchain(&ctx);
2968 }
2969
2970 /*
2971  * Expressions
2972  */
2973
2974 /**
2975  *      nft_register_expr - register nf_tables expr type
2976  *      @type: expr type
2977  *
2978  *      Registers the expr type for use with nf_tables. Returns zero on
2979  *      success or a negative errno code otherwise.
2980  */
2981 int nft_register_expr(struct nft_expr_type *type)
2982 {
2983         nfnl_lock(NFNL_SUBSYS_NFTABLES);
2984         if (type->family == NFPROTO_UNSPEC)
2985                 list_add_tail_rcu(&type->list, &nf_tables_expressions);
2986         else
2987                 list_add_rcu(&type->list, &nf_tables_expressions);
2988         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
2989         return 0;
2990 }
2991 EXPORT_SYMBOL_GPL(nft_register_expr);
2992
2993 /**
2994  *      nft_unregister_expr - unregister nf_tables expr type
2995  *      @type: expr type
2996  *
2997  *      Unregisters the expr typefor use with nf_tables.
2998  */
2999 void nft_unregister_expr(struct nft_expr_type *type)
3000 {
3001         nfnl_lock(NFNL_SUBSYS_NFTABLES);
3002         list_del_rcu(&type->list);
3003         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
3004 }
3005 EXPORT_SYMBOL_GPL(nft_unregister_expr);
3006
3007 static const struct nft_expr_type *__nft_expr_type_get(u8 family,
3008                                                        struct nlattr *nla)
3009 {
3010         const struct nft_expr_type *type, *candidate = NULL;
3011
3012         list_for_each_entry(type, &nf_tables_expressions, list) {
3013                 if (!nla_strcmp(nla, type->name)) {
3014                         if (!type->family && !candidate)
3015                                 candidate = type;
3016                         else if (type->family == family)
3017                                 candidate = type;
3018                 }
3019         }
3020         return candidate;
3021 }
3022
3023 #ifdef CONFIG_MODULES
3024 static int nft_expr_type_request_module(struct net *net, u8 family,
3025                                         struct nlattr *nla)
3026 {
3027         if (nft_request_module(net, "nft-expr-%u-%.*s", family,
3028                                nla_len(nla), (char *)nla_data(nla)) == -EAGAIN)
3029                 return -EAGAIN;
3030
3031         return 0;
3032 }
3033 #endif
3034
3035 static const struct nft_expr_type *nft_expr_type_get(struct net *net,
3036                                                      u8 family,
3037                                                      struct nlattr *nla)
3038 {
3039         const struct nft_expr_type *type;
3040
3041         if (nla == NULL)
3042                 return ERR_PTR(-EINVAL);
3043
3044         type = __nft_expr_type_get(family, nla);
3045         if (type != NULL && try_module_get(type->owner))
3046                 return type;
3047
3048         lockdep_nfnl_nft_mutex_not_held();
3049 #ifdef CONFIG_MODULES
3050         if (type == NULL) {
3051                 if (nft_expr_type_request_module(net, family, nla) == -EAGAIN)
3052                         return ERR_PTR(-EAGAIN);
3053
3054                 if (nft_request_module(net, "nft-expr-%.*s",
3055                                        nla_len(nla),
3056                                        (char *)nla_data(nla)) == -EAGAIN)
3057                         return ERR_PTR(-EAGAIN);
3058         }
3059 #endif
3060         return ERR_PTR(-ENOENT);
3061 }
3062
3063 static const struct nla_policy nft_expr_policy[NFTA_EXPR_MAX + 1] = {
3064         [NFTA_EXPR_NAME]        = { .type = NLA_STRING,
3065                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
3066         [NFTA_EXPR_DATA]        = { .type = NLA_NESTED },
3067 };
3068
3069 static int nf_tables_fill_expr_info(struct sk_buff *skb,
3070                                     const struct nft_expr *expr, bool reset)
3071 {
3072         if (nla_put_string(skb, NFTA_EXPR_NAME, expr->ops->type->name))
3073                 goto nla_put_failure;
3074
3075         if (expr->ops->dump) {
3076                 struct nlattr *data = nla_nest_start_noflag(skb,
3077                                                             NFTA_EXPR_DATA);
3078                 if (data == NULL)
3079                         goto nla_put_failure;
3080                 if (expr->ops->dump(skb, expr, reset) < 0)
3081                         goto nla_put_failure;
3082                 nla_nest_end(skb, data);
3083         }
3084
3085         return skb->len;
3086
3087 nla_put_failure:
3088         return -1;
3089 };
3090
3091 int nft_expr_dump(struct sk_buff *skb, unsigned int attr,
3092                   const struct nft_expr *expr, bool reset)
3093 {
3094         struct nlattr *nest;
3095
3096         nest = nla_nest_start_noflag(skb, attr);
3097         if (!nest)
3098                 goto nla_put_failure;
3099         if (nf_tables_fill_expr_info(skb, expr, reset) < 0)
3100                 goto nla_put_failure;
3101         nla_nest_end(skb, nest);
3102         return 0;
3103
3104 nla_put_failure:
3105         return -1;
3106 }
3107
3108 struct nft_expr_info {
3109         const struct nft_expr_ops       *ops;
3110         const struct nlattr             *attr;
3111         struct nlattr                   *tb[NFT_EXPR_MAXATTR + 1];
3112 };
3113
3114 static int nf_tables_expr_parse(const struct nft_ctx *ctx,
3115                                 const struct nlattr *nla,
3116                                 struct nft_expr_info *info)
3117 {
3118         const struct nft_expr_type *type;
3119         const struct nft_expr_ops *ops;
3120         struct nlattr *tb[NFTA_EXPR_MAX + 1];
3121         int err;
3122
3123         err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
3124                                           nft_expr_policy, NULL);
3125         if (err < 0)
3126                 return err;
3127
3128         type = nft_expr_type_get(ctx->net, ctx->family, tb[NFTA_EXPR_NAME]);
3129         if (IS_ERR(type))
3130                 return PTR_ERR(type);
3131
3132         if (tb[NFTA_EXPR_DATA]) {
3133                 err = nla_parse_nested_deprecated(info->tb, type->maxattr,
3134                                                   tb[NFTA_EXPR_DATA],
3135                                                   type->policy, NULL);
3136                 if (err < 0)
3137                         goto err1;
3138         } else
3139                 memset(info->tb, 0, sizeof(info->tb[0]) * (type->maxattr + 1));
3140
3141         if (type->select_ops != NULL) {
3142                 ops = type->select_ops(ctx,
3143                                        (const struct nlattr * const *)info->tb);
3144                 if (IS_ERR(ops)) {
3145                         err = PTR_ERR(ops);
3146 #ifdef CONFIG_MODULES
3147                         if (err == -EAGAIN)
3148                                 if (nft_expr_type_request_module(ctx->net,
3149                                                                  ctx->family,
3150                                                                  tb[NFTA_EXPR_NAME]) != -EAGAIN)
3151                                         err = -ENOENT;
3152 #endif
3153                         goto err1;
3154                 }
3155         } else
3156                 ops = type->ops;
3157
3158         info->attr = nla;
3159         info->ops = ops;
3160
3161         return 0;
3162
3163 err1:
3164         module_put(type->owner);
3165         return err;
3166 }
3167
3168 int nft_expr_inner_parse(const struct nft_ctx *ctx, const struct nlattr *nla,
3169                          struct nft_expr_info *info)
3170 {
3171         struct nlattr *tb[NFTA_EXPR_MAX + 1];
3172         const struct nft_expr_type *type;
3173         int err;
3174
3175         err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
3176                                           nft_expr_policy, NULL);
3177         if (err < 0)
3178                 return err;
3179
3180         if (!tb[NFTA_EXPR_DATA] || !tb[NFTA_EXPR_NAME])
3181                 return -EINVAL;
3182
3183         type = __nft_expr_type_get(ctx->family, tb[NFTA_EXPR_NAME]);
3184         if (!type)
3185                 return -ENOENT;
3186
3187         if (!type->inner_ops)
3188                 return -EOPNOTSUPP;
3189
3190         err = nla_parse_nested_deprecated(info->tb, type->maxattr,
3191                                           tb[NFTA_EXPR_DATA],
3192                                           type->policy, NULL);
3193         if (err < 0)
3194                 goto err_nla_parse;
3195
3196         info->attr = nla;
3197         info->ops = type->inner_ops;
3198
3199         return 0;
3200
3201 err_nla_parse:
3202         return err;
3203 }
3204
3205 static int nf_tables_newexpr(const struct nft_ctx *ctx,
3206                              const struct nft_expr_info *expr_info,
3207                              struct nft_expr *expr)
3208 {
3209         const struct nft_expr_ops *ops = expr_info->ops;
3210         int err;
3211
3212         expr->ops = ops;
3213         if (ops->init) {
3214                 err = ops->init(ctx, expr, (const struct nlattr **)expr_info->tb);
3215                 if (err < 0)
3216                         goto err1;
3217         }
3218
3219         return 0;
3220 err1:
3221         expr->ops = NULL;
3222         return err;
3223 }
3224
3225 static void nf_tables_expr_destroy(const struct nft_ctx *ctx,
3226                                    struct nft_expr *expr)
3227 {
3228         const struct nft_expr_type *type = expr->ops->type;
3229
3230         if (expr->ops->destroy)
3231                 expr->ops->destroy(ctx, expr);
3232         module_put(type->owner);
3233 }
3234
3235 static struct nft_expr *nft_expr_init(const struct nft_ctx *ctx,
3236                                       const struct nlattr *nla)
3237 {
3238         struct nft_expr_info expr_info;
3239         struct nft_expr *expr;
3240         struct module *owner;
3241         int err;
3242
3243         err = nf_tables_expr_parse(ctx, nla, &expr_info);
3244         if (err < 0)
3245                 goto err_expr_parse;
3246
3247         err = -EOPNOTSUPP;
3248         if (!(expr_info.ops->type->flags & NFT_EXPR_STATEFUL))
3249                 goto err_expr_stateful;
3250
3251         err = -ENOMEM;
3252         expr = kzalloc(expr_info.ops->size, GFP_KERNEL_ACCOUNT);
3253         if (expr == NULL)
3254                 goto err_expr_stateful;
3255
3256         err = nf_tables_newexpr(ctx, &expr_info, expr);
3257         if (err < 0)
3258                 goto err_expr_new;
3259
3260         return expr;
3261 err_expr_new:
3262         kfree(expr);
3263 err_expr_stateful:
3264         owner = expr_info.ops->type->owner;
3265         if (expr_info.ops->type->release_ops)
3266                 expr_info.ops->type->release_ops(expr_info.ops);
3267
3268         module_put(owner);
3269 err_expr_parse:
3270         return ERR_PTR(err);
3271 }
3272
3273 int nft_expr_clone(struct nft_expr *dst, struct nft_expr *src)
3274 {
3275         int err;
3276
3277         if (src->ops->clone) {
3278                 dst->ops = src->ops;
3279                 err = src->ops->clone(dst, src);
3280                 if (err < 0)
3281                         return err;
3282         } else {
3283                 memcpy(dst, src, src->ops->size);
3284         }
3285
3286         __module_get(src->ops->type->owner);
3287
3288         return 0;
3289 }
3290
3291 void nft_expr_destroy(const struct nft_ctx *ctx, struct nft_expr *expr)
3292 {
3293         nf_tables_expr_destroy(ctx, expr);
3294         kfree(expr);
3295 }
3296
3297 /*
3298  * Rules
3299  */
3300
3301 static struct nft_rule *__nft_rule_lookup(const struct nft_chain *chain,
3302                                           u64 handle)
3303 {
3304         struct nft_rule *rule;
3305
3306         // FIXME: this sucks
3307         list_for_each_entry_rcu(rule, &chain->rules, list) {
3308                 if (handle == rule->handle)
3309                         return rule;
3310         }
3311
3312         return ERR_PTR(-ENOENT);
3313 }
3314
3315 static struct nft_rule *nft_rule_lookup(const struct nft_chain *chain,
3316                                         const struct nlattr *nla)
3317 {
3318         if (nla == NULL)
3319                 return ERR_PTR(-EINVAL);
3320
3321         return __nft_rule_lookup(chain, be64_to_cpu(nla_get_be64(nla)));
3322 }
3323
3324 static const struct nla_policy nft_rule_policy[NFTA_RULE_MAX + 1] = {
3325         [NFTA_RULE_TABLE]       = { .type = NLA_STRING,
3326                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
3327         [NFTA_RULE_CHAIN]       = { .type = NLA_STRING,
3328                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
3329         [NFTA_RULE_HANDLE]      = { .type = NLA_U64 },
3330         [NFTA_RULE_EXPRESSIONS] = { .type = NLA_NESTED },
3331         [NFTA_RULE_COMPAT]      = { .type = NLA_NESTED },
3332         [NFTA_RULE_POSITION]    = { .type = NLA_U64 },
3333         [NFTA_RULE_USERDATA]    = { .type = NLA_BINARY,
3334                                     .len = NFT_USERDATA_MAXLEN },
3335         [NFTA_RULE_ID]          = { .type = NLA_U32 },
3336         [NFTA_RULE_POSITION_ID] = { .type = NLA_U32 },
3337         [NFTA_RULE_CHAIN_ID]    = { .type = NLA_U32 },
3338 };
3339
3340 static int nf_tables_fill_rule_info(struct sk_buff *skb, struct net *net,
3341                                     u32 portid, u32 seq, int event,
3342                                     u32 flags, int family,
3343                                     const struct nft_table *table,
3344                                     const struct nft_chain *chain,
3345                                     const struct nft_rule *rule, u64 handle,
3346                                     bool reset)
3347 {
3348         struct nlmsghdr *nlh;
3349         const struct nft_expr *expr, *next;
3350         struct nlattr *list;
3351         u16 type = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
3352
3353         nlh = nfnl_msg_put(skb, portid, seq, type, flags, family, NFNETLINK_V0,
3354                            nft_base_seq(net));
3355         if (!nlh)
3356                 goto nla_put_failure;
3357
3358         if (nla_put_string(skb, NFTA_RULE_TABLE, table->name))
3359                 goto nla_put_failure;
3360         if (nla_put_string(skb, NFTA_RULE_CHAIN, chain->name))
3361                 goto nla_put_failure;
3362         if (nla_put_be64(skb, NFTA_RULE_HANDLE, cpu_to_be64(rule->handle),
3363                          NFTA_RULE_PAD))
3364                 goto nla_put_failure;
3365
3366         if (event != NFT_MSG_DELRULE && handle) {
3367                 if (nla_put_be64(skb, NFTA_RULE_POSITION, cpu_to_be64(handle),
3368                                  NFTA_RULE_PAD))
3369                         goto nla_put_failure;
3370         }
3371
3372         if (chain->flags & NFT_CHAIN_HW_OFFLOAD)
3373                 nft_flow_rule_stats(chain, rule);
3374
3375         list = nla_nest_start_noflag(skb, NFTA_RULE_EXPRESSIONS);
3376         if (list == NULL)
3377                 goto nla_put_failure;
3378         nft_rule_for_each_expr(expr, next, rule) {
3379                 if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr, reset) < 0)
3380                         goto nla_put_failure;
3381         }
3382         nla_nest_end(skb, list);
3383
3384         if (rule->udata) {
3385                 struct nft_userdata *udata = nft_userdata(rule);
3386                 if (nla_put(skb, NFTA_RULE_USERDATA, udata->len + 1,
3387                             udata->data) < 0)
3388                         goto nla_put_failure;
3389         }
3390
3391         nlmsg_end(skb, nlh);
3392         return 0;
3393
3394 nla_put_failure:
3395         nlmsg_trim(skb, nlh);
3396         return -1;
3397 }
3398
3399 static void nf_tables_rule_notify(const struct nft_ctx *ctx,
3400                                   const struct nft_rule *rule, int event)
3401 {
3402         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
3403         const struct nft_rule *prule;
3404         struct sk_buff *skb;
3405         u64 handle = 0;
3406         u16 flags = 0;
3407         int err;
3408
3409         if (!ctx->report &&
3410             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
3411                 return;
3412
3413         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
3414         if (skb == NULL)
3415                 goto err;
3416
3417         if (event == NFT_MSG_NEWRULE &&
3418             !list_is_first(&rule->list, &ctx->chain->rules) &&
3419             !list_is_last(&rule->list, &ctx->chain->rules)) {
3420                 prule = list_prev_entry(rule, list);
3421                 handle = prule->handle;
3422         }
3423         if (ctx->flags & (NLM_F_APPEND | NLM_F_REPLACE))
3424                 flags |= NLM_F_APPEND;
3425         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
3426                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
3427
3428         err = nf_tables_fill_rule_info(skb, ctx->net, ctx->portid, ctx->seq,
3429                                        event, flags, ctx->family, ctx->table,
3430                                        ctx->chain, rule, handle, false);
3431         if (err < 0) {
3432                 kfree_skb(skb);
3433                 goto err;
3434         }
3435
3436         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
3437         return;
3438 err:
3439         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
3440 }
3441
3442 static void audit_log_rule_reset(const struct nft_table *table,
3443                                  unsigned int base_seq,
3444                                  unsigned int nentries)
3445 {
3446         char *buf = kasprintf(GFP_ATOMIC, "%s:%u",
3447                               table->name, base_seq);
3448
3449         audit_log_nfcfg(buf, table->family, nentries,
3450                         AUDIT_NFT_OP_RULE_RESET, GFP_ATOMIC);
3451         kfree(buf);
3452 }
3453
3454 struct nft_rule_dump_ctx {
3455         char *table;
3456         char *chain;
3457 };
3458
3459 static int __nf_tables_dump_rules(struct sk_buff *skb,
3460                                   unsigned int *idx,
3461                                   struct netlink_callback *cb,
3462                                   const struct nft_table *table,
3463                                   const struct nft_chain *chain,
3464                                   bool reset)
3465 {
3466         struct net *net = sock_net(skb->sk);
3467         const struct nft_rule *rule, *prule;
3468         unsigned int s_idx = cb->args[0];
3469         unsigned int entries = 0;
3470         int ret = 0;
3471         u64 handle;
3472
3473         prule = NULL;
3474         list_for_each_entry_rcu(rule, &chain->rules, list) {
3475                 if (!nft_is_active(net, rule))
3476                         goto cont_skip;
3477                 if (*idx < s_idx)
3478                         goto cont;
3479                 if (prule)
3480                         handle = prule->handle;
3481                 else
3482                         handle = 0;
3483
3484                 if (nf_tables_fill_rule_info(skb, net, NETLINK_CB(cb->skb).portid,
3485                                         cb->nlh->nlmsg_seq,
3486                                         NFT_MSG_NEWRULE,
3487                                         NLM_F_MULTI | NLM_F_APPEND,
3488                                         table->family,
3489                                         table, chain, rule, handle, reset) < 0) {
3490                         ret = 1;
3491                         break;
3492                 }
3493                 entries++;
3494                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
3495 cont:
3496                 prule = rule;
3497 cont_skip:
3498                 (*idx)++;
3499         }
3500
3501         if (reset && entries)
3502                 audit_log_rule_reset(table, cb->seq, entries);
3503
3504         return ret;
3505 }
3506
3507 static int nf_tables_dump_rules(struct sk_buff *skb,
3508                                 struct netlink_callback *cb)
3509 {
3510         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
3511         const struct nft_rule_dump_ctx *ctx = cb->data;
3512         struct nft_table *table;
3513         const struct nft_chain *chain;
3514         unsigned int idx = 0;
3515         struct net *net = sock_net(skb->sk);
3516         int family = nfmsg->nfgen_family;
3517         struct nftables_pernet *nft_net;
3518         bool reset = false;
3519
3520         if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETRULE_RESET)
3521                 reset = true;
3522
3523         rcu_read_lock();
3524         nft_net = nft_pernet(net);
3525         cb->seq = READ_ONCE(nft_net->base_seq);
3526
3527         list_for_each_entry_rcu(table, &nft_net->tables, list) {
3528                 if (family != NFPROTO_UNSPEC && family != table->family)
3529                         continue;
3530
3531                 if (ctx && ctx->table && strcmp(ctx->table, table->name) != 0)
3532                         continue;
3533
3534                 if (ctx && ctx->table && ctx->chain) {
3535                         struct rhlist_head *list, *tmp;
3536
3537                         list = rhltable_lookup(&table->chains_ht, ctx->chain,
3538                                                nft_chain_ht_params);
3539                         if (!list)
3540                                 goto done;
3541
3542                         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
3543                                 if (!nft_is_active(net, chain))
3544                                         continue;
3545                                 __nf_tables_dump_rules(skb, &idx,
3546                                                        cb, table, chain, reset);
3547                                 break;
3548                         }
3549                         goto done;
3550                 }
3551
3552                 list_for_each_entry_rcu(chain, &table->chains, list) {
3553                         if (__nf_tables_dump_rules(skb, &idx,
3554                                                    cb, table, chain, reset))
3555                                 goto done;
3556                 }
3557
3558                 if (ctx && ctx->table)
3559                         break;
3560         }
3561 done:
3562         rcu_read_unlock();
3563
3564         cb->args[0] = idx;
3565         return skb->len;
3566 }
3567
3568 static int nf_tables_dump_rules_start(struct netlink_callback *cb)
3569 {
3570         const struct nlattr * const *nla = cb->data;
3571         struct nft_rule_dump_ctx *ctx = NULL;
3572
3573         if (nla[NFTA_RULE_TABLE] || nla[NFTA_RULE_CHAIN]) {
3574                 ctx = kzalloc(sizeof(*ctx), GFP_ATOMIC);
3575                 if (!ctx)
3576                         return -ENOMEM;
3577
3578                 if (nla[NFTA_RULE_TABLE]) {
3579                         ctx->table = nla_strdup(nla[NFTA_RULE_TABLE],
3580                                                         GFP_ATOMIC);
3581                         if (!ctx->table) {
3582                                 kfree(ctx);
3583                                 return -ENOMEM;
3584                         }
3585                 }
3586                 if (nla[NFTA_RULE_CHAIN]) {
3587                         ctx->chain = nla_strdup(nla[NFTA_RULE_CHAIN],
3588                                                 GFP_ATOMIC);
3589                         if (!ctx->chain) {
3590                                 kfree(ctx->table);
3591                                 kfree(ctx);
3592                                 return -ENOMEM;
3593                         }
3594                 }
3595         }
3596
3597         cb->data = ctx;
3598         return 0;
3599 }
3600
3601 static int nf_tables_dump_rules_done(struct netlink_callback *cb)
3602 {
3603         struct nft_rule_dump_ctx *ctx = cb->data;
3604
3605         if (ctx) {
3606                 kfree(ctx->table);
3607                 kfree(ctx->chain);
3608                 kfree(ctx);
3609         }
3610         return 0;
3611 }
3612
3613 /* called with rcu_read_lock held */
3614 static int nf_tables_getrule(struct sk_buff *skb, const struct nfnl_info *info,
3615                              const struct nlattr * const nla[])
3616 {
3617         struct netlink_ext_ack *extack = info->extack;
3618         u8 genmask = nft_genmask_cur(info->net);
3619         u8 family = info->nfmsg->nfgen_family;
3620         const struct nft_chain *chain;
3621         const struct nft_rule *rule;
3622         struct net *net = info->net;
3623         struct nft_table *table;
3624         struct sk_buff *skb2;
3625         bool reset = false;
3626         int err;
3627
3628         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
3629                 struct netlink_dump_control c = {
3630                         .start= nf_tables_dump_rules_start,
3631                         .dump = nf_tables_dump_rules,
3632                         .done = nf_tables_dump_rules_done,
3633                         .module = THIS_MODULE,
3634                         .data = (void *)nla,
3635                 };
3636
3637                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
3638         }
3639
3640         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask, 0);
3641         if (IS_ERR(table)) {
3642                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3643                 return PTR_ERR(table);
3644         }
3645
3646         chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN], genmask);
3647         if (IS_ERR(chain)) {
3648                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3649                 return PTR_ERR(chain);
3650         }
3651
3652         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
3653         if (IS_ERR(rule)) {
3654                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3655                 return PTR_ERR(rule);
3656         }
3657
3658         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
3659         if (!skb2)
3660                 return -ENOMEM;
3661
3662         if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETRULE_RESET)
3663                 reset = true;
3664
3665         err = nf_tables_fill_rule_info(skb2, net, NETLINK_CB(skb).portid,
3666                                        info->nlh->nlmsg_seq, NFT_MSG_NEWRULE, 0,
3667                                        family, table, chain, rule, 0, reset);
3668         if (err < 0)
3669                 goto err_fill_rule_info;
3670
3671         if (reset)
3672                 audit_log_rule_reset(table, nft_pernet(net)->base_seq, 1);
3673
3674         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
3675
3676 err_fill_rule_info:
3677         kfree_skb(skb2);
3678         return err;
3679 }
3680
3681 void nf_tables_rule_destroy(const struct nft_ctx *ctx, struct nft_rule *rule)
3682 {
3683         struct nft_expr *expr, *next;
3684
3685         /*
3686          * Careful: some expressions might not be initialized in case this
3687          * is called on error from nf_tables_newrule().
3688          */
3689         expr = nft_expr_first(rule);
3690         while (nft_expr_more(rule, expr)) {
3691                 next = nft_expr_next(expr);
3692                 nf_tables_expr_destroy(ctx, expr);
3693                 expr = next;
3694         }
3695         kfree(rule);
3696 }
3697
3698 static void nf_tables_rule_release(const struct nft_ctx *ctx, struct nft_rule *rule)
3699 {
3700         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_RELEASE);
3701         nf_tables_rule_destroy(ctx, rule);
3702 }
3703
3704 int nft_chain_validate(const struct nft_ctx *ctx, const struct nft_chain *chain)
3705 {
3706         struct nft_expr *expr, *last;
3707         const struct nft_data *data;
3708         struct nft_rule *rule;
3709         int err;
3710
3711         if (ctx->level == NFT_JUMP_STACK_SIZE)
3712                 return -EMLINK;
3713
3714         list_for_each_entry(rule, &chain->rules, list) {
3715                 if (fatal_signal_pending(current))
3716                         return -EINTR;
3717
3718                 if (!nft_is_active_next(ctx->net, rule))
3719                         continue;
3720
3721                 nft_rule_for_each_expr(expr, last, rule) {
3722                         if (!expr->ops->validate)
3723                                 continue;
3724
3725                         err = expr->ops->validate(ctx, expr, &data);
3726                         if (err < 0)
3727                                 return err;
3728                 }
3729         }
3730
3731         return 0;
3732 }
3733 EXPORT_SYMBOL_GPL(nft_chain_validate);
3734
3735 static int nft_table_validate(struct net *net, const struct nft_table *table)
3736 {
3737         struct nft_chain *chain;
3738         struct nft_ctx ctx = {
3739                 .net    = net,
3740                 .family = table->family,
3741         };
3742         int err;
3743
3744         list_for_each_entry(chain, &table->chains, list) {
3745                 if (!nft_is_base_chain(chain))
3746                         continue;
3747
3748                 ctx.chain = chain;
3749                 err = nft_chain_validate(&ctx, chain);
3750                 if (err < 0)
3751                         return err;
3752
3753                 cond_resched();
3754         }
3755
3756         return 0;
3757 }
3758
3759 int nft_setelem_validate(const struct nft_ctx *ctx, struct nft_set *set,
3760                          const struct nft_set_iter *iter,
3761                          struct nft_set_elem *elem)
3762 {
3763         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
3764         struct nft_ctx *pctx = (struct nft_ctx *)ctx;
3765         const struct nft_data *data;
3766         int err;
3767
3768         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
3769             *nft_set_ext_flags(ext) & NFT_SET_ELEM_INTERVAL_END)
3770                 return 0;
3771
3772         data = nft_set_ext_data(ext);
3773         switch (data->verdict.code) {
3774         case NFT_JUMP:
3775         case NFT_GOTO:
3776                 pctx->level++;
3777                 err = nft_chain_validate(ctx, data->verdict.chain);
3778                 if (err < 0)
3779                         return err;
3780                 pctx->level--;
3781                 break;
3782         default:
3783                 break;
3784         }
3785
3786         return 0;
3787 }
3788
3789 int nft_set_catchall_validate(const struct nft_ctx *ctx, struct nft_set *set)
3790 {
3791         u8 genmask = nft_genmask_next(ctx->net);
3792         struct nft_set_elem_catchall *catchall;
3793         struct nft_set_elem elem;
3794         struct nft_set_ext *ext;
3795         int ret = 0;
3796
3797         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
3798                 ext = nft_set_elem_ext(set, catchall->elem);
3799                 if (!nft_set_elem_active(ext, genmask))
3800                         continue;
3801
3802                 elem.priv = catchall->elem;
3803                 ret = nft_setelem_validate(ctx, set, NULL, &elem);
3804                 if (ret < 0)
3805                         return ret;
3806         }
3807
3808         return ret;
3809 }
3810
3811 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
3812                                              const struct nft_chain *chain,
3813                                              const struct nlattr *nla);
3814
3815 #define NFT_RULE_MAXEXPRS       128
3816
3817 static int nf_tables_newrule(struct sk_buff *skb, const struct nfnl_info *info,
3818                              const struct nlattr * const nla[])
3819 {
3820         struct nftables_pernet *nft_net = nft_pernet(info->net);
3821         struct netlink_ext_ack *extack = info->extack;
3822         unsigned int size, i, n, ulen = 0, usize = 0;
3823         u8 genmask = nft_genmask_next(info->net);
3824         struct nft_rule *rule, *old_rule = NULL;
3825         struct nft_expr_info *expr_info = NULL;
3826         u8 family = info->nfmsg->nfgen_family;
3827         struct nft_flow_rule *flow = NULL;
3828         struct net *net = info->net;
3829         struct nft_userdata *udata;
3830         struct nft_table *table;
3831         struct nft_chain *chain;
3832         struct nft_trans *trans;
3833         u64 handle, pos_handle;
3834         struct nft_expr *expr;
3835         struct nft_ctx ctx;
3836         struct nlattr *tmp;
3837         int err, rem;
3838
3839         lockdep_assert_held(&nft_net->commit_mutex);
3840
3841         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
3842                                  NETLINK_CB(skb).portid);
3843         if (IS_ERR(table)) {
3844                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3845                 return PTR_ERR(table);
3846         }
3847
3848         if (nla[NFTA_RULE_CHAIN]) {
3849                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
3850                                          genmask);
3851                 if (IS_ERR(chain)) {
3852                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3853                         return PTR_ERR(chain);
3854                 }
3855
3856         } else if (nla[NFTA_RULE_CHAIN_ID]) {
3857                 chain = nft_chain_lookup_byid(net, table, nla[NFTA_RULE_CHAIN_ID],
3858                                               genmask);
3859                 if (IS_ERR(chain)) {
3860                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN_ID]);
3861                         return PTR_ERR(chain);
3862                 }
3863         } else {
3864                 return -EINVAL;
3865         }
3866
3867         if (nft_chain_is_bound(chain))
3868                 return -EOPNOTSUPP;
3869
3870         if (nla[NFTA_RULE_HANDLE]) {
3871                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_HANDLE]));
3872                 rule = __nft_rule_lookup(chain, handle);
3873                 if (IS_ERR(rule)) {
3874                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3875                         return PTR_ERR(rule);
3876                 }
3877
3878                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
3879                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3880                         return -EEXIST;
3881                 }
3882                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
3883                         old_rule = rule;
3884                 else
3885                         return -EOPNOTSUPP;
3886         } else {
3887                 if (!(info->nlh->nlmsg_flags & NLM_F_CREATE) ||
3888                     info->nlh->nlmsg_flags & NLM_F_REPLACE)
3889                         return -EINVAL;
3890                 handle = nf_tables_alloc_handle(table);
3891
3892                 if (nla[NFTA_RULE_POSITION]) {
3893                         pos_handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_POSITION]));
3894                         old_rule = __nft_rule_lookup(chain, pos_handle);
3895                         if (IS_ERR(old_rule)) {
3896                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION]);
3897                                 return PTR_ERR(old_rule);
3898                         }
3899                 } else if (nla[NFTA_RULE_POSITION_ID]) {
3900                         old_rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_POSITION_ID]);
3901                         if (IS_ERR(old_rule)) {
3902                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION_ID]);
3903                                 return PTR_ERR(old_rule);
3904                         }
3905                 }
3906         }
3907
3908         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
3909
3910         n = 0;
3911         size = 0;
3912         if (nla[NFTA_RULE_EXPRESSIONS]) {
3913                 expr_info = kvmalloc_array(NFT_RULE_MAXEXPRS,
3914                                            sizeof(struct nft_expr_info),
3915                                            GFP_KERNEL);
3916                 if (!expr_info)
3917                         return -ENOMEM;
3918
3919                 nla_for_each_nested(tmp, nla[NFTA_RULE_EXPRESSIONS], rem) {
3920                         err = -EINVAL;
3921                         if (nla_type(tmp) != NFTA_LIST_ELEM)
3922                                 goto err_release_expr;
3923                         if (n == NFT_RULE_MAXEXPRS)
3924                                 goto err_release_expr;
3925                         err = nf_tables_expr_parse(&ctx, tmp, &expr_info[n]);
3926                         if (err < 0) {
3927                                 NL_SET_BAD_ATTR(extack, tmp);
3928                                 goto err_release_expr;
3929                         }
3930                         size += expr_info[n].ops->size;
3931                         n++;
3932                 }
3933         }
3934         /* Check for overflow of dlen field */
3935         err = -EFBIG;
3936         if (size >= 1 << 12)
3937                 goto err_release_expr;
3938
3939         if (nla[NFTA_RULE_USERDATA]) {
3940                 ulen = nla_len(nla[NFTA_RULE_USERDATA]);
3941                 if (ulen > 0)
3942                         usize = sizeof(struct nft_userdata) + ulen;
3943         }
3944
3945         err = -ENOMEM;
3946         rule = kzalloc(sizeof(*rule) + size + usize, GFP_KERNEL_ACCOUNT);
3947         if (rule == NULL)
3948                 goto err_release_expr;
3949
3950         nft_activate_next(net, rule);
3951
3952         rule->handle = handle;
3953         rule->dlen   = size;
3954         rule->udata  = ulen ? 1 : 0;
3955
3956         if (ulen) {
3957                 udata = nft_userdata(rule);
3958                 udata->len = ulen - 1;
3959                 nla_memcpy(udata->data, nla[NFTA_RULE_USERDATA], ulen);
3960         }
3961
3962         expr = nft_expr_first(rule);
3963         for (i = 0; i < n; i++) {
3964                 err = nf_tables_newexpr(&ctx, &expr_info[i], expr);
3965                 if (err < 0) {
3966                         NL_SET_BAD_ATTR(extack, expr_info[i].attr);
3967                         goto err_release_rule;
3968                 }
3969
3970                 if (expr_info[i].ops->validate)
3971                         nft_validate_state_update(table, NFT_VALIDATE_NEED);
3972
3973                 expr_info[i].ops = NULL;
3974                 expr = nft_expr_next(expr);
3975         }
3976
3977         if (chain->flags & NFT_CHAIN_HW_OFFLOAD) {
3978                 flow = nft_flow_rule_create(net, rule);
3979                 if (IS_ERR(flow)) {
3980                         err = PTR_ERR(flow);
3981                         goto err_release_rule;
3982                 }
3983         }
3984
3985         if (!nft_use_inc(&chain->use)) {
3986                 err = -EMFILE;
3987                 goto err_release_rule;
3988         }
3989
3990         if (info->nlh->nlmsg_flags & NLM_F_REPLACE) {
3991                 if (nft_chain_binding(chain)) {
3992                         err = -EOPNOTSUPP;
3993                         goto err_destroy_flow_rule;
3994                 }
3995
3996                 err = nft_delrule(&ctx, old_rule);
3997                 if (err < 0)
3998                         goto err_destroy_flow_rule;
3999
4000                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
4001                 if (trans == NULL) {
4002                         err = -ENOMEM;
4003                         goto err_destroy_flow_rule;
4004                 }
4005                 list_add_tail_rcu(&rule->list, &old_rule->list);
4006         } else {
4007                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
4008                 if (!trans) {
4009                         err = -ENOMEM;
4010                         goto err_destroy_flow_rule;
4011                 }
4012
4013                 if (info->nlh->nlmsg_flags & NLM_F_APPEND) {
4014                         if (old_rule)
4015                                 list_add_rcu(&rule->list, &old_rule->list);
4016                         else
4017                                 list_add_tail_rcu(&rule->list, &chain->rules);
4018                  } else {
4019                         if (old_rule)
4020                                 list_add_tail_rcu(&rule->list, &old_rule->list);
4021                         else
4022                                 list_add_rcu(&rule->list, &chain->rules);
4023                 }
4024         }
4025         kvfree(expr_info);
4026
4027         if (flow)
4028                 nft_trans_flow_rule(trans) = flow;
4029
4030         if (table->validate_state == NFT_VALIDATE_DO)
4031                 return nft_table_validate(net, table);
4032
4033         return 0;
4034
4035 err_destroy_flow_rule:
4036         nft_use_dec_restore(&chain->use);
4037         if (flow)
4038                 nft_flow_rule_destroy(flow);
4039 err_release_rule:
4040         nft_rule_expr_deactivate(&ctx, rule, NFT_TRANS_PREPARE_ERROR);
4041         nf_tables_rule_destroy(&ctx, rule);
4042 err_release_expr:
4043         for (i = 0; i < n; i++) {
4044                 if (expr_info[i].ops) {
4045                         module_put(expr_info[i].ops->type->owner);
4046                         if (expr_info[i].ops->type->release_ops)
4047                                 expr_info[i].ops->type->release_ops(expr_info[i].ops);
4048                 }
4049         }
4050         kvfree(expr_info);
4051
4052         return err;
4053 }
4054
4055 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
4056                                              const struct nft_chain *chain,
4057                                              const struct nlattr *nla)
4058 {
4059         struct nftables_pernet *nft_net = nft_pernet(net);
4060         u32 id = ntohl(nla_get_be32(nla));
4061         struct nft_trans *trans;
4062
4063         list_for_each_entry(trans, &nft_net->commit_list, list) {
4064                 if (trans->msg_type == NFT_MSG_NEWRULE &&
4065                     trans->ctx.chain == chain &&
4066                     id == nft_trans_rule_id(trans))
4067                         return nft_trans_rule(trans);
4068         }
4069         return ERR_PTR(-ENOENT);
4070 }
4071
4072 static int nf_tables_delrule(struct sk_buff *skb, const struct nfnl_info *info,
4073                              const struct nlattr * const nla[])
4074 {
4075         struct netlink_ext_ack *extack = info->extack;
4076         u8 genmask = nft_genmask_next(info->net);
4077         u8 family = info->nfmsg->nfgen_family;
4078         struct nft_chain *chain = NULL;
4079         struct net *net = info->net;
4080         struct nft_table *table;
4081         struct nft_rule *rule;
4082         struct nft_ctx ctx;
4083         int err = 0;
4084
4085         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
4086                                  NETLINK_CB(skb).portid);
4087         if (IS_ERR(table)) {
4088                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
4089                 return PTR_ERR(table);
4090         }
4091
4092         if (nla[NFTA_RULE_CHAIN]) {
4093                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
4094                                          genmask);
4095                 if (IS_ERR(chain)) {
4096                         if (PTR_ERR(chain) == -ENOENT &&
4097                             NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYRULE)
4098                                 return 0;
4099
4100                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
4101                         return PTR_ERR(chain);
4102                 }
4103                 if (nft_chain_binding(chain))
4104                         return -EOPNOTSUPP;
4105         }
4106
4107         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
4108
4109         if (chain) {
4110                 if (nla[NFTA_RULE_HANDLE]) {
4111                         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
4112                         if (IS_ERR(rule)) {
4113                                 if (PTR_ERR(rule) == -ENOENT &&
4114                                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYRULE)
4115                                         return 0;
4116
4117                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
4118                                 return PTR_ERR(rule);
4119                         }
4120
4121                         err = nft_delrule(&ctx, rule);
4122                 } else if (nla[NFTA_RULE_ID]) {
4123                         rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_ID]);
4124                         if (IS_ERR(rule)) {
4125                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_ID]);
4126                                 return PTR_ERR(rule);
4127                         }
4128
4129                         err = nft_delrule(&ctx, rule);
4130                 } else {
4131                         err = nft_delrule_by_chain(&ctx);
4132                 }
4133         } else {
4134                 list_for_each_entry(chain, &table->chains, list) {
4135                         if (!nft_is_active_next(net, chain))
4136                                 continue;
4137                         if (nft_chain_binding(chain))
4138                                 continue;
4139
4140                         ctx.chain = chain;
4141                         err = nft_delrule_by_chain(&ctx);
4142                         if (err < 0)
4143                                 break;
4144                 }
4145         }
4146
4147         return err;
4148 }
4149
4150 /*
4151  * Sets
4152  */
4153 static const struct nft_set_type *nft_set_types[] = {
4154         &nft_set_hash_fast_type,
4155         &nft_set_hash_type,
4156         &nft_set_rhash_type,
4157         &nft_set_bitmap_type,
4158         &nft_set_rbtree_type,
4159 #if defined(CONFIG_X86_64) && !defined(CONFIG_UML)
4160         &nft_set_pipapo_avx2_type,
4161 #endif
4162         &nft_set_pipapo_type,
4163 };
4164
4165 #define NFT_SET_FEATURES        (NFT_SET_INTERVAL | NFT_SET_MAP | \
4166                                  NFT_SET_TIMEOUT | NFT_SET_OBJECT | \
4167                                  NFT_SET_EVAL)
4168
4169 static bool nft_set_ops_candidate(const struct nft_set_type *type, u32 flags)
4170 {
4171         return (flags & type->features) == (flags & NFT_SET_FEATURES);
4172 }
4173
4174 /*
4175  * Select a set implementation based on the data characteristics and the
4176  * given policy. The total memory use might not be known if no size is
4177  * given, in that case the amount of memory per element is used.
4178  */
4179 static const struct nft_set_ops *
4180 nft_select_set_ops(const struct nft_ctx *ctx,
4181                    const struct nlattr * const nla[],
4182                    const struct nft_set_desc *desc)
4183 {
4184         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
4185         const struct nft_set_ops *ops, *bops;
4186         struct nft_set_estimate est, best;
4187         const struct nft_set_type *type;
4188         u32 flags = 0;
4189         int i;
4190
4191         lockdep_assert_held(&nft_net->commit_mutex);
4192         lockdep_nfnl_nft_mutex_not_held();
4193
4194         if (nla[NFTA_SET_FLAGS] != NULL)
4195                 flags = ntohl(nla_get_be32(nla[NFTA_SET_FLAGS]));
4196
4197         bops        = NULL;
4198         best.size   = ~0;
4199         best.lookup = ~0;
4200         best.space  = ~0;
4201
4202         for (i = 0; i < ARRAY_SIZE(nft_set_types); i++) {
4203                 type = nft_set_types[i];
4204                 ops = &type->ops;
4205
4206                 if (!nft_set_ops_candidate(type, flags))
4207                         continue;
4208                 if (!ops->estimate(desc, flags, &est))
4209                         continue;
4210
4211                 switch (desc->policy) {
4212                 case NFT_SET_POL_PERFORMANCE:
4213                         if (est.lookup < best.lookup)
4214                                 break;
4215                         if (est.lookup == best.lookup &&
4216                             est.space < best.space)
4217                                 break;
4218                         continue;
4219                 case NFT_SET_POL_MEMORY:
4220                         if (!desc->size) {
4221                                 if (est.space < best.space)
4222                                         break;
4223                                 if (est.space == best.space &&
4224                                     est.lookup < best.lookup)
4225                                         break;
4226                         } else if (est.size < best.size || !bops) {
4227                                 break;
4228                         }
4229                         continue;
4230                 default:
4231                         break;
4232                 }
4233
4234                 bops = ops;
4235                 best = est;
4236         }
4237
4238         if (bops != NULL)
4239                 return bops;
4240
4241         return ERR_PTR(-EOPNOTSUPP);
4242 }
4243
4244 static const struct nla_policy nft_set_policy[NFTA_SET_MAX + 1] = {
4245         [NFTA_SET_TABLE]                = { .type = NLA_STRING,
4246                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
4247         [NFTA_SET_NAME]                 = { .type = NLA_STRING,
4248                                             .len = NFT_SET_MAXNAMELEN - 1 },
4249         [NFTA_SET_FLAGS]                = { .type = NLA_U32 },
4250         [NFTA_SET_KEY_TYPE]             = { .type = NLA_U32 },
4251         [NFTA_SET_KEY_LEN]              = { .type = NLA_U32 },
4252         [NFTA_SET_DATA_TYPE]            = { .type = NLA_U32 },
4253         [NFTA_SET_DATA_LEN]             = { .type = NLA_U32 },
4254         [NFTA_SET_POLICY]               = { .type = NLA_U32 },
4255         [NFTA_SET_DESC]                 = { .type = NLA_NESTED },
4256         [NFTA_SET_ID]                   = { .type = NLA_U32 },
4257         [NFTA_SET_TIMEOUT]              = { .type = NLA_U64 },
4258         [NFTA_SET_GC_INTERVAL]          = { .type = NLA_U32 },
4259         [NFTA_SET_USERDATA]             = { .type = NLA_BINARY,
4260                                             .len  = NFT_USERDATA_MAXLEN },
4261         [NFTA_SET_OBJ_TYPE]             = { .type = NLA_U32 },
4262         [NFTA_SET_HANDLE]               = { .type = NLA_U64 },
4263         [NFTA_SET_EXPR]                 = { .type = NLA_NESTED },
4264         [NFTA_SET_EXPRESSIONS]          = { .type = NLA_NESTED },
4265 };
4266
4267 static const struct nla_policy nft_set_desc_policy[NFTA_SET_DESC_MAX + 1] = {
4268         [NFTA_SET_DESC_SIZE]            = { .type = NLA_U32 },
4269         [NFTA_SET_DESC_CONCAT]          = { .type = NLA_NESTED },
4270 };
4271
4272 static struct nft_set *nft_set_lookup(const struct nft_table *table,
4273                                       const struct nlattr *nla, u8 genmask)
4274 {
4275         struct nft_set *set;
4276
4277         if (nla == NULL)
4278                 return ERR_PTR(-EINVAL);
4279
4280         list_for_each_entry_rcu(set, &table->sets, list) {
4281                 if (!nla_strcmp(nla, set->name) &&
4282                     nft_active_genmask(set, genmask))
4283                         return set;
4284         }
4285         return ERR_PTR(-ENOENT);
4286 }
4287
4288 static struct nft_set *nft_set_lookup_byhandle(const struct nft_table *table,
4289                                                const struct nlattr *nla,
4290                                                u8 genmask)
4291 {
4292         struct nft_set *set;
4293
4294         list_for_each_entry(set, &table->sets, list) {
4295                 if (be64_to_cpu(nla_get_be64(nla)) == set->handle &&
4296                     nft_active_genmask(set, genmask))
4297                         return set;
4298         }
4299         return ERR_PTR(-ENOENT);
4300 }
4301
4302 static struct nft_set *nft_set_lookup_byid(const struct net *net,
4303                                            const struct nft_table *table,
4304                                            const struct nlattr *nla, u8 genmask)
4305 {
4306         struct nftables_pernet *nft_net = nft_pernet(net);
4307         u32 id = ntohl(nla_get_be32(nla));
4308         struct nft_trans *trans;
4309
4310         list_for_each_entry(trans, &nft_net->commit_list, list) {
4311                 if (trans->msg_type == NFT_MSG_NEWSET) {
4312                         struct nft_set *set = nft_trans_set(trans);
4313
4314                         if (id == nft_trans_set_id(trans) &&
4315                             set->table == table &&
4316                             nft_active_genmask(set, genmask))
4317                                 return set;
4318                 }
4319         }
4320         return ERR_PTR(-ENOENT);
4321 }
4322
4323 struct nft_set *nft_set_lookup_global(const struct net *net,
4324                                       const struct nft_table *table,
4325                                       const struct nlattr *nla_set_name,
4326                                       const struct nlattr *nla_set_id,
4327                                       u8 genmask)
4328 {
4329         struct nft_set *set;
4330
4331         set = nft_set_lookup(table, nla_set_name, genmask);
4332         if (IS_ERR(set)) {
4333                 if (!nla_set_id)
4334                         return set;
4335
4336                 set = nft_set_lookup_byid(net, table, nla_set_id, genmask);
4337         }
4338         return set;
4339 }
4340 EXPORT_SYMBOL_GPL(nft_set_lookup_global);
4341
4342 static int nf_tables_set_alloc_name(struct nft_ctx *ctx, struct nft_set *set,
4343                                     const char *name)
4344 {
4345         const struct nft_set *i;
4346         const char *p;
4347         unsigned long *inuse;
4348         unsigned int n = 0, min = 0;
4349
4350         p = strchr(name, '%');
4351         if (p != NULL) {
4352                 if (p[1] != 'd' || strchr(p + 2, '%'))
4353                         return -EINVAL;
4354
4355                 if (strnlen(name, NFT_SET_MAX_ANONLEN) >= NFT_SET_MAX_ANONLEN)
4356                         return -EINVAL;
4357
4358                 inuse = (unsigned long *)get_zeroed_page(GFP_KERNEL);
4359                 if (inuse == NULL)
4360                         return -ENOMEM;
4361 cont:
4362                 list_for_each_entry(i, &ctx->table->sets, list) {
4363                         int tmp;
4364
4365                         if (!nft_is_active_next(ctx->net, i))
4366                                 continue;
4367                         if (!sscanf(i->name, name, &tmp))
4368                                 continue;
4369                         if (tmp < min || tmp >= min + BITS_PER_BYTE * PAGE_SIZE)
4370                                 continue;
4371
4372                         set_bit(tmp - min, inuse);
4373                 }
4374
4375                 n = find_first_zero_bit(inuse, BITS_PER_BYTE * PAGE_SIZE);
4376                 if (n >= BITS_PER_BYTE * PAGE_SIZE) {
4377                         min += BITS_PER_BYTE * PAGE_SIZE;
4378                         memset(inuse, 0, PAGE_SIZE);
4379                         goto cont;
4380                 }
4381                 free_page((unsigned long)inuse);
4382         }
4383
4384         set->name = kasprintf(GFP_KERNEL_ACCOUNT, name, min + n);
4385         if (!set->name)
4386                 return -ENOMEM;
4387
4388         list_for_each_entry(i, &ctx->table->sets, list) {
4389                 if (!nft_is_active_next(ctx->net, i))
4390                         continue;
4391                 if (!strcmp(set->name, i->name)) {
4392                         kfree(set->name);
4393                         set->name = NULL;
4394                         return -ENFILE;
4395                 }
4396         }
4397         return 0;
4398 }
4399
4400 int nf_msecs_to_jiffies64(const struct nlattr *nla, u64 *result)
4401 {
4402         u64 ms = be64_to_cpu(nla_get_be64(nla));
4403         u64 max = (u64)(~((u64)0));
4404
4405         max = div_u64(max, NSEC_PER_MSEC);
4406         if (ms >= max)
4407                 return -ERANGE;
4408
4409         ms *= NSEC_PER_MSEC;
4410         *result = nsecs_to_jiffies64(ms);
4411         return 0;
4412 }
4413
4414 __be64 nf_jiffies64_to_msecs(u64 input)
4415 {
4416         return cpu_to_be64(jiffies64_to_msecs(input));
4417 }
4418
4419 static int nf_tables_fill_set_concat(struct sk_buff *skb,
4420                                      const struct nft_set *set)
4421 {
4422         struct nlattr *concat, *field;
4423         int i;
4424
4425         concat = nla_nest_start_noflag(skb, NFTA_SET_DESC_CONCAT);
4426         if (!concat)
4427                 return -ENOMEM;
4428
4429         for (i = 0; i < set->field_count; i++) {
4430                 field = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
4431                 if (!field)
4432                         return -ENOMEM;
4433
4434                 if (nla_put_be32(skb, NFTA_SET_FIELD_LEN,
4435                                  htonl(set->field_len[i])))
4436                         return -ENOMEM;
4437
4438                 nla_nest_end(skb, field);
4439         }
4440
4441         nla_nest_end(skb, concat);
4442
4443         return 0;
4444 }
4445
4446 static int nf_tables_fill_set(struct sk_buff *skb, const struct nft_ctx *ctx,
4447                               const struct nft_set *set, u16 event, u16 flags)
4448 {
4449         u64 timeout = READ_ONCE(set->timeout);
4450         u32 gc_int = READ_ONCE(set->gc_int);
4451         u32 portid = ctx->portid;
4452         struct nlmsghdr *nlh;
4453         struct nlattr *nest;
4454         u32 seq = ctx->seq;
4455         int i;
4456
4457         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
4458         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
4459                            NFNETLINK_V0, nft_base_seq(ctx->net));
4460         if (!nlh)
4461                 goto nla_put_failure;
4462
4463         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
4464                 goto nla_put_failure;
4465         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
4466                 goto nla_put_failure;
4467         if (nla_put_be64(skb, NFTA_SET_HANDLE, cpu_to_be64(set->handle),
4468                          NFTA_SET_PAD))
4469                 goto nla_put_failure;
4470
4471         if (event == NFT_MSG_DELSET) {
4472                 nlmsg_end(skb, nlh);
4473                 return 0;
4474         }
4475
4476         if (set->flags != 0)
4477                 if (nla_put_be32(skb, NFTA_SET_FLAGS, htonl(set->flags)))
4478                         goto nla_put_failure;
4479
4480         if (nla_put_be32(skb, NFTA_SET_KEY_TYPE, htonl(set->ktype)))
4481                 goto nla_put_failure;
4482         if (nla_put_be32(skb, NFTA_SET_KEY_LEN, htonl(set->klen)))
4483                 goto nla_put_failure;
4484         if (set->flags & NFT_SET_MAP) {
4485                 if (nla_put_be32(skb, NFTA_SET_DATA_TYPE, htonl(set->dtype)))
4486                         goto nla_put_failure;
4487                 if (nla_put_be32(skb, NFTA_SET_DATA_LEN, htonl(set->dlen)))
4488                         goto nla_put_failure;
4489         }
4490         if (set->flags & NFT_SET_OBJECT &&
4491             nla_put_be32(skb, NFTA_SET_OBJ_TYPE, htonl(set->objtype)))
4492                 goto nla_put_failure;
4493
4494         if (timeout &&
4495             nla_put_be64(skb, NFTA_SET_TIMEOUT,
4496                          nf_jiffies64_to_msecs(timeout),
4497                          NFTA_SET_PAD))
4498                 goto nla_put_failure;
4499         if (gc_int &&
4500             nla_put_be32(skb, NFTA_SET_GC_INTERVAL, htonl(gc_int)))
4501                 goto nla_put_failure;
4502
4503         if (set->policy != NFT_SET_POL_PERFORMANCE) {
4504                 if (nla_put_be32(skb, NFTA_SET_POLICY, htonl(set->policy)))
4505                         goto nla_put_failure;
4506         }
4507
4508         if (set->udata &&
4509             nla_put(skb, NFTA_SET_USERDATA, set->udlen, set->udata))
4510                 goto nla_put_failure;
4511
4512         nest = nla_nest_start_noflag(skb, NFTA_SET_DESC);
4513         if (!nest)
4514                 goto nla_put_failure;
4515         if (set->size &&
4516             nla_put_be32(skb, NFTA_SET_DESC_SIZE, htonl(set->size)))
4517                 goto nla_put_failure;
4518
4519         if (set->field_count > 1 &&
4520             nf_tables_fill_set_concat(skb, set))
4521                 goto nla_put_failure;
4522
4523         nla_nest_end(skb, nest);
4524
4525         if (set->num_exprs == 1) {
4526                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPR);
4527                 if (nf_tables_fill_expr_info(skb, set->exprs[0], false) < 0)
4528                         goto nla_put_failure;
4529
4530                 nla_nest_end(skb, nest);
4531         } else if (set->num_exprs > 1) {
4532                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPRESSIONS);
4533                 if (nest == NULL)
4534                         goto nla_put_failure;
4535
4536                 for (i = 0; i < set->num_exprs; i++) {
4537                         if (nft_expr_dump(skb, NFTA_LIST_ELEM,
4538                                           set->exprs[i], false) < 0)
4539                                 goto nla_put_failure;
4540                 }
4541                 nla_nest_end(skb, nest);
4542         }
4543
4544         nlmsg_end(skb, nlh);
4545         return 0;
4546
4547 nla_put_failure:
4548         nlmsg_trim(skb, nlh);
4549         return -1;
4550 }
4551
4552 static void nf_tables_set_notify(const struct nft_ctx *ctx,
4553                                  const struct nft_set *set, int event,
4554                                  gfp_t gfp_flags)
4555 {
4556         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
4557         u32 portid = ctx->portid;
4558         struct sk_buff *skb;
4559         u16 flags = 0;
4560         int err;
4561
4562         if (!ctx->report &&
4563             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
4564                 return;
4565
4566         skb = nlmsg_new(NLMSG_GOODSIZE, gfp_flags);
4567         if (skb == NULL)
4568                 goto err;
4569
4570         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
4571                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
4572
4573         err = nf_tables_fill_set(skb, ctx, set, event, flags);
4574         if (err < 0) {
4575                 kfree_skb(skb);
4576                 goto err;
4577         }
4578
4579         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
4580         return;
4581 err:
4582         nfnetlink_set_err(ctx->net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
4583 }
4584
4585 static int nf_tables_dump_sets(struct sk_buff *skb, struct netlink_callback *cb)
4586 {
4587         const struct nft_set *set;
4588         unsigned int idx, s_idx = cb->args[0];
4589         struct nft_table *table, *cur_table = (struct nft_table *)cb->args[2];
4590         struct net *net = sock_net(skb->sk);
4591         struct nft_ctx *ctx = cb->data, ctx_set;
4592         struct nftables_pernet *nft_net;
4593
4594         if (cb->args[1])
4595                 return skb->len;
4596
4597         rcu_read_lock();
4598         nft_net = nft_pernet(net);
4599         cb->seq = READ_ONCE(nft_net->base_seq);
4600
4601         list_for_each_entry_rcu(table, &nft_net->tables, list) {
4602                 if (ctx->family != NFPROTO_UNSPEC &&
4603                     ctx->family != table->family)
4604                         continue;
4605
4606                 if (ctx->table && ctx->table != table)
4607                         continue;
4608
4609                 if (cur_table) {
4610                         if (cur_table != table)
4611                                 continue;
4612
4613                         cur_table = NULL;
4614                 }
4615                 idx = 0;
4616                 list_for_each_entry_rcu(set, &table->sets, list) {
4617                         if (idx < s_idx)
4618                                 goto cont;
4619                         if (!nft_is_active(net, set))
4620                                 goto cont;
4621
4622                         ctx_set = *ctx;
4623                         ctx_set.table = table;
4624                         ctx_set.family = table->family;
4625
4626                         if (nf_tables_fill_set(skb, &ctx_set, set,
4627                                                NFT_MSG_NEWSET,
4628                                                NLM_F_MULTI) < 0) {
4629                                 cb->args[0] = idx;
4630                                 cb->args[2] = (unsigned long) table;
4631                                 goto done;
4632                         }
4633                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
4634 cont:
4635                         idx++;
4636                 }
4637                 if (s_idx)
4638                         s_idx = 0;
4639         }
4640         cb->args[1] = 1;
4641 done:
4642         rcu_read_unlock();
4643         return skb->len;
4644 }
4645
4646 static int nf_tables_dump_sets_start(struct netlink_callback *cb)
4647 {
4648         struct nft_ctx *ctx_dump = NULL;
4649
4650         ctx_dump = kmemdup(cb->data, sizeof(*ctx_dump), GFP_ATOMIC);
4651         if (ctx_dump == NULL)
4652                 return -ENOMEM;
4653
4654         cb->data = ctx_dump;
4655         return 0;
4656 }
4657
4658 static int nf_tables_dump_sets_done(struct netlink_callback *cb)
4659 {
4660         kfree(cb->data);
4661         return 0;
4662 }
4663
4664 /* called with rcu_read_lock held */
4665 static int nf_tables_getset(struct sk_buff *skb, const struct nfnl_info *info,
4666                             const struct nlattr * const nla[])
4667 {
4668         struct netlink_ext_ack *extack = info->extack;
4669         u8 genmask = nft_genmask_cur(info->net);
4670         u8 family = info->nfmsg->nfgen_family;
4671         struct nft_table *table = NULL;
4672         struct net *net = info->net;
4673         const struct nft_set *set;
4674         struct sk_buff *skb2;
4675         struct nft_ctx ctx;
4676         int err;
4677
4678         if (nla[NFTA_SET_TABLE]) {
4679                 table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
4680                                          genmask, 0);
4681                 if (IS_ERR(table)) {
4682                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
4683                         return PTR_ERR(table);
4684                 }
4685         }
4686
4687         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
4688
4689         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
4690                 struct netlink_dump_control c = {
4691                         .start = nf_tables_dump_sets_start,
4692                         .dump = nf_tables_dump_sets,
4693                         .done = nf_tables_dump_sets_done,
4694                         .data = &ctx,
4695                         .module = THIS_MODULE,
4696                 };
4697
4698                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
4699         }
4700
4701         /* Only accept unspec with dump */
4702         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
4703                 return -EAFNOSUPPORT;
4704         if (!nla[NFTA_SET_TABLE])
4705                 return -EINVAL;
4706
4707         set = nft_set_lookup(table, nla[NFTA_SET_NAME], genmask);
4708         if (IS_ERR(set))
4709                 return PTR_ERR(set);
4710
4711         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
4712         if (skb2 == NULL)
4713                 return -ENOMEM;
4714
4715         err = nf_tables_fill_set(skb2, &ctx, set, NFT_MSG_NEWSET, 0);
4716         if (err < 0)
4717                 goto err_fill_set_info;
4718
4719         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
4720
4721 err_fill_set_info:
4722         kfree_skb(skb2);
4723         return err;
4724 }
4725
4726 static const struct nla_policy nft_concat_policy[NFTA_SET_FIELD_MAX + 1] = {
4727         [NFTA_SET_FIELD_LEN]    = { .type = NLA_U32 },
4728 };
4729
4730 static int nft_set_desc_concat_parse(const struct nlattr *attr,
4731                                      struct nft_set_desc *desc)
4732 {
4733         struct nlattr *tb[NFTA_SET_FIELD_MAX + 1];
4734         u32 len;
4735         int err;
4736
4737         if (desc->field_count >= ARRAY_SIZE(desc->field_len))
4738                 return -E2BIG;
4739
4740         err = nla_parse_nested_deprecated(tb, NFTA_SET_FIELD_MAX, attr,
4741                                           nft_concat_policy, NULL);
4742         if (err < 0)
4743                 return err;
4744
4745         if (!tb[NFTA_SET_FIELD_LEN])
4746                 return -EINVAL;
4747
4748         len = ntohl(nla_get_be32(tb[NFTA_SET_FIELD_LEN]));
4749         if (!len || len > U8_MAX)
4750                 return -EINVAL;
4751
4752         desc->field_len[desc->field_count++] = len;
4753
4754         return 0;
4755 }
4756
4757 static int nft_set_desc_concat(struct nft_set_desc *desc,
4758                                const struct nlattr *nla)
4759 {
4760         u32 num_regs = 0, key_num_regs = 0;
4761         struct nlattr *attr;
4762         int rem, err, i;
4763
4764         nla_for_each_nested(attr, nla, rem) {
4765                 if (nla_type(attr) != NFTA_LIST_ELEM)
4766                         return -EINVAL;
4767
4768                 err = nft_set_desc_concat_parse(attr, desc);
4769                 if (err < 0)
4770                         return err;
4771         }
4772
4773         for (i = 0; i < desc->field_count; i++)
4774                 num_regs += DIV_ROUND_UP(desc->field_len[i], sizeof(u32));
4775
4776         key_num_regs = DIV_ROUND_UP(desc->klen, sizeof(u32));
4777         if (key_num_regs != num_regs)
4778                 return -EINVAL;
4779
4780         if (num_regs > NFT_REG32_COUNT)
4781                 return -E2BIG;
4782
4783         return 0;
4784 }
4785
4786 static int nf_tables_set_desc_parse(struct nft_set_desc *desc,
4787                                     const struct nlattr *nla)
4788 {
4789         struct nlattr *da[NFTA_SET_DESC_MAX + 1];
4790         int err;
4791
4792         err = nla_parse_nested_deprecated(da, NFTA_SET_DESC_MAX, nla,
4793                                           nft_set_desc_policy, NULL);
4794         if (err < 0)
4795                 return err;
4796
4797         if (da[NFTA_SET_DESC_SIZE] != NULL)
4798                 desc->size = ntohl(nla_get_be32(da[NFTA_SET_DESC_SIZE]));
4799         if (da[NFTA_SET_DESC_CONCAT])
4800                 err = nft_set_desc_concat(desc, da[NFTA_SET_DESC_CONCAT]);
4801
4802         return err;
4803 }
4804
4805 static int nft_set_expr_alloc(struct nft_ctx *ctx, struct nft_set *set,
4806                               const struct nlattr * const *nla,
4807                               struct nft_expr **exprs, int *num_exprs,
4808                               u32 flags)
4809 {
4810         struct nft_expr *expr;
4811         int err, i;
4812
4813         if (nla[NFTA_SET_EXPR]) {
4814                 expr = nft_set_elem_expr_alloc(ctx, set, nla[NFTA_SET_EXPR]);
4815                 if (IS_ERR(expr)) {
4816                         err = PTR_ERR(expr);
4817                         goto err_set_expr_alloc;
4818                 }
4819                 exprs[0] = expr;
4820                 (*num_exprs)++;
4821         } else if (nla[NFTA_SET_EXPRESSIONS]) {
4822                 struct nlattr *tmp;
4823                 int left;
4824
4825                 if (!(flags & NFT_SET_EXPR)) {
4826                         err = -EINVAL;
4827                         goto err_set_expr_alloc;
4828                 }
4829                 i = 0;
4830                 nla_for_each_nested(tmp, nla[NFTA_SET_EXPRESSIONS], left) {
4831                         if (i == NFT_SET_EXPR_MAX) {
4832                                 err = -E2BIG;
4833                                 goto err_set_expr_alloc;
4834                         }
4835                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
4836                                 err = -EINVAL;
4837                                 goto err_set_expr_alloc;
4838                         }
4839                         expr = nft_set_elem_expr_alloc(ctx, set, tmp);
4840                         if (IS_ERR(expr)) {
4841                                 err = PTR_ERR(expr);
4842                                 goto err_set_expr_alloc;
4843                         }
4844                         exprs[i++] = expr;
4845                         (*num_exprs)++;
4846                 }
4847         }
4848
4849         return 0;
4850
4851 err_set_expr_alloc:
4852         for (i = 0; i < *num_exprs; i++)
4853                 nft_expr_destroy(ctx, exprs[i]);
4854
4855         return err;
4856 }
4857
4858 static bool nft_set_is_same(const struct nft_set *set,
4859                             const struct nft_set_desc *desc,
4860                             struct nft_expr *exprs[], u32 num_exprs, u32 flags)
4861 {
4862         int i;
4863
4864         if (set->ktype != desc->ktype ||
4865             set->dtype != desc->dtype ||
4866             set->flags != flags ||
4867             set->klen != desc->klen ||
4868             set->dlen != desc->dlen ||
4869             set->field_count != desc->field_count ||
4870             set->num_exprs != num_exprs)
4871                 return false;
4872
4873         for (i = 0; i < desc->field_count; i++) {
4874                 if (set->field_len[i] != desc->field_len[i])
4875                         return false;
4876         }
4877
4878         for (i = 0; i < num_exprs; i++) {
4879                 if (set->exprs[i]->ops != exprs[i]->ops)
4880                         return false;
4881         }
4882
4883         return true;
4884 }
4885
4886 static int nf_tables_newset(struct sk_buff *skb, const struct nfnl_info *info,
4887                             const struct nlattr * const nla[])
4888 {
4889         struct netlink_ext_ack *extack = info->extack;
4890         u8 genmask = nft_genmask_next(info->net);
4891         u8 family = info->nfmsg->nfgen_family;
4892         const struct nft_set_ops *ops;
4893         struct net *net = info->net;
4894         struct nft_set_desc desc;
4895         struct nft_table *table;
4896         unsigned char *udata;
4897         struct nft_set *set;
4898         struct nft_ctx ctx;
4899         size_t alloc_size;
4900         int num_exprs = 0;
4901         char *name;
4902         int err, i;
4903         u16 udlen;
4904         u32 flags;
4905         u64 size;
4906
4907         if (nla[NFTA_SET_TABLE] == NULL ||
4908             nla[NFTA_SET_NAME] == NULL ||
4909             nla[NFTA_SET_KEY_LEN] == NULL ||
4910             nla[NFTA_SET_ID] == NULL)
4911                 return -EINVAL;
4912
4913         memset(&desc, 0, sizeof(desc));
4914
4915         desc.ktype = NFT_DATA_VALUE;
4916         if (nla[NFTA_SET_KEY_TYPE] != NULL) {
4917                 desc.ktype = ntohl(nla_get_be32(nla[NFTA_SET_KEY_TYPE]));
4918                 if ((desc.ktype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK)
4919                         return -EINVAL;
4920         }
4921
4922         desc.klen = ntohl(nla_get_be32(nla[NFTA_SET_KEY_LEN]));
4923         if (desc.klen == 0 || desc.klen > NFT_DATA_VALUE_MAXLEN)
4924                 return -EINVAL;
4925
4926         flags = 0;
4927         if (nla[NFTA_SET_FLAGS] != NULL) {
4928                 flags = ntohl(nla_get_be32(nla[NFTA_SET_FLAGS]));
4929                 if (flags & ~(NFT_SET_ANONYMOUS | NFT_SET_CONSTANT |
4930                               NFT_SET_INTERVAL | NFT_SET_TIMEOUT |
4931                               NFT_SET_MAP | NFT_SET_EVAL |
4932                               NFT_SET_OBJECT | NFT_SET_CONCAT | NFT_SET_EXPR))
4933                         return -EOPNOTSUPP;
4934                 /* Only one of these operations is supported */
4935                 if ((flags & (NFT_SET_MAP | NFT_SET_OBJECT)) ==
4936                              (NFT_SET_MAP | NFT_SET_OBJECT))
4937                         return -EOPNOTSUPP;
4938                 if ((flags & (NFT_SET_EVAL | NFT_SET_OBJECT)) ==
4939                              (NFT_SET_EVAL | NFT_SET_OBJECT))
4940                         return -EOPNOTSUPP;
4941         }
4942
4943         desc.dtype = 0;
4944         if (nla[NFTA_SET_DATA_TYPE] != NULL) {
4945                 if (!(flags & NFT_SET_MAP))
4946                         return -EINVAL;
4947
4948                 desc.dtype = ntohl(nla_get_be32(nla[NFTA_SET_DATA_TYPE]));
4949                 if ((desc.dtype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK &&
4950                     desc.dtype != NFT_DATA_VERDICT)
4951                         return -EINVAL;
4952
4953                 if (desc.dtype != NFT_DATA_VERDICT) {
4954                         if (nla[NFTA_SET_DATA_LEN] == NULL)
4955                                 return -EINVAL;
4956                         desc.dlen = ntohl(nla_get_be32(nla[NFTA_SET_DATA_LEN]));
4957                         if (desc.dlen == 0 || desc.dlen > NFT_DATA_VALUE_MAXLEN)
4958                                 return -EINVAL;
4959                 } else
4960                         desc.dlen = sizeof(struct nft_verdict);
4961         } else if (flags & NFT_SET_MAP)
4962                 return -EINVAL;
4963
4964         if (nla[NFTA_SET_OBJ_TYPE] != NULL) {
4965                 if (!(flags & NFT_SET_OBJECT))
4966                         return -EINVAL;
4967
4968                 desc.objtype = ntohl(nla_get_be32(nla[NFTA_SET_OBJ_TYPE]));
4969                 if (desc.objtype == NFT_OBJECT_UNSPEC ||
4970                     desc.objtype > NFT_OBJECT_MAX)
4971                         return -EOPNOTSUPP;
4972         } else if (flags & NFT_SET_OBJECT)
4973                 return -EINVAL;
4974         else
4975                 desc.objtype = NFT_OBJECT_UNSPEC;
4976
4977         desc.timeout = 0;
4978         if (nla[NFTA_SET_TIMEOUT] != NULL) {
4979                 if (!(flags & NFT_SET_TIMEOUT))
4980                         return -EINVAL;
4981
4982                 if (flags & NFT_SET_ANONYMOUS)
4983                         return -EOPNOTSUPP;
4984
4985                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_TIMEOUT], &desc.timeout);
4986                 if (err)
4987                         return err;
4988         }
4989         desc.gc_int = 0;
4990         if (nla[NFTA_SET_GC_INTERVAL] != NULL) {
4991                 if (!(flags & NFT_SET_TIMEOUT))
4992                         return -EINVAL;
4993
4994                 if (flags & NFT_SET_ANONYMOUS)
4995                         return -EOPNOTSUPP;
4996
4997                 desc.gc_int = ntohl(nla_get_be32(nla[NFTA_SET_GC_INTERVAL]));
4998         }
4999
5000         desc.policy = NFT_SET_POL_PERFORMANCE;
5001         if (nla[NFTA_SET_POLICY] != NULL) {
5002                 desc.policy = ntohl(nla_get_be32(nla[NFTA_SET_POLICY]));
5003                 switch (desc.policy) {
5004                 case NFT_SET_POL_PERFORMANCE:
5005                 case NFT_SET_POL_MEMORY:
5006                         break;
5007                 default:
5008                         return -EOPNOTSUPP;
5009                 }
5010         }
5011
5012         if (nla[NFTA_SET_DESC] != NULL) {
5013                 err = nf_tables_set_desc_parse(&desc, nla[NFTA_SET_DESC]);
5014                 if (err < 0)
5015                         return err;
5016
5017                 if (desc.field_count > 1) {
5018                         if (!(flags & NFT_SET_CONCAT))
5019                                 return -EINVAL;
5020                 } else if (flags & NFT_SET_CONCAT) {
5021                         return -EINVAL;
5022                 }
5023         } else if (flags & NFT_SET_CONCAT) {
5024                 return -EINVAL;
5025         }
5026
5027         if (nla[NFTA_SET_EXPR] || nla[NFTA_SET_EXPRESSIONS])
5028                 desc.expr = true;
5029
5030         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family, genmask,
5031                                  NETLINK_CB(skb).portid);
5032         if (IS_ERR(table)) {
5033                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
5034                 return PTR_ERR(table);
5035         }
5036
5037         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
5038
5039         set = nft_set_lookup(table, nla[NFTA_SET_NAME], genmask);
5040         if (IS_ERR(set)) {
5041                 if (PTR_ERR(set) != -ENOENT) {
5042                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
5043                         return PTR_ERR(set);
5044                 }
5045         } else {
5046                 struct nft_expr *exprs[NFT_SET_EXPR_MAX] = {};
5047
5048                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
5049                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
5050                         return -EEXIST;
5051                 }
5052                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
5053                         return -EOPNOTSUPP;
5054
5055                 if (nft_set_is_anonymous(set))
5056                         return -EOPNOTSUPP;
5057
5058                 err = nft_set_expr_alloc(&ctx, set, nla, exprs, &num_exprs, flags);
5059                 if (err < 0)
5060                         return err;
5061
5062                 err = 0;
5063                 if (!nft_set_is_same(set, &desc, exprs, num_exprs, flags)) {
5064                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
5065                         err = -EEXIST;
5066                 }
5067
5068                 for (i = 0; i < num_exprs; i++)
5069                         nft_expr_destroy(&ctx, exprs[i]);
5070
5071                 if (err < 0)
5072                         return err;
5073
5074                 return __nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set, &desc);
5075         }
5076
5077         if (!(info->nlh->nlmsg_flags & NLM_F_CREATE))
5078                 return -ENOENT;
5079
5080         ops = nft_select_set_ops(&ctx, nla, &desc);
5081         if (IS_ERR(ops))
5082                 return PTR_ERR(ops);
5083
5084         udlen = 0;
5085         if (nla[NFTA_SET_USERDATA])
5086                 udlen = nla_len(nla[NFTA_SET_USERDATA]);
5087
5088         size = 0;
5089         if (ops->privsize != NULL)
5090                 size = ops->privsize(nla, &desc);
5091         alloc_size = sizeof(*set) + size + udlen;
5092         if (alloc_size < size || alloc_size > INT_MAX)
5093                 return -ENOMEM;
5094
5095         if (!nft_use_inc(&table->use))
5096                 return -EMFILE;
5097
5098         set = kvzalloc(alloc_size, GFP_KERNEL_ACCOUNT);
5099         if (!set) {
5100                 err = -ENOMEM;
5101                 goto err_alloc;
5102         }
5103
5104         name = nla_strdup(nla[NFTA_SET_NAME], GFP_KERNEL_ACCOUNT);
5105         if (!name) {
5106                 err = -ENOMEM;
5107                 goto err_set_name;
5108         }
5109
5110         err = nf_tables_set_alloc_name(&ctx, set, name);
5111         kfree(name);
5112         if (err < 0)
5113                 goto err_set_name;
5114
5115         udata = NULL;
5116         if (udlen) {
5117                 udata = set->data + size;
5118                 nla_memcpy(udata, nla[NFTA_SET_USERDATA], udlen);
5119         }
5120
5121         INIT_LIST_HEAD(&set->bindings);
5122         INIT_LIST_HEAD(&set->catchall_list);
5123         refcount_set(&set->refs, 1);
5124         set->table = table;
5125         write_pnet(&set->net, net);
5126         set->ops = ops;
5127         set->ktype = desc.ktype;
5128         set->klen = desc.klen;
5129         set->dtype = desc.dtype;
5130         set->objtype = desc.objtype;
5131         set->dlen = desc.dlen;
5132         set->flags = flags;
5133         set->size = desc.size;
5134         set->policy = desc.policy;
5135         set->udlen = udlen;
5136         set->udata = udata;
5137         set->timeout = desc.timeout;
5138         set->gc_int = desc.gc_int;
5139
5140         set->field_count = desc.field_count;
5141         for (i = 0; i < desc.field_count; i++)
5142                 set->field_len[i] = desc.field_len[i];
5143
5144         err = ops->init(set, &desc, nla);
5145         if (err < 0)
5146                 goto err_set_init;
5147
5148         err = nft_set_expr_alloc(&ctx, set, nla, set->exprs, &num_exprs, flags);
5149         if (err < 0)
5150                 goto err_set_destroy;
5151
5152         set->num_exprs = num_exprs;
5153         set->handle = nf_tables_alloc_handle(table);
5154         INIT_LIST_HEAD(&set->pending_update);
5155
5156         err = nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set);
5157         if (err < 0)
5158                 goto err_set_expr_alloc;
5159
5160         list_add_tail_rcu(&set->list, &table->sets);
5161
5162         return 0;
5163
5164 err_set_expr_alloc:
5165         for (i = 0; i < set->num_exprs; i++)
5166                 nft_expr_destroy(&ctx, set->exprs[i]);
5167 err_set_destroy:
5168         ops->destroy(&ctx, set);
5169 err_set_init:
5170         kfree(set->name);
5171 err_set_name:
5172         kvfree(set);
5173 err_alloc:
5174         nft_use_dec_restore(&table->use);
5175
5176         return err;
5177 }
5178
5179 static void nft_set_catchall_destroy(const struct nft_ctx *ctx,
5180                                      struct nft_set *set)
5181 {
5182         struct nft_set_elem_catchall *next, *catchall;
5183
5184         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
5185                 list_del_rcu(&catchall->list);
5186                 nf_tables_set_elem_destroy(ctx, set, catchall->elem);
5187                 kfree_rcu(catchall, rcu);
5188         }
5189 }
5190
5191 static void nft_set_put(struct nft_set *set)
5192 {
5193         if (refcount_dec_and_test(&set->refs)) {
5194                 kfree(set->name);
5195                 kvfree(set);
5196         }
5197 }
5198
5199 static void nft_set_destroy(const struct nft_ctx *ctx, struct nft_set *set)
5200 {
5201         int i;
5202
5203         if (WARN_ON(set->use > 0))
5204                 return;
5205
5206         for (i = 0; i < set->num_exprs; i++)
5207                 nft_expr_destroy(ctx, set->exprs[i]);
5208
5209         set->ops->destroy(ctx, set);
5210         nft_set_catchall_destroy(ctx, set);
5211         nft_set_put(set);
5212 }
5213
5214 static int nf_tables_delset(struct sk_buff *skb, const struct nfnl_info *info,
5215                             const struct nlattr * const nla[])
5216 {
5217         struct netlink_ext_ack *extack = info->extack;
5218         u8 genmask = nft_genmask_next(info->net);
5219         u8 family = info->nfmsg->nfgen_family;
5220         struct net *net = info->net;
5221         const struct nlattr *attr;
5222         struct nft_table *table;
5223         struct nft_set *set;
5224         struct nft_ctx ctx;
5225
5226         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
5227                 return -EAFNOSUPPORT;
5228
5229         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
5230                                  genmask, NETLINK_CB(skb).portid);
5231         if (IS_ERR(table)) {
5232                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
5233                 return PTR_ERR(table);
5234         }
5235
5236         if (nla[NFTA_SET_HANDLE]) {
5237                 attr = nla[NFTA_SET_HANDLE];
5238                 set = nft_set_lookup_byhandle(table, attr, genmask);
5239         } else {
5240                 attr = nla[NFTA_SET_NAME];
5241                 set = nft_set_lookup(table, attr, genmask);
5242         }
5243
5244         if (IS_ERR(set)) {
5245                 if (PTR_ERR(set) == -ENOENT &&
5246                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYSET)
5247                         return 0;
5248
5249                 NL_SET_BAD_ATTR(extack, attr);
5250                 return PTR_ERR(set);
5251         }
5252         if (set->use ||
5253             (info->nlh->nlmsg_flags & NLM_F_NONREC &&
5254              atomic_read(&set->nelems) > 0)) {
5255                 NL_SET_BAD_ATTR(extack, attr);
5256                 return -EBUSY;
5257         }
5258
5259         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
5260
5261         return nft_delset(&ctx, set);
5262 }
5263
5264 static int nft_validate_register_store(const struct nft_ctx *ctx,
5265                                        enum nft_registers reg,
5266                                        const struct nft_data *data,
5267                                        enum nft_data_types type,
5268                                        unsigned int len);
5269
5270 static int nft_setelem_data_validate(const struct nft_ctx *ctx,
5271                                      struct nft_set *set,
5272                                      struct nft_set_elem *elem)
5273 {
5274         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
5275         enum nft_registers dreg;
5276
5277         dreg = nft_type_to_reg(set->dtype);
5278         return nft_validate_register_store(ctx, dreg, nft_set_ext_data(ext),
5279                                            set->dtype == NFT_DATA_VERDICT ?
5280                                            NFT_DATA_VERDICT : NFT_DATA_VALUE,
5281                                            set->dlen);
5282 }
5283
5284 static int nf_tables_bind_check_setelem(const struct nft_ctx *ctx,
5285                                         struct nft_set *set,
5286                                         const struct nft_set_iter *iter,
5287                                         struct nft_set_elem *elem)
5288 {
5289         return nft_setelem_data_validate(ctx, set, elem);
5290 }
5291
5292 static int nft_set_catchall_bind_check(const struct nft_ctx *ctx,
5293                                        struct nft_set *set)
5294 {
5295         u8 genmask = nft_genmask_next(ctx->net);
5296         struct nft_set_elem_catchall *catchall;
5297         struct nft_set_elem elem;
5298         struct nft_set_ext *ext;
5299         int ret = 0;
5300
5301         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5302                 ext = nft_set_elem_ext(set, catchall->elem);
5303                 if (!nft_set_elem_active(ext, genmask))
5304                         continue;
5305
5306                 elem.priv = catchall->elem;
5307                 ret = nft_setelem_data_validate(ctx, set, &elem);
5308                 if (ret < 0)
5309                         break;
5310         }
5311
5312         return ret;
5313 }
5314
5315 int nf_tables_bind_set(const struct nft_ctx *ctx, struct nft_set *set,
5316                        struct nft_set_binding *binding)
5317 {
5318         struct nft_set_binding *i;
5319         struct nft_set_iter iter;
5320
5321         if (!list_empty(&set->bindings) && nft_set_is_anonymous(set))
5322                 return -EBUSY;
5323
5324         if (binding->flags & NFT_SET_MAP) {
5325                 /* If the set is already bound to the same chain all
5326                  * jumps are already validated for that chain.
5327                  */
5328                 list_for_each_entry(i, &set->bindings, list) {
5329                         if (i->flags & NFT_SET_MAP &&
5330                             i->chain == binding->chain)
5331                                 goto bind;
5332                 }
5333
5334                 iter.genmask    = nft_genmask_next(ctx->net);
5335                 iter.skip       = 0;
5336                 iter.count      = 0;
5337                 iter.err        = 0;
5338                 iter.fn         = nf_tables_bind_check_setelem;
5339
5340                 set->ops->walk(ctx, set, &iter);
5341                 if (!iter.err)
5342                         iter.err = nft_set_catchall_bind_check(ctx, set);
5343
5344                 if (iter.err < 0)
5345                         return iter.err;
5346         }
5347 bind:
5348         if (!nft_use_inc(&set->use))
5349                 return -EMFILE;
5350
5351         binding->chain = ctx->chain;
5352         list_add_tail_rcu(&binding->list, &set->bindings);
5353         nft_set_trans_bind(ctx, set);
5354
5355         return 0;
5356 }
5357 EXPORT_SYMBOL_GPL(nf_tables_bind_set);
5358
5359 static void nf_tables_unbind_set(const struct nft_ctx *ctx, struct nft_set *set,
5360                                  struct nft_set_binding *binding, bool event)
5361 {
5362         list_del_rcu(&binding->list);
5363
5364         if (list_empty(&set->bindings) && nft_set_is_anonymous(set)) {
5365                 list_del_rcu(&set->list);
5366                 if (event)
5367                         nf_tables_set_notify(ctx, set, NFT_MSG_DELSET,
5368                                              GFP_KERNEL);
5369         }
5370 }
5371
5372 static void nft_setelem_data_activate(const struct net *net,
5373                                       const struct nft_set *set,
5374                                       struct nft_set_elem *elem);
5375
5376 static int nft_mapelem_activate(const struct nft_ctx *ctx,
5377                                 struct nft_set *set,
5378                                 const struct nft_set_iter *iter,
5379                                 struct nft_set_elem *elem)
5380 {
5381         nft_setelem_data_activate(ctx->net, set, elem);
5382
5383         return 0;
5384 }
5385
5386 static void nft_map_catchall_activate(const struct nft_ctx *ctx,
5387                                       struct nft_set *set)
5388 {
5389         u8 genmask = nft_genmask_next(ctx->net);
5390         struct nft_set_elem_catchall *catchall;
5391         struct nft_set_elem elem;
5392         struct nft_set_ext *ext;
5393
5394         list_for_each_entry(catchall, &set->catchall_list, list) {
5395                 ext = nft_set_elem_ext(set, catchall->elem);
5396                 if (!nft_set_elem_active(ext, genmask))
5397                         continue;
5398
5399                 elem.priv = catchall->elem;
5400                 nft_setelem_data_activate(ctx->net, set, &elem);
5401                 break;
5402         }
5403 }
5404
5405 static void nft_map_activate(const struct nft_ctx *ctx, struct nft_set *set)
5406 {
5407         struct nft_set_iter iter = {
5408                 .genmask        = nft_genmask_next(ctx->net),
5409                 .fn             = nft_mapelem_activate,
5410         };
5411
5412         set->ops->walk(ctx, set, &iter);
5413         WARN_ON_ONCE(iter.err);
5414
5415         nft_map_catchall_activate(ctx, set);
5416 }
5417
5418 void nf_tables_activate_set(const struct nft_ctx *ctx, struct nft_set *set)
5419 {
5420         if (nft_set_is_anonymous(set)) {
5421                 if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
5422                         nft_map_activate(ctx, set);
5423
5424                 nft_clear(ctx->net, set);
5425         }
5426
5427         nft_use_inc_restore(&set->use);
5428 }
5429 EXPORT_SYMBOL_GPL(nf_tables_activate_set);
5430
5431 void nf_tables_deactivate_set(const struct nft_ctx *ctx, struct nft_set *set,
5432                               struct nft_set_binding *binding,
5433                               enum nft_trans_phase phase)
5434 {
5435         switch (phase) {
5436         case NFT_TRANS_PREPARE_ERROR:
5437                 nft_set_trans_unbind(ctx, set);
5438                 if (nft_set_is_anonymous(set))
5439                         nft_deactivate_next(ctx->net, set);
5440                 else
5441                         list_del_rcu(&binding->list);
5442
5443                 nft_use_dec(&set->use);
5444                 break;
5445         case NFT_TRANS_PREPARE:
5446                 if (nft_set_is_anonymous(set)) {
5447                         if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
5448                                 nft_map_deactivate(ctx, set);
5449
5450                         nft_deactivate_next(ctx->net, set);
5451                 }
5452                 nft_use_dec(&set->use);
5453                 return;
5454         case NFT_TRANS_ABORT:
5455         case NFT_TRANS_RELEASE:
5456                 if (nft_set_is_anonymous(set) &&
5457                     set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
5458                         nft_map_deactivate(ctx, set);
5459
5460                 nft_use_dec(&set->use);
5461                 fallthrough;
5462         default:
5463                 nf_tables_unbind_set(ctx, set, binding,
5464                                      phase == NFT_TRANS_COMMIT);
5465         }
5466 }
5467 EXPORT_SYMBOL_GPL(nf_tables_deactivate_set);
5468
5469 void nf_tables_destroy_set(const struct nft_ctx *ctx, struct nft_set *set)
5470 {
5471         if (list_empty(&set->bindings) && nft_set_is_anonymous(set))
5472                 nft_set_destroy(ctx, set);
5473 }
5474 EXPORT_SYMBOL_GPL(nf_tables_destroy_set);
5475
5476 const struct nft_set_ext_type nft_set_ext_types[] = {
5477         [NFT_SET_EXT_KEY]               = {
5478                 .align  = __alignof__(u32),
5479         },
5480         [NFT_SET_EXT_DATA]              = {
5481                 .align  = __alignof__(u32),
5482         },
5483         [NFT_SET_EXT_EXPRESSIONS]       = {
5484                 .align  = __alignof__(struct nft_set_elem_expr),
5485         },
5486         [NFT_SET_EXT_OBJREF]            = {
5487                 .len    = sizeof(struct nft_object *),
5488                 .align  = __alignof__(struct nft_object *),
5489         },
5490         [NFT_SET_EXT_FLAGS]             = {
5491                 .len    = sizeof(u8),
5492                 .align  = __alignof__(u8),
5493         },
5494         [NFT_SET_EXT_TIMEOUT]           = {
5495                 .len    = sizeof(u64),
5496                 .align  = __alignof__(u64),
5497         },
5498         [NFT_SET_EXT_EXPIRATION]        = {
5499                 .len    = sizeof(u64),
5500                 .align  = __alignof__(u64),
5501         },
5502         [NFT_SET_EXT_USERDATA]          = {
5503                 .len    = sizeof(struct nft_userdata),
5504                 .align  = __alignof__(struct nft_userdata),
5505         },
5506         [NFT_SET_EXT_KEY_END]           = {
5507                 .align  = __alignof__(u32),
5508         },
5509 };
5510
5511 /*
5512  * Set elements
5513  */
5514
5515 static const struct nla_policy nft_set_elem_policy[NFTA_SET_ELEM_MAX + 1] = {
5516         [NFTA_SET_ELEM_KEY]             = { .type = NLA_NESTED },
5517         [NFTA_SET_ELEM_DATA]            = { .type = NLA_NESTED },
5518         [NFTA_SET_ELEM_FLAGS]           = { .type = NLA_U32 },
5519         [NFTA_SET_ELEM_TIMEOUT]         = { .type = NLA_U64 },
5520         [NFTA_SET_ELEM_EXPIRATION]      = { .type = NLA_U64 },
5521         [NFTA_SET_ELEM_USERDATA]        = { .type = NLA_BINARY,
5522                                             .len = NFT_USERDATA_MAXLEN },
5523         [NFTA_SET_ELEM_EXPR]            = { .type = NLA_NESTED },
5524         [NFTA_SET_ELEM_OBJREF]          = { .type = NLA_STRING,
5525                                             .len = NFT_OBJ_MAXNAMELEN - 1 },
5526         [NFTA_SET_ELEM_KEY_END]         = { .type = NLA_NESTED },
5527         [NFTA_SET_ELEM_EXPRESSIONS]     = { .type = NLA_NESTED },
5528 };
5529
5530 static const struct nla_policy nft_set_elem_list_policy[NFTA_SET_ELEM_LIST_MAX + 1] = {
5531         [NFTA_SET_ELEM_LIST_TABLE]      = { .type = NLA_STRING,
5532                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
5533         [NFTA_SET_ELEM_LIST_SET]        = { .type = NLA_STRING,
5534                                             .len = NFT_SET_MAXNAMELEN - 1 },
5535         [NFTA_SET_ELEM_LIST_ELEMENTS]   = { .type = NLA_NESTED },
5536         [NFTA_SET_ELEM_LIST_SET_ID]     = { .type = NLA_U32 },
5537 };
5538
5539 static int nft_set_elem_expr_dump(struct sk_buff *skb,
5540                                   const struct nft_set *set,
5541                                   const struct nft_set_ext *ext,
5542                                   bool reset)
5543 {
5544         struct nft_set_elem_expr *elem_expr;
5545         u32 size, num_exprs = 0;
5546         struct nft_expr *expr;
5547         struct nlattr *nest;
5548
5549         elem_expr = nft_set_ext_expr(ext);
5550         nft_setelem_expr_foreach(expr, elem_expr, size)
5551                 num_exprs++;
5552
5553         if (num_exprs == 1) {
5554                 expr = nft_setelem_expr_at(elem_expr, 0);
5555                 if (nft_expr_dump(skb, NFTA_SET_ELEM_EXPR, expr, reset) < 0)
5556                         return -1;
5557
5558                 return 0;
5559         } else if (num_exprs > 1) {
5560                 nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_EXPRESSIONS);
5561                 if (nest == NULL)
5562                         goto nla_put_failure;
5563
5564                 nft_setelem_expr_foreach(expr, elem_expr, size) {
5565                         expr = nft_setelem_expr_at(elem_expr, size);
5566                         if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr, reset) < 0)
5567                                 goto nla_put_failure;
5568                 }
5569                 nla_nest_end(skb, nest);
5570         }
5571         return 0;
5572
5573 nla_put_failure:
5574         return -1;
5575 }
5576
5577 static int nf_tables_fill_setelem(struct sk_buff *skb,
5578                                   const struct nft_set *set,
5579                                   const struct nft_set_elem *elem,
5580                                   bool reset)
5581 {
5582         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
5583         unsigned char *b = skb_tail_pointer(skb);
5584         struct nlattr *nest;
5585
5586         nest = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
5587         if (nest == NULL)
5588                 goto nla_put_failure;
5589
5590         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
5591             nft_data_dump(skb, NFTA_SET_ELEM_KEY, nft_set_ext_key(ext),
5592                           NFT_DATA_VALUE, set->klen) < 0)
5593                 goto nla_put_failure;
5594
5595         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
5596             nft_data_dump(skb, NFTA_SET_ELEM_KEY_END, nft_set_ext_key_end(ext),
5597                           NFT_DATA_VALUE, set->klen) < 0)
5598                 goto nla_put_failure;
5599
5600         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
5601             nft_data_dump(skb, NFTA_SET_ELEM_DATA, nft_set_ext_data(ext),
5602                           set->dtype == NFT_DATA_VERDICT ? NFT_DATA_VERDICT : NFT_DATA_VALUE,
5603                           set->dlen) < 0)
5604                 goto nla_put_failure;
5605
5606         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS) &&
5607             nft_set_elem_expr_dump(skb, set, ext, reset))
5608                 goto nla_put_failure;
5609
5610         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
5611             nla_put_string(skb, NFTA_SET_ELEM_OBJREF,
5612                            (*nft_set_ext_obj(ext))->key.name) < 0)
5613                 goto nla_put_failure;
5614
5615         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
5616             nla_put_be32(skb, NFTA_SET_ELEM_FLAGS,
5617                          htonl(*nft_set_ext_flags(ext))))
5618                 goto nla_put_failure;
5619
5620         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT) &&
5621             nla_put_be64(skb, NFTA_SET_ELEM_TIMEOUT,
5622                          nf_jiffies64_to_msecs(*nft_set_ext_timeout(ext)),
5623                          NFTA_SET_ELEM_PAD))
5624                 goto nla_put_failure;
5625
5626         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
5627                 u64 expires, now = get_jiffies_64();
5628
5629                 expires = *nft_set_ext_expiration(ext);
5630                 if (time_before64(now, expires))
5631                         expires -= now;
5632                 else
5633                         expires = 0;
5634
5635                 if (nla_put_be64(skb, NFTA_SET_ELEM_EXPIRATION,
5636                                  nf_jiffies64_to_msecs(expires),
5637                                  NFTA_SET_ELEM_PAD))
5638                         goto nla_put_failure;
5639         }
5640
5641         if (nft_set_ext_exists(ext, NFT_SET_EXT_USERDATA)) {
5642                 struct nft_userdata *udata;
5643
5644                 udata = nft_set_ext_userdata(ext);
5645                 if (nla_put(skb, NFTA_SET_ELEM_USERDATA,
5646                             udata->len + 1, udata->data))
5647                         goto nla_put_failure;
5648         }
5649
5650         nla_nest_end(skb, nest);
5651         return 0;
5652
5653 nla_put_failure:
5654         nlmsg_trim(skb, b);
5655         return -EMSGSIZE;
5656 }
5657
5658 struct nft_set_dump_args {
5659         const struct netlink_callback   *cb;
5660         struct nft_set_iter             iter;
5661         struct sk_buff                  *skb;
5662         bool                            reset;
5663 };
5664
5665 static int nf_tables_dump_setelem(const struct nft_ctx *ctx,
5666                                   struct nft_set *set,
5667                                   const struct nft_set_iter *iter,
5668                                   struct nft_set_elem *elem)
5669 {
5670         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
5671         struct nft_set_dump_args *args;
5672
5673         if (nft_set_elem_expired(ext) || nft_set_elem_is_dead(ext))
5674                 return 0;
5675
5676         args = container_of(iter, struct nft_set_dump_args, iter);
5677         return nf_tables_fill_setelem(args->skb, set, elem, args->reset);
5678 }
5679
5680 static void audit_log_nft_set_reset(const struct nft_table *table,
5681                                     unsigned int base_seq,
5682                                     unsigned int nentries)
5683 {
5684         char *buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, base_seq);
5685
5686         audit_log_nfcfg(buf, table->family, nentries,
5687                         AUDIT_NFT_OP_SETELEM_RESET, GFP_ATOMIC);
5688         kfree(buf);
5689 }
5690
5691 struct nft_set_dump_ctx {
5692         const struct nft_set    *set;
5693         struct nft_ctx          ctx;
5694 };
5695
5696 static int nft_set_catchall_dump(struct net *net, struct sk_buff *skb,
5697                                  const struct nft_set *set, bool reset,
5698                                  unsigned int base_seq)
5699 {
5700         struct nft_set_elem_catchall *catchall;
5701         u8 genmask = nft_genmask_cur(net);
5702         struct nft_set_elem elem;
5703         struct nft_set_ext *ext;
5704         int ret = 0;
5705
5706         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5707                 ext = nft_set_elem_ext(set, catchall->elem);
5708                 if (!nft_set_elem_active(ext, genmask) ||
5709                     nft_set_elem_expired(ext))
5710                         continue;
5711
5712                 elem.priv = catchall->elem;
5713                 ret = nf_tables_fill_setelem(skb, set, &elem, reset);
5714                 if (reset && !ret)
5715                         audit_log_nft_set_reset(set->table, base_seq, 1);
5716                 break;
5717         }
5718
5719         return ret;
5720 }
5721
5722 static int nf_tables_dump_set(struct sk_buff *skb, struct netlink_callback *cb)
5723 {
5724         struct nft_set_dump_ctx *dump_ctx = cb->data;
5725         struct net *net = sock_net(skb->sk);
5726         struct nftables_pernet *nft_net;
5727         struct nft_table *table;
5728         struct nft_set *set;
5729         struct nft_set_dump_args args;
5730         bool set_found = false;
5731         struct nlmsghdr *nlh;
5732         struct nlattr *nest;
5733         bool reset = false;
5734         u32 portid, seq;
5735         int event;
5736
5737         rcu_read_lock();
5738         nft_net = nft_pernet(net);
5739         cb->seq = READ_ONCE(nft_net->base_seq);
5740
5741         list_for_each_entry_rcu(table, &nft_net->tables, list) {
5742                 if (dump_ctx->ctx.family != NFPROTO_UNSPEC &&
5743                     dump_ctx->ctx.family != table->family)
5744                         continue;
5745
5746                 if (table != dump_ctx->ctx.table)
5747                         continue;
5748
5749                 list_for_each_entry_rcu(set, &table->sets, list) {
5750                         if (set == dump_ctx->set) {
5751                                 set_found = true;
5752                                 break;
5753                         }
5754                 }
5755                 break;
5756         }
5757
5758         if (!set_found) {
5759                 rcu_read_unlock();
5760                 return -ENOENT;
5761         }
5762
5763         event  = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWSETELEM);
5764         portid = NETLINK_CB(cb->skb).portid;
5765         seq    = cb->nlh->nlmsg_seq;
5766
5767         nlh = nfnl_msg_put(skb, portid, seq, event, NLM_F_MULTI,
5768                            table->family, NFNETLINK_V0, nft_base_seq(net));
5769         if (!nlh)
5770                 goto nla_put_failure;
5771
5772         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_TABLE, table->name))
5773                 goto nla_put_failure;
5774         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_SET, set->name))
5775                 goto nla_put_failure;
5776
5777         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
5778         if (nest == NULL)
5779                 goto nla_put_failure;
5780
5781         if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETSETELEM_RESET)
5782                 reset = true;
5783
5784         args.cb                 = cb;
5785         args.skb                = skb;
5786         args.reset              = reset;
5787         args.iter.genmask       = nft_genmask_cur(net);
5788         args.iter.skip          = cb->args[0];
5789         args.iter.count         = 0;
5790         args.iter.err           = 0;
5791         args.iter.fn            = nf_tables_dump_setelem;
5792         set->ops->walk(&dump_ctx->ctx, set, &args.iter);
5793
5794         if (!args.iter.err && args.iter.count == cb->args[0])
5795                 args.iter.err = nft_set_catchall_dump(net, skb, set,
5796                                                       reset, cb->seq);
5797         nla_nest_end(skb, nest);
5798         nlmsg_end(skb, nlh);
5799
5800         if (reset && args.iter.count > args.iter.skip)
5801                 audit_log_nft_set_reset(table, cb->seq,
5802                                         args.iter.count - args.iter.skip);
5803
5804         rcu_read_unlock();
5805
5806         if (args.iter.err && args.iter.err != -EMSGSIZE)
5807                 return args.iter.err;
5808         if (args.iter.count == cb->args[0])
5809                 return 0;
5810
5811         cb->args[0] = args.iter.count;
5812         return skb->len;
5813
5814 nla_put_failure:
5815         rcu_read_unlock();
5816         return -ENOSPC;
5817 }
5818
5819 static int nf_tables_dump_set_start(struct netlink_callback *cb)
5820 {
5821         struct nft_set_dump_ctx *dump_ctx = cb->data;
5822
5823         cb->data = kmemdup(dump_ctx, sizeof(*dump_ctx), GFP_ATOMIC);
5824
5825         return cb->data ? 0 : -ENOMEM;
5826 }
5827
5828 static int nf_tables_dump_set_done(struct netlink_callback *cb)
5829 {
5830         kfree(cb->data);
5831         return 0;
5832 }
5833
5834 static int nf_tables_fill_setelem_info(struct sk_buff *skb,
5835                                        const struct nft_ctx *ctx, u32 seq,
5836                                        u32 portid, int event, u16 flags,
5837                                        const struct nft_set *set,
5838                                        const struct nft_set_elem *elem,
5839                                        bool reset)
5840 {
5841         struct nlmsghdr *nlh;
5842         struct nlattr *nest;
5843         int err;
5844
5845         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
5846         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
5847                            NFNETLINK_V0, nft_base_seq(ctx->net));
5848         if (!nlh)
5849                 goto nla_put_failure;
5850
5851         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
5852                 goto nla_put_failure;
5853         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
5854                 goto nla_put_failure;
5855
5856         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
5857         if (nest == NULL)
5858                 goto nla_put_failure;
5859
5860         err = nf_tables_fill_setelem(skb, set, elem, reset);
5861         if (err < 0)
5862                 goto nla_put_failure;
5863
5864         nla_nest_end(skb, nest);
5865
5866         nlmsg_end(skb, nlh);
5867         return 0;
5868
5869 nla_put_failure:
5870         nlmsg_trim(skb, nlh);
5871         return -1;
5872 }
5873
5874 static int nft_setelem_parse_flags(const struct nft_set *set,
5875                                    const struct nlattr *attr, u32 *flags)
5876 {
5877         if (attr == NULL)
5878                 return 0;
5879
5880         *flags = ntohl(nla_get_be32(attr));
5881         if (*flags & ~(NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
5882                 return -EOPNOTSUPP;
5883         if (!(set->flags & NFT_SET_INTERVAL) &&
5884             *flags & NFT_SET_ELEM_INTERVAL_END)
5885                 return -EINVAL;
5886         if ((*flags & (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL)) ==
5887             (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
5888                 return -EINVAL;
5889
5890         return 0;
5891 }
5892
5893 static int nft_setelem_parse_key(struct nft_ctx *ctx, struct nft_set *set,
5894                                  struct nft_data *key, struct nlattr *attr)
5895 {
5896         struct nft_data_desc desc = {
5897                 .type   = NFT_DATA_VALUE,
5898                 .size   = NFT_DATA_VALUE_MAXLEN,
5899                 .len    = set->klen,
5900         };
5901
5902         return nft_data_init(ctx, key, &desc, attr);
5903 }
5904
5905 static int nft_setelem_parse_data(struct nft_ctx *ctx, struct nft_set *set,
5906                                   struct nft_data_desc *desc,
5907                                   struct nft_data *data,
5908                                   struct nlattr *attr)
5909 {
5910         u32 dtype;
5911
5912         if (set->dtype == NFT_DATA_VERDICT)
5913                 dtype = NFT_DATA_VERDICT;
5914         else
5915                 dtype = NFT_DATA_VALUE;
5916
5917         desc->type = dtype;
5918         desc->size = NFT_DATA_VALUE_MAXLEN;
5919         desc->len = set->dlen;
5920         desc->flags = NFT_DATA_DESC_SETELEM;
5921
5922         return nft_data_init(ctx, data, desc, attr);
5923 }
5924
5925 static void *nft_setelem_catchall_get(const struct net *net,
5926                                       const struct nft_set *set)
5927 {
5928         struct nft_set_elem_catchall *catchall;
5929         u8 genmask = nft_genmask_cur(net);
5930         struct nft_set_ext *ext;
5931         void *priv = NULL;
5932
5933         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5934                 ext = nft_set_elem_ext(set, catchall->elem);
5935                 if (!nft_set_elem_active(ext, genmask) ||
5936                     nft_set_elem_expired(ext))
5937                         continue;
5938
5939                 priv = catchall->elem;
5940                 break;
5941         }
5942
5943         return priv;
5944 }
5945
5946 static int nft_setelem_get(struct nft_ctx *ctx, struct nft_set *set,
5947                            struct nft_set_elem *elem, u32 flags)
5948 {
5949         void *priv;
5950
5951         if (!(flags & NFT_SET_ELEM_CATCHALL)) {
5952                 priv = set->ops->get(ctx->net, set, elem, flags);
5953                 if (IS_ERR(priv))
5954                         return PTR_ERR(priv);
5955         } else {
5956                 priv = nft_setelem_catchall_get(ctx->net, set);
5957                 if (!priv)
5958                         return -ENOENT;
5959         }
5960         elem->priv = priv;
5961
5962         return 0;
5963 }
5964
5965 static int nft_get_set_elem(struct nft_ctx *ctx, struct nft_set *set,
5966                             const struct nlattr *attr, bool reset)
5967 {
5968         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
5969         struct nft_set_elem elem;
5970         struct sk_buff *skb;
5971         uint32_t flags = 0;
5972         int err;
5973
5974         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
5975                                           nft_set_elem_policy, NULL);
5976         if (err < 0)
5977                 return err;
5978
5979         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
5980         if (err < 0)
5981                 return err;
5982
5983         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
5984                 return -EINVAL;
5985
5986         if (nla[NFTA_SET_ELEM_KEY]) {
5987                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
5988                                             nla[NFTA_SET_ELEM_KEY]);
5989                 if (err < 0)
5990                         return err;
5991         }
5992
5993         if (nla[NFTA_SET_ELEM_KEY_END]) {
5994                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
5995                                             nla[NFTA_SET_ELEM_KEY_END]);
5996                 if (err < 0)
5997                         return err;
5998         }
5999
6000         err = nft_setelem_get(ctx, set, &elem, flags);
6001         if (err < 0)
6002                 return err;
6003
6004         err = -ENOMEM;
6005         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_ATOMIC);
6006         if (skb == NULL)
6007                 return err;
6008
6009         err = nf_tables_fill_setelem_info(skb, ctx, ctx->seq, ctx->portid,
6010                                           NFT_MSG_NEWSETELEM, 0, set, &elem,
6011                                           reset);
6012         if (err < 0)
6013                 goto err_fill_setelem;
6014
6015         return nfnetlink_unicast(skb, ctx->net, ctx->portid);
6016
6017 err_fill_setelem:
6018         kfree_skb(skb);
6019         return err;
6020 }
6021
6022 /* called with rcu_read_lock held */
6023 static int nf_tables_getsetelem(struct sk_buff *skb,
6024                                 const struct nfnl_info *info,
6025                                 const struct nlattr * const nla[])
6026 {
6027         struct netlink_ext_ack *extack = info->extack;
6028         u8 genmask = nft_genmask_cur(info->net);
6029         u8 family = info->nfmsg->nfgen_family;
6030         int rem, err = 0, nelems = 0;
6031         struct net *net = info->net;
6032         struct nft_table *table;
6033         struct nft_set *set;
6034         struct nlattr *attr;
6035         struct nft_ctx ctx;
6036         bool reset = false;
6037
6038         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
6039                                  genmask, 0);
6040         if (IS_ERR(table)) {
6041                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
6042                 return PTR_ERR(table);
6043         }
6044
6045         set = nft_set_lookup(table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
6046         if (IS_ERR(set))
6047                 return PTR_ERR(set);
6048
6049         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
6050
6051         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
6052                 struct netlink_dump_control c = {
6053                         .start = nf_tables_dump_set_start,
6054                         .dump = nf_tables_dump_set,
6055                         .done = nf_tables_dump_set_done,
6056                         .module = THIS_MODULE,
6057                 };
6058                 struct nft_set_dump_ctx dump_ctx = {
6059                         .set = set,
6060                         .ctx = ctx,
6061                 };
6062
6063                 c.data = &dump_ctx;
6064                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
6065         }
6066
6067         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
6068                 return -EINVAL;
6069
6070         if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETSETELEM_RESET)
6071                 reset = true;
6072
6073         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
6074                 err = nft_get_set_elem(&ctx, set, attr, reset);
6075                 if (err < 0) {
6076                         NL_SET_BAD_ATTR(extack, attr);
6077                         break;
6078                 }
6079                 nelems++;
6080         }
6081
6082         if (reset)
6083                 audit_log_nft_set_reset(table, nft_pernet(net)->base_seq,
6084                                         nelems);
6085
6086         return err;
6087 }
6088
6089 static void nf_tables_setelem_notify(const struct nft_ctx *ctx,
6090                                      const struct nft_set *set,
6091                                      const struct nft_set_elem *elem,
6092                                      int event)
6093 {
6094         struct nftables_pernet *nft_net;
6095         struct net *net = ctx->net;
6096         u32 portid = ctx->portid;
6097         struct sk_buff *skb;
6098         u16 flags = 0;
6099         int err;
6100
6101         if (!ctx->report && !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
6102                 return;
6103
6104         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
6105         if (skb == NULL)
6106                 goto err;
6107
6108         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
6109                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
6110
6111         err = nf_tables_fill_setelem_info(skb, ctx, 0, portid, event, flags,
6112                                           set, elem, false);
6113         if (err < 0) {
6114                 kfree_skb(skb);
6115                 goto err;
6116         }
6117
6118         nft_net = nft_pernet(net);
6119         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
6120         return;
6121 err:
6122         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
6123 }
6124
6125 static struct nft_trans *nft_trans_elem_alloc(struct nft_ctx *ctx,
6126                                               int msg_type,
6127                                               struct nft_set *set)
6128 {
6129         struct nft_trans *trans;
6130
6131         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_elem));
6132         if (trans == NULL)
6133                 return NULL;
6134
6135         nft_trans_elem_set(trans) = set;
6136         return trans;
6137 }
6138
6139 struct nft_expr *nft_set_elem_expr_alloc(const struct nft_ctx *ctx,
6140                                          const struct nft_set *set,
6141                                          const struct nlattr *attr)
6142 {
6143         struct nft_expr *expr;
6144         int err;
6145
6146         expr = nft_expr_init(ctx, attr);
6147         if (IS_ERR(expr))
6148                 return expr;
6149
6150         err = -EOPNOTSUPP;
6151         if (expr->ops->type->flags & NFT_EXPR_GC) {
6152                 if (set->flags & NFT_SET_TIMEOUT)
6153                         goto err_set_elem_expr;
6154                 if (!set->ops->gc_init)
6155                         goto err_set_elem_expr;
6156                 set->ops->gc_init(set);
6157         }
6158
6159         return expr;
6160
6161 err_set_elem_expr:
6162         nft_expr_destroy(ctx, expr);
6163         return ERR_PTR(err);
6164 }
6165
6166 static int nft_set_ext_check(const struct nft_set_ext_tmpl *tmpl, u8 id, u32 len)
6167 {
6168         len += nft_set_ext_types[id].len;
6169         if (len > tmpl->ext_len[id] ||
6170             len > U8_MAX)
6171                 return -1;
6172
6173         return 0;
6174 }
6175
6176 static int nft_set_ext_memcpy(const struct nft_set_ext_tmpl *tmpl, u8 id,
6177                               void *to, const void *from, u32 len)
6178 {
6179         if (nft_set_ext_check(tmpl, id, len) < 0)
6180                 return -1;
6181
6182         memcpy(to, from, len);
6183
6184         return 0;
6185 }
6186
6187 void *nft_set_elem_init(const struct nft_set *set,
6188                         const struct nft_set_ext_tmpl *tmpl,
6189                         const u32 *key, const u32 *key_end,
6190                         const u32 *data, u64 timeout, u64 expiration, gfp_t gfp)
6191 {
6192         struct nft_set_ext *ext;
6193         void *elem;
6194
6195         elem = kzalloc(set->ops->elemsize + tmpl->len, gfp);
6196         if (elem == NULL)
6197                 return ERR_PTR(-ENOMEM);
6198
6199         ext = nft_set_elem_ext(set, elem);
6200         nft_set_ext_init(ext, tmpl);
6201
6202         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
6203             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY,
6204                                nft_set_ext_key(ext), key, set->klen) < 0)
6205                 goto err_ext_check;
6206
6207         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
6208             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY_END,
6209                                nft_set_ext_key_end(ext), key_end, set->klen) < 0)
6210                 goto err_ext_check;
6211
6212         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
6213             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_DATA,
6214                                nft_set_ext_data(ext), data, set->dlen) < 0)
6215                 goto err_ext_check;
6216
6217         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
6218                 *nft_set_ext_expiration(ext) = get_jiffies_64() + expiration;
6219                 if (expiration == 0)
6220                         *nft_set_ext_expiration(ext) += timeout;
6221         }
6222         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT))
6223                 *nft_set_ext_timeout(ext) = timeout;
6224
6225         return elem;
6226
6227 err_ext_check:
6228         kfree(elem);
6229
6230         return ERR_PTR(-EINVAL);
6231 }
6232
6233 static void __nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
6234                                         struct nft_expr *expr)
6235 {
6236         if (expr->ops->destroy_clone) {
6237                 expr->ops->destroy_clone(ctx, expr);
6238                 module_put(expr->ops->type->owner);
6239         } else {
6240                 nf_tables_expr_destroy(ctx, expr);
6241         }
6242 }
6243
6244 static void nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
6245                                       struct nft_set_elem_expr *elem_expr)
6246 {
6247         struct nft_expr *expr;
6248         u32 size;
6249
6250         nft_setelem_expr_foreach(expr, elem_expr, size)
6251                 __nft_set_elem_expr_destroy(ctx, expr);
6252 }
6253
6254 /* Drop references and destroy. Called from gc, dynset and abort path. */
6255 void nft_set_elem_destroy(const struct nft_set *set, void *elem,
6256                           bool destroy_expr)
6257 {
6258         struct nft_set_ext *ext = nft_set_elem_ext(set, elem);
6259         struct nft_ctx ctx = {
6260                 .net    = read_pnet(&set->net),
6261                 .family = set->table->family,
6262         };
6263
6264         nft_data_release(nft_set_ext_key(ext), NFT_DATA_VALUE);
6265         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
6266                 nft_data_release(nft_set_ext_data(ext), set->dtype);
6267         if (destroy_expr && nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
6268                 nft_set_elem_expr_destroy(&ctx, nft_set_ext_expr(ext));
6269
6270         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
6271                 nft_use_dec(&(*nft_set_ext_obj(ext))->use);
6272         kfree(elem);
6273 }
6274 EXPORT_SYMBOL_GPL(nft_set_elem_destroy);
6275
6276 /* Destroy element. References have been already dropped in the preparation
6277  * path via nft_setelem_data_deactivate().
6278  */
6279 void nf_tables_set_elem_destroy(const struct nft_ctx *ctx,
6280                                 const struct nft_set *set, void *elem)
6281 {
6282         struct nft_set_ext *ext = nft_set_elem_ext(set, elem);
6283
6284         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
6285                 nft_set_elem_expr_destroy(ctx, nft_set_ext_expr(ext));
6286
6287         kfree(elem);
6288 }
6289
6290 int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set,
6291                             struct nft_expr *expr_array[])
6292 {
6293         struct nft_expr *expr;
6294         int err, i, k;
6295
6296         for (i = 0; i < set->num_exprs; i++) {
6297                 expr = kzalloc(set->exprs[i]->ops->size, GFP_KERNEL_ACCOUNT);
6298                 if (!expr)
6299                         goto err_expr;
6300
6301                 err = nft_expr_clone(expr, set->exprs[i]);
6302                 if (err < 0) {
6303                         kfree(expr);
6304                         goto err_expr;
6305                 }
6306                 expr_array[i] = expr;
6307         }
6308
6309         return 0;
6310
6311 err_expr:
6312         for (k = i - 1; k >= 0; k--)
6313                 nft_expr_destroy(ctx, expr_array[k]);
6314
6315         return -ENOMEM;
6316 }
6317
6318 static int nft_set_elem_expr_setup(struct nft_ctx *ctx,
6319                                    const struct nft_set_ext_tmpl *tmpl,
6320                                    const struct nft_set_ext *ext,
6321                                    struct nft_expr *expr_array[],
6322                                    u32 num_exprs)
6323 {
6324         struct nft_set_elem_expr *elem_expr = nft_set_ext_expr(ext);
6325         u32 len = sizeof(struct nft_set_elem_expr);
6326         struct nft_expr *expr;
6327         int i, err;
6328
6329         if (num_exprs == 0)
6330                 return 0;
6331
6332         for (i = 0; i < num_exprs; i++)
6333                 len += expr_array[i]->ops->size;
6334
6335         if (nft_set_ext_check(tmpl, NFT_SET_EXT_EXPRESSIONS, len) < 0)
6336                 return -EINVAL;
6337
6338         for (i = 0; i < num_exprs; i++) {
6339                 expr = nft_setelem_expr_at(elem_expr, elem_expr->size);
6340                 err = nft_expr_clone(expr, expr_array[i]);
6341                 if (err < 0)
6342                         goto err_elem_expr_setup;
6343
6344                 elem_expr->size += expr_array[i]->ops->size;
6345                 nft_expr_destroy(ctx, expr_array[i]);
6346                 expr_array[i] = NULL;
6347         }
6348
6349         return 0;
6350
6351 err_elem_expr_setup:
6352         for (; i < num_exprs; i++) {
6353                 nft_expr_destroy(ctx, expr_array[i]);
6354                 expr_array[i] = NULL;
6355         }
6356
6357         return -ENOMEM;
6358 }
6359
6360 struct nft_set_ext *nft_set_catchall_lookup(const struct net *net,
6361                                             const struct nft_set *set)
6362 {
6363         struct nft_set_elem_catchall *catchall;
6364         u8 genmask = nft_genmask_cur(net);
6365         struct nft_set_ext *ext;
6366
6367         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
6368                 ext = nft_set_elem_ext(set, catchall->elem);
6369                 if (nft_set_elem_active(ext, genmask) &&
6370                     !nft_set_elem_expired(ext) &&
6371                     !nft_set_elem_is_dead(ext))
6372                         return ext;
6373         }
6374
6375         return NULL;
6376 }
6377 EXPORT_SYMBOL_GPL(nft_set_catchall_lookup);
6378
6379 static int nft_setelem_catchall_insert(const struct net *net,
6380                                        struct nft_set *set,
6381                                        const struct nft_set_elem *elem,
6382                                        struct nft_set_ext **pext)
6383 {
6384         struct nft_set_elem_catchall *catchall;
6385         u8 genmask = nft_genmask_next(net);
6386         struct nft_set_ext *ext;
6387
6388         list_for_each_entry(catchall, &set->catchall_list, list) {
6389                 ext = nft_set_elem_ext(set, catchall->elem);
6390                 if (nft_set_elem_active(ext, genmask)) {
6391                         *pext = ext;
6392                         return -EEXIST;
6393                 }
6394         }
6395
6396         catchall = kmalloc(sizeof(*catchall), GFP_KERNEL);
6397         if (!catchall)
6398                 return -ENOMEM;
6399
6400         catchall->elem = elem->priv;
6401         list_add_tail_rcu(&catchall->list, &set->catchall_list);
6402
6403         return 0;
6404 }
6405
6406 static int nft_setelem_insert(const struct net *net,
6407                               struct nft_set *set,
6408                               const struct nft_set_elem *elem,
6409                               struct nft_set_ext **ext, unsigned int flags)
6410 {
6411         int ret;
6412
6413         if (flags & NFT_SET_ELEM_CATCHALL)
6414                 ret = nft_setelem_catchall_insert(net, set, elem, ext);
6415         else
6416                 ret = set->ops->insert(net, set, elem, ext);
6417
6418         return ret;
6419 }
6420
6421 static bool nft_setelem_is_catchall(const struct nft_set *set,
6422                                     const struct nft_set_elem *elem)
6423 {
6424         struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6425
6426         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
6427             *nft_set_ext_flags(ext) & NFT_SET_ELEM_CATCHALL)
6428                 return true;
6429
6430         return false;
6431 }
6432
6433 static void nft_setelem_activate(struct net *net, struct nft_set *set,
6434                                  struct nft_set_elem *elem)
6435 {
6436         struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6437
6438         if (nft_setelem_is_catchall(set, elem)) {
6439                 nft_set_elem_change_active(net, set, ext);
6440         } else {
6441                 set->ops->activate(net, set, elem);
6442         }
6443 }
6444
6445 static int nft_setelem_catchall_deactivate(const struct net *net,
6446                                            struct nft_set *set,
6447                                            struct nft_set_elem *elem)
6448 {
6449         struct nft_set_elem_catchall *catchall;
6450         struct nft_set_ext *ext;
6451
6452         list_for_each_entry(catchall, &set->catchall_list, list) {
6453                 ext = nft_set_elem_ext(set, catchall->elem);
6454                 if (!nft_is_active_next(net, ext))
6455                         continue;
6456
6457                 kfree(elem->priv);
6458                 elem->priv = catchall->elem;
6459                 nft_set_elem_change_active(net, set, ext);
6460                 return 0;
6461         }
6462
6463         return -ENOENT;
6464 }
6465
6466 static int __nft_setelem_deactivate(const struct net *net,
6467                                     struct nft_set *set,
6468                                     struct nft_set_elem *elem)
6469 {
6470         void *priv;
6471
6472         priv = set->ops->deactivate(net, set, elem);
6473         if (!priv)
6474                 return -ENOENT;
6475
6476         kfree(elem->priv);
6477         elem->priv = priv;
6478         set->ndeact++;
6479
6480         return 0;
6481 }
6482
6483 static int nft_setelem_deactivate(const struct net *net,
6484                                   struct nft_set *set,
6485                                   struct nft_set_elem *elem, u32 flags)
6486 {
6487         int ret;
6488
6489         if (flags & NFT_SET_ELEM_CATCHALL)
6490                 ret = nft_setelem_catchall_deactivate(net, set, elem);
6491         else
6492                 ret = __nft_setelem_deactivate(net, set, elem);
6493
6494         return ret;
6495 }
6496
6497 static void nft_setelem_catchall_destroy(struct nft_set_elem_catchall *catchall)
6498 {
6499         list_del_rcu(&catchall->list);
6500         kfree_rcu(catchall, rcu);
6501 }
6502
6503 static void nft_setelem_catchall_remove(const struct net *net,
6504                                         const struct nft_set *set,
6505                                         const struct nft_set_elem *elem)
6506 {
6507         struct nft_set_elem_catchall *catchall, *next;
6508
6509         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
6510                 if (catchall->elem == elem->priv) {
6511                         nft_setelem_catchall_destroy(catchall);
6512                         break;
6513                 }
6514         }
6515 }
6516
6517 static void nft_setelem_remove(const struct net *net,
6518                                const struct nft_set *set,
6519                                const struct nft_set_elem *elem)
6520 {
6521         if (nft_setelem_is_catchall(set, elem))
6522                 nft_setelem_catchall_remove(net, set, elem);
6523         else
6524                 set->ops->remove(net, set, elem);
6525 }
6526
6527 static bool nft_setelem_valid_key_end(const struct nft_set *set,
6528                                       struct nlattr **nla, u32 flags)
6529 {
6530         if ((set->flags & (NFT_SET_CONCAT | NFT_SET_INTERVAL)) ==
6531                           (NFT_SET_CONCAT | NFT_SET_INTERVAL)) {
6532                 if (flags & NFT_SET_ELEM_INTERVAL_END)
6533                         return false;
6534
6535                 if (nla[NFTA_SET_ELEM_KEY_END] &&
6536                     flags & NFT_SET_ELEM_CATCHALL)
6537                         return false;
6538         } else {
6539                 if (nla[NFTA_SET_ELEM_KEY_END])
6540                         return false;
6541         }
6542
6543         return true;
6544 }
6545
6546 static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set,
6547                             const struct nlattr *attr, u32 nlmsg_flags)
6548 {
6549         struct nft_expr *expr_array[NFT_SET_EXPR_MAX] = {};
6550         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
6551         u8 genmask = nft_genmask_next(ctx->net);
6552         u32 flags = 0, size = 0, num_exprs = 0;
6553         struct nft_set_ext_tmpl tmpl;
6554         struct nft_set_ext *ext, *ext2;
6555         struct nft_set_elem elem;
6556         struct nft_set_binding *binding;
6557         struct nft_object *obj = NULL;
6558         struct nft_userdata *udata;
6559         struct nft_data_desc desc;
6560         enum nft_registers dreg;
6561         struct nft_trans *trans;
6562         u64 timeout;
6563         u64 expiration;
6564         int err, i;
6565         u8 ulen;
6566
6567         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
6568                                           nft_set_elem_policy, NULL);
6569         if (err < 0)
6570                 return err;
6571
6572         nft_set_ext_prepare(&tmpl);
6573
6574         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
6575         if (err < 0)
6576                 return err;
6577
6578         if (((flags & NFT_SET_ELEM_CATCHALL) && nla[NFTA_SET_ELEM_KEY]) ||
6579             (!(flags & NFT_SET_ELEM_CATCHALL) && !nla[NFTA_SET_ELEM_KEY]))
6580                 return -EINVAL;
6581
6582         if (flags != 0) {
6583                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
6584                 if (err < 0)
6585                         return err;
6586         }
6587
6588         if (set->flags & NFT_SET_MAP) {
6589                 if (nla[NFTA_SET_ELEM_DATA] == NULL &&
6590                     !(flags & NFT_SET_ELEM_INTERVAL_END))
6591                         return -EINVAL;
6592         } else {
6593                 if (nla[NFTA_SET_ELEM_DATA] != NULL)
6594                         return -EINVAL;
6595         }
6596
6597         if (set->flags & NFT_SET_OBJECT) {
6598                 if (!nla[NFTA_SET_ELEM_OBJREF] &&
6599                     !(flags & NFT_SET_ELEM_INTERVAL_END))
6600                         return -EINVAL;
6601         } else {
6602                 if (nla[NFTA_SET_ELEM_OBJREF])
6603                         return -EINVAL;
6604         }
6605
6606         if (!nft_setelem_valid_key_end(set, nla, flags))
6607                 return -EINVAL;
6608
6609         if ((flags & NFT_SET_ELEM_INTERVAL_END) &&
6610              (nla[NFTA_SET_ELEM_DATA] ||
6611               nla[NFTA_SET_ELEM_OBJREF] ||
6612               nla[NFTA_SET_ELEM_TIMEOUT] ||
6613               nla[NFTA_SET_ELEM_EXPIRATION] ||
6614               nla[NFTA_SET_ELEM_USERDATA] ||
6615               nla[NFTA_SET_ELEM_EXPR] ||
6616               nla[NFTA_SET_ELEM_KEY_END] ||
6617               nla[NFTA_SET_ELEM_EXPRESSIONS]))
6618                 return -EINVAL;
6619
6620         timeout = 0;
6621         if (nla[NFTA_SET_ELEM_TIMEOUT] != NULL) {
6622                 if (!(set->flags & NFT_SET_TIMEOUT))
6623                         return -EINVAL;
6624                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_TIMEOUT],
6625                                             &timeout);
6626                 if (err)
6627                         return err;
6628         } else if (set->flags & NFT_SET_TIMEOUT &&
6629                    !(flags & NFT_SET_ELEM_INTERVAL_END)) {
6630                 timeout = READ_ONCE(set->timeout);
6631         }
6632
6633         expiration = 0;
6634         if (nla[NFTA_SET_ELEM_EXPIRATION] != NULL) {
6635                 if (!(set->flags & NFT_SET_TIMEOUT))
6636                         return -EINVAL;
6637                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_EXPIRATION],
6638                                             &expiration);
6639                 if (err)
6640                         return err;
6641         }
6642
6643         if (nla[NFTA_SET_ELEM_EXPR]) {
6644                 struct nft_expr *expr;
6645
6646                 if (set->num_exprs && set->num_exprs != 1)
6647                         return -EOPNOTSUPP;
6648
6649                 expr = nft_set_elem_expr_alloc(ctx, set,
6650                                                nla[NFTA_SET_ELEM_EXPR]);
6651                 if (IS_ERR(expr))
6652                         return PTR_ERR(expr);
6653
6654                 expr_array[0] = expr;
6655                 num_exprs = 1;
6656
6657                 if (set->num_exprs && set->exprs[0]->ops != expr->ops) {
6658                         err = -EOPNOTSUPP;
6659                         goto err_set_elem_expr;
6660                 }
6661         } else if (nla[NFTA_SET_ELEM_EXPRESSIONS]) {
6662                 struct nft_expr *expr;
6663                 struct nlattr *tmp;
6664                 int left;
6665
6666                 i = 0;
6667                 nla_for_each_nested(tmp, nla[NFTA_SET_ELEM_EXPRESSIONS], left) {
6668                         if (i == NFT_SET_EXPR_MAX ||
6669                             (set->num_exprs && set->num_exprs == i)) {
6670                                 err = -E2BIG;
6671                                 goto err_set_elem_expr;
6672                         }
6673                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
6674                                 err = -EINVAL;
6675                                 goto err_set_elem_expr;
6676                         }
6677                         expr = nft_set_elem_expr_alloc(ctx, set, tmp);
6678                         if (IS_ERR(expr)) {
6679                                 err = PTR_ERR(expr);
6680                                 goto err_set_elem_expr;
6681                         }
6682                         expr_array[i] = expr;
6683                         num_exprs++;
6684
6685                         if (set->num_exprs && expr->ops != set->exprs[i]->ops) {
6686                                 err = -EOPNOTSUPP;
6687                                 goto err_set_elem_expr;
6688                         }
6689                         i++;
6690                 }
6691                 if (set->num_exprs && set->num_exprs != i) {
6692                         err = -EOPNOTSUPP;
6693                         goto err_set_elem_expr;
6694                 }
6695         } else if (set->num_exprs > 0 &&
6696                    !(flags & NFT_SET_ELEM_INTERVAL_END)) {
6697                 err = nft_set_elem_expr_clone(ctx, set, expr_array);
6698                 if (err < 0)
6699                         goto err_set_elem_expr_clone;
6700
6701                 num_exprs = set->num_exprs;
6702         }
6703
6704         if (nla[NFTA_SET_ELEM_KEY]) {
6705                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
6706                                             nla[NFTA_SET_ELEM_KEY]);
6707                 if (err < 0)
6708                         goto err_set_elem_expr;
6709
6710                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
6711                 if (err < 0)
6712                         goto err_parse_key;
6713         }
6714
6715         if (nla[NFTA_SET_ELEM_KEY_END]) {
6716                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
6717                                             nla[NFTA_SET_ELEM_KEY_END]);
6718                 if (err < 0)
6719                         goto err_parse_key;
6720
6721                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
6722                 if (err < 0)
6723                         goto err_parse_key_end;
6724         }
6725
6726         if (timeout > 0) {
6727                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_EXPIRATION);
6728                 if (err < 0)
6729                         goto err_parse_key_end;
6730
6731                 if (timeout != READ_ONCE(set->timeout)) {
6732                         err = nft_set_ext_add(&tmpl, NFT_SET_EXT_TIMEOUT);
6733                         if (err < 0)
6734                                 goto err_parse_key_end;
6735                 }
6736         }
6737
6738         if (num_exprs) {
6739                 for (i = 0; i < num_exprs; i++)
6740                         size += expr_array[i]->ops->size;
6741
6742                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_EXPRESSIONS,
6743                                              sizeof(struct nft_set_elem_expr) + size);
6744                 if (err < 0)
6745                         goto err_parse_key_end;
6746         }
6747
6748         if (nla[NFTA_SET_ELEM_OBJREF] != NULL) {
6749                 obj = nft_obj_lookup(ctx->net, ctx->table,
6750                                      nla[NFTA_SET_ELEM_OBJREF],
6751                                      set->objtype, genmask);
6752                 if (IS_ERR(obj)) {
6753                         err = PTR_ERR(obj);
6754                         obj = NULL;
6755                         goto err_parse_key_end;
6756                 }
6757
6758                 if (!nft_use_inc(&obj->use)) {
6759                         err = -EMFILE;
6760                         obj = NULL;
6761                         goto err_parse_key_end;
6762                 }
6763
6764                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_OBJREF);
6765                 if (err < 0)
6766                         goto err_parse_key_end;
6767         }
6768
6769         if (nla[NFTA_SET_ELEM_DATA] != NULL) {
6770                 err = nft_setelem_parse_data(ctx, set, &desc, &elem.data.val,
6771                                              nla[NFTA_SET_ELEM_DATA]);
6772                 if (err < 0)
6773                         goto err_parse_key_end;
6774
6775                 dreg = nft_type_to_reg(set->dtype);
6776                 list_for_each_entry(binding, &set->bindings, list) {
6777                         struct nft_ctx bind_ctx = {
6778                                 .net    = ctx->net,
6779                                 .family = ctx->family,
6780                                 .table  = ctx->table,
6781                                 .chain  = (struct nft_chain *)binding->chain,
6782                         };
6783
6784                         if (!(binding->flags & NFT_SET_MAP))
6785                                 continue;
6786
6787                         err = nft_validate_register_store(&bind_ctx, dreg,
6788                                                           &elem.data.val,
6789                                                           desc.type, desc.len);
6790                         if (err < 0)
6791                                 goto err_parse_data;
6792
6793                         if (desc.type == NFT_DATA_VERDICT &&
6794                             (elem.data.val.verdict.code == NFT_GOTO ||
6795                              elem.data.val.verdict.code == NFT_JUMP))
6796                                 nft_validate_state_update(ctx->table,
6797                                                           NFT_VALIDATE_NEED);
6798                 }
6799
6800                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_DATA, desc.len);
6801                 if (err < 0)
6802                         goto err_parse_data;
6803         }
6804
6805         /* The full maximum length of userdata can exceed the maximum
6806          * offset value (U8_MAX) for following extensions, therefor it
6807          * must be the last extension added.
6808          */
6809         ulen = 0;
6810         if (nla[NFTA_SET_ELEM_USERDATA] != NULL) {
6811                 ulen = nla_len(nla[NFTA_SET_ELEM_USERDATA]);
6812                 if (ulen > 0) {
6813                         err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_USERDATA,
6814                                                      ulen);
6815                         if (err < 0)
6816                                 goto err_parse_data;
6817                 }
6818         }
6819
6820         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
6821                                       elem.key_end.val.data, elem.data.val.data,
6822                                       timeout, expiration, GFP_KERNEL_ACCOUNT);
6823         if (IS_ERR(elem.priv)) {
6824                 err = PTR_ERR(elem.priv);
6825                 goto err_parse_data;
6826         }
6827
6828         ext = nft_set_elem_ext(set, elem.priv);
6829         if (flags)
6830                 *nft_set_ext_flags(ext) = flags;
6831
6832         if (obj)
6833                 *nft_set_ext_obj(ext) = obj;
6834
6835         if (ulen > 0) {
6836                 if (nft_set_ext_check(&tmpl, NFT_SET_EXT_USERDATA, ulen) < 0) {
6837                         err = -EINVAL;
6838                         goto err_elem_free;
6839                 }
6840                 udata = nft_set_ext_userdata(ext);
6841                 udata->len = ulen - 1;
6842                 nla_memcpy(&udata->data, nla[NFTA_SET_ELEM_USERDATA], ulen);
6843         }
6844         err = nft_set_elem_expr_setup(ctx, &tmpl, ext, expr_array, num_exprs);
6845         if (err < 0)
6846                 goto err_elem_free;
6847
6848         trans = nft_trans_elem_alloc(ctx, NFT_MSG_NEWSETELEM, set);
6849         if (trans == NULL) {
6850                 err = -ENOMEM;
6851                 goto err_elem_free;
6852         }
6853
6854         ext->genmask = nft_genmask_cur(ctx->net);
6855
6856         err = nft_setelem_insert(ctx->net, set, &elem, &ext2, flags);
6857         if (err) {
6858                 if (err == -EEXIST) {
6859                         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) ^
6860                             nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) ||
6861                             nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) ^
6862                             nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF))
6863                                 goto err_element_clash;
6864                         if ((nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
6865                              nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) &&
6866                              memcmp(nft_set_ext_data(ext),
6867                                     nft_set_ext_data(ext2), set->dlen) != 0) ||
6868                             (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
6869                              nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF) &&
6870                              *nft_set_ext_obj(ext) != *nft_set_ext_obj(ext2)))
6871                                 goto err_element_clash;
6872                         else if (!(nlmsg_flags & NLM_F_EXCL))
6873                                 err = 0;
6874                 } else if (err == -ENOTEMPTY) {
6875                         /* ENOTEMPTY reports overlapping between this element
6876                          * and an existing one.
6877                          */
6878                         err = -EEXIST;
6879                 }
6880                 goto err_element_clash;
6881         }
6882
6883         if (!(flags & NFT_SET_ELEM_CATCHALL)) {
6884                 unsigned int max = set->size ? set->size + set->ndeact : UINT_MAX;
6885
6886                 if (!atomic_add_unless(&set->nelems, 1, max)) {
6887                         err = -ENFILE;
6888                         goto err_set_full;
6889                 }
6890         }
6891
6892         nft_trans_elem(trans) = elem;
6893         nft_trans_commit_list_add_tail(ctx->net, trans);
6894         return 0;
6895
6896 err_set_full:
6897         nft_setelem_remove(ctx->net, set, &elem);
6898 err_element_clash:
6899         kfree(trans);
6900 err_elem_free:
6901         nf_tables_set_elem_destroy(ctx, set, elem.priv);
6902 err_parse_data:
6903         if (nla[NFTA_SET_ELEM_DATA] != NULL)
6904                 nft_data_release(&elem.data.val, desc.type);
6905 err_parse_key_end:
6906         if (obj)
6907                 nft_use_dec_restore(&obj->use);
6908
6909         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
6910 err_parse_key:
6911         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
6912 err_set_elem_expr:
6913         for (i = 0; i < num_exprs && expr_array[i]; i++)
6914                 nft_expr_destroy(ctx, expr_array[i]);
6915 err_set_elem_expr_clone:
6916         return err;
6917 }
6918
6919 static int nf_tables_newsetelem(struct sk_buff *skb,
6920                                 const struct nfnl_info *info,
6921                                 const struct nlattr * const nla[])
6922 {
6923         struct netlink_ext_ack *extack = info->extack;
6924         u8 genmask = nft_genmask_next(info->net);
6925         u8 family = info->nfmsg->nfgen_family;
6926         struct net *net = info->net;
6927         const struct nlattr *attr;
6928         struct nft_table *table;
6929         struct nft_set *set;
6930         struct nft_ctx ctx;
6931         int rem, err;
6932
6933         if (nla[NFTA_SET_ELEM_LIST_ELEMENTS] == NULL)
6934                 return -EINVAL;
6935
6936         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
6937                                  genmask, NETLINK_CB(skb).portid);
6938         if (IS_ERR(table)) {
6939                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
6940                 return PTR_ERR(table);
6941         }
6942
6943         set = nft_set_lookup_global(net, table, nla[NFTA_SET_ELEM_LIST_SET],
6944                                     nla[NFTA_SET_ELEM_LIST_SET_ID], genmask);
6945         if (IS_ERR(set))
6946                 return PTR_ERR(set);
6947
6948         if (!list_empty(&set->bindings) &&
6949             (set->flags & (NFT_SET_CONSTANT | NFT_SET_ANONYMOUS)))
6950                 return -EBUSY;
6951
6952         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
6953
6954         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
6955                 err = nft_add_set_elem(&ctx, set, attr, info->nlh->nlmsg_flags);
6956                 if (err < 0) {
6957                         NL_SET_BAD_ATTR(extack, attr);
6958                         return err;
6959                 }
6960         }
6961
6962         if (table->validate_state == NFT_VALIDATE_DO)
6963                 return nft_table_validate(net, table);
6964
6965         return 0;
6966 }
6967
6968 /**
6969  *      nft_data_hold - hold a nft_data item
6970  *
6971  *      @data: struct nft_data to release
6972  *      @type: type of data
6973  *
6974  *      Hold a nft_data item. NFT_DATA_VALUE types can be silently discarded,
6975  *      NFT_DATA_VERDICT bumps the reference to chains in case of NFT_JUMP and
6976  *      NFT_GOTO verdicts. This function must be called on active data objects
6977  *      from the second phase of the commit protocol.
6978  */
6979 void nft_data_hold(const struct nft_data *data, enum nft_data_types type)
6980 {
6981         struct nft_chain *chain;
6982
6983         if (type == NFT_DATA_VERDICT) {
6984                 switch (data->verdict.code) {
6985                 case NFT_JUMP:
6986                 case NFT_GOTO:
6987                         chain = data->verdict.chain;
6988                         nft_use_inc_restore(&chain->use);
6989                         break;
6990                 }
6991         }
6992 }
6993
6994 static void nft_setelem_data_activate(const struct net *net,
6995                                       const struct nft_set *set,
6996                                       struct nft_set_elem *elem)
6997 {
6998         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6999
7000         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
7001                 nft_data_hold(nft_set_ext_data(ext), set->dtype);
7002         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
7003                 nft_use_inc_restore(&(*nft_set_ext_obj(ext))->use);
7004 }
7005
7006 void nft_setelem_data_deactivate(const struct net *net,
7007                                  const struct nft_set *set,
7008                                  struct nft_set_elem *elem)
7009 {
7010         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
7011
7012         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
7013                 nft_data_release(nft_set_ext_data(ext), set->dtype);
7014         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
7015                 nft_use_dec(&(*nft_set_ext_obj(ext))->use);
7016 }
7017
7018 static int nft_del_setelem(struct nft_ctx *ctx, struct nft_set *set,
7019                            const struct nlattr *attr)
7020 {
7021         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
7022         struct nft_set_ext_tmpl tmpl;
7023         struct nft_set_elem elem;
7024         struct nft_set_ext *ext;
7025         struct nft_trans *trans;
7026         u32 flags = 0;
7027         int err;
7028
7029         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
7030                                           nft_set_elem_policy, NULL);
7031         if (err < 0)
7032                 return err;
7033
7034         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
7035         if (err < 0)
7036                 return err;
7037
7038         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
7039                 return -EINVAL;
7040
7041         if (!nft_setelem_valid_key_end(set, nla, flags))
7042                 return -EINVAL;
7043
7044         nft_set_ext_prepare(&tmpl);
7045
7046         if (flags != 0) {
7047                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
7048                 if (err < 0)
7049                         return err;
7050         }
7051
7052         if (nla[NFTA_SET_ELEM_KEY]) {
7053                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
7054                                             nla[NFTA_SET_ELEM_KEY]);
7055                 if (err < 0)
7056                         return err;
7057
7058                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
7059                 if (err < 0)
7060                         goto fail_elem;
7061         }
7062
7063         if (nla[NFTA_SET_ELEM_KEY_END]) {
7064                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
7065                                             nla[NFTA_SET_ELEM_KEY_END]);
7066                 if (err < 0)
7067                         goto fail_elem;
7068
7069                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
7070                 if (err < 0)
7071                         goto fail_elem_key_end;
7072         }
7073
7074         err = -ENOMEM;
7075         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
7076                                       elem.key_end.val.data, NULL, 0, 0,
7077                                       GFP_KERNEL_ACCOUNT);
7078         if (IS_ERR(elem.priv)) {
7079                 err = PTR_ERR(elem.priv);
7080                 goto fail_elem_key_end;
7081         }
7082
7083         ext = nft_set_elem_ext(set, elem.priv);
7084         if (flags)
7085                 *nft_set_ext_flags(ext) = flags;
7086
7087         trans = nft_trans_elem_alloc(ctx, NFT_MSG_DELSETELEM, set);
7088         if (trans == NULL)
7089                 goto fail_trans;
7090
7091         err = nft_setelem_deactivate(ctx->net, set, &elem, flags);
7092         if (err < 0)
7093                 goto fail_ops;
7094
7095         nft_setelem_data_deactivate(ctx->net, set, &elem);
7096
7097         nft_trans_elem(trans) = elem;
7098         nft_trans_commit_list_add_tail(ctx->net, trans);
7099         return 0;
7100
7101 fail_ops:
7102         kfree(trans);
7103 fail_trans:
7104         kfree(elem.priv);
7105 fail_elem_key_end:
7106         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
7107 fail_elem:
7108         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
7109         return err;
7110 }
7111
7112 static int nft_setelem_flush(const struct nft_ctx *ctx,
7113                              struct nft_set *set,
7114                              const struct nft_set_iter *iter,
7115                              struct nft_set_elem *elem)
7116 {
7117         struct nft_trans *trans;
7118         int err;
7119
7120         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
7121                                     sizeof(struct nft_trans_elem), GFP_ATOMIC);
7122         if (!trans)
7123                 return -ENOMEM;
7124
7125         if (!set->ops->flush(ctx->net, set, elem->priv)) {
7126                 err = -ENOENT;
7127                 goto err1;
7128         }
7129         set->ndeact++;
7130
7131         nft_setelem_data_deactivate(ctx->net, set, elem);
7132         nft_trans_elem_set(trans) = set;
7133         nft_trans_elem(trans) = *elem;
7134         nft_trans_commit_list_add_tail(ctx->net, trans);
7135
7136         return 0;
7137 err1:
7138         kfree(trans);
7139         return err;
7140 }
7141
7142 static int __nft_set_catchall_flush(const struct nft_ctx *ctx,
7143                                     struct nft_set *set,
7144                                     struct nft_set_elem *elem)
7145 {
7146         struct nft_trans *trans;
7147
7148         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
7149                                     sizeof(struct nft_trans_elem), GFP_KERNEL);
7150         if (!trans)
7151                 return -ENOMEM;
7152
7153         nft_setelem_data_deactivate(ctx->net, set, elem);
7154         nft_trans_elem_set(trans) = set;
7155         nft_trans_elem(trans) = *elem;
7156         nft_trans_commit_list_add_tail(ctx->net, trans);
7157
7158         return 0;
7159 }
7160
7161 static int nft_set_catchall_flush(const struct nft_ctx *ctx,
7162                                   struct nft_set *set)
7163 {
7164         u8 genmask = nft_genmask_next(ctx->net);
7165         struct nft_set_elem_catchall *catchall;
7166         struct nft_set_elem elem;
7167         struct nft_set_ext *ext;
7168         int ret = 0;
7169
7170         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
7171                 ext = nft_set_elem_ext(set, catchall->elem);
7172                 if (!nft_set_elem_active(ext, genmask))
7173                         continue;
7174
7175                 elem.priv = catchall->elem;
7176                 ret = __nft_set_catchall_flush(ctx, set, &elem);
7177                 if (ret < 0)
7178                         break;
7179                 nft_set_elem_change_active(ctx->net, set, ext);
7180         }
7181
7182         return ret;
7183 }
7184
7185 static int nft_set_flush(struct nft_ctx *ctx, struct nft_set *set, u8 genmask)
7186 {
7187         struct nft_set_iter iter = {
7188                 .genmask        = genmask,
7189                 .fn             = nft_setelem_flush,
7190         };
7191
7192         set->ops->walk(ctx, set, &iter);
7193         if (!iter.err)
7194                 iter.err = nft_set_catchall_flush(ctx, set);
7195
7196         return iter.err;
7197 }
7198
7199 static int nf_tables_delsetelem(struct sk_buff *skb,
7200                                 const struct nfnl_info *info,
7201                                 const struct nlattr * const nla[])
7202 {
7203         struct netlink_ext_ack *extack = info->extack;
7204         u8 genmask = nft_genmask_next(info->net);
7205         u8 family = info->nfmsg->nfgen_family;
7206         struct net *net = info->net;
7207         const struct nlattr *attr;
7208         struct nft_table *table;
7209         struct nft_set *set;
7210         struct nft_ctx ctx;
7211         int rem, err = 0;
7212
7213         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
7214                                  genmask, NETLINK_CB(skb).portid);
7215         if (IS_ERR(table)) {
7216                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
7217                 return PTR_ERR(table);
7218         }
7219
7220         set = nft_set_lookup(table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
7221         if (IS_ERR(set))
7222                 return PTR_ERR(set);
7223
7224         if (nft_set_is_anonymous(set))
7225                 return -EOPNOTSUPP;
7226
7227         if (!list_empty(&set->bindings) && (set->flags & NFT_SET_CONSTANT))
7228                 return -EBUSY;
7229
7230         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7231
7232         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
7233                 return nft_set_flush(&ctx, set, genmask);
7234
7235         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
7236                 err = nft_del_setelem(&ctx, set, attr);
7237                 if (err == -ENOENT &&
7238                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYSETELEM)
7239                         continue;
7240
7241                 if (err < 0) {
7242                         NL_SET_BAD_ATTR(extack, attr);
7243                         return err;
7244                 }
7245         }
7246
7247         return 0;
7248 }
7249
7250 /*
7251  * Stateful objects
7252  */
7253
7254 /**
7255  *      nft_register_obj- register nf_tables stateful object type
7256  *      @obj_type: object type
7257  *
7258  *      Registers the object type for use with nf_tables. Returns zero on
7259  *      success or a negative errno code otherwise.
7260  */
7261 int nft_register_obj(struct nft_object_type *obj_type)
7262 {
7263         if (obj_type->type == NFT_OBJECT_UNSPEC)
7264                 return -EINVAL;
7265
7266         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7267         list_add_rcu(&obj_type->list, &nf_tables_objects);
7268         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7269         return 0;
7270 }
7271 EXPORT_SYMBOL_GPL(nft_register_obj);
7272
7273 /**
7274  *      nft_unregister_obj - unregister nf_tables object type
7275  *      @obj_type: object type
7276  *
7277  *      Unregisters the object type for use with nf_tables.
7278  */
7279 void nft_unregister_obj(struct nft_object_type *obj_type)
7280 {
7281         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7282         list_del_rcu(&obj_type->list);
7283         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7284 }
7285 EXPORT_SYMBOL_GPL(nft_unregister_obj);
7286
7287 struct nft_object *nft_obj_lookup(const struct net *net,
7288                                   const struct nft_table *table,
7289                                   const struct nlattr *nla, u32 objtype,
7290                                   u8 genmask)
7291 {
7292         struct nft_object_hash_key k = { .table = table };
7293         char search[NFT_OBJ_MAXNAMELEN];
7294         struct rhlist_head *tmp, *list;
7295         struct nft_object *obj;
7296
7297         nla_strscpy(search, nla, sizeof(search));
7298         k.name = search;
7299
7300         WARN_ON_ONCE(!rcu_read_lock_held() &&
7301                      !lockdep_commit_lock_is_held(net));
7302
7303         rcu_read_lock();
7304         list = rhltable_lookup(&nft_objname_ht, &k, nft_objname_ht_params);
7305         if (!list)
7306                 goto out;
7307
7308         rhl_for_each_entry_rcu(obj, tmp, list, rhlhead) {
7309                 if (objtype == obj->ops->type->type &&
7310                     nft_active_genmask(obj, genmask)) {
7311                         rcu_read_unlock();
7312                         return obj;
7313                 }
7314         }
7315 out:
7316         rcu_read_unlock();
7317         return ERR_PTR(-ENOENT);
7318 }
7319 EXPORT_SYMBOL_GPL(nft_obj_lookup);
7320
7321 static struct nft_object *nft_obj_lookup_byhandle(const struct nft_table *table,
7322                                                   const struct nlattr *nla,
7323                                                   u32 objtype, u8 genmask)
7324 {
7325         struct nft_object *obj;
7326
7327         list_for_each_entry(obj, &table->objects, list) {
7328                 if (be64_to_cpu(nla_get_be64(nla)) == obj->handle &&
7329                     objtype == obj->ops->type->type &&
7330                     nft_active_genmask(obj, genmask))
7331                         return obj;
7332         }
7333         return ERR_PTR(-ENOENT);
7334 }
7335
7336 static const struct nla_policy nft_obj_policy[NFTA_OBJ_MAX + 1] = {
7337         [NFTA_OBJ_TABLE]        = { .type = NLA_STRING,
7338                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
7339         [NFTA_OBJ_NAME]         = { .type = NLA_STRING,
7340                                     .len = NFT_OBJ_MAXNAMELEN - 1 },
7341         [NFTA_OBJ_TYPE]         = { .type = NLA_U32 },
7342         [NFTA_OBJ_DATA]         = { .type = NLA_NESTED },
7343         [NFTA_OBJ_HANDLE]       = { .type = NLA_U64},
7344         [NFTA_OBJ_USERDATA]     = { .type = NLA_BINARY,
7345                                     .len = NFT_USERDATA_MAXLEN },
7346 };
7347
7348 static struct nft_object *nft_obj_init(const struct nft_ctx *ctx,
7349                                        const struct nft_object_type *type,
7350                                        const struct nlattr *attr)
7351 {
7352         struct nlattr **tb;
7353         const struct nft_object_ops *ops;
7354         struct nft_object *obj;
7355         int err = -ENOMEM;
7356
7357         tb = kmalloc_array(type->maxattr + 1, sizeof(*tb), GFP_KERNEL);
7358         if (!tb)
7359                 goto err1;
7360
7361         if (attr) {
7362                 err = nla_parse_nested_deprecated(tb, type->maxattr, attr,
7363                                                   type->policy, NULL);
7364                 if (err < 0)
7365                         goto err2;
7366         } else {
7367                 memset(tb, 0, sizeof(tb[0]) * (type->maxattr + 1));
7368         }
7369
7370         if (type->select_ops) {
7371                 ops = type->select_ops(ctx, (const struct nlattr * const *)tb);
7372                 if (IS_ERR(ops)) {
7373                         err = PTR_ERR(ops);
7374                         goto err2;
7375                 }
7376         } else {
7377                 ops = type->ops;
7378         }
7379
7380         err = -ENOMEM;
7381         obj = kzalloc(sizeof(*obj) + ops->size, GFP_KERNEL_ACCOUNT);
7382         if (!obj)
7383                 goto err2;
7384
7385         err = ops->init(ctx, (const struct nlattr * const *)tb, obj);
7386         if (err < 0)
7387                 goto err3;
7388
7389         obj->ops = ops;
7390
7391         kfree(tb);
7392         return obj;
7393 err3:
7394         kfree(obj);
7395 err2:
7396         kfree(tb);
7397 err1:
7398         return ERR_PTR(err);
7399 }
7400
7401 static int nft_object_dump(struct sk_buff *skb, unsigned int attr,
7402                            struct nft_object *obj, bool reset)
7403 {
7404         struct nlattr *nest;
7405
7406         nest = nla_nest_start_noflag(skb, attr);
7407         if (!nest)
7408                 goto nla_put_failure;
7409         if (obj->ops->dump(skb, obj, reset) < 0)
7410                 goto nla_put_failure;
7411         nla_nest_end(skb, nest);
7412         return 0;
7413
7414 nla_put_failure:
7415         return -1;
7416 }
7417
7418 static const struct nft_object_type *__nft_obj_type_get(u32 objtype, u8 family)
7419 {
7420         const struct nft_object_type *type;
7421
7422         list_for_each_entry(type, &nf_tables_objects, list) {
7423                 if (type->family != NFPROTO_UNSPEC &&
7424                     type->family != family)
7425                         continue;
7426
7427                 if (objtype == type->type)
7428                         return type;
7429         }
7430         return NULL;
7431 }
7432
7433 static const struct nft_object_type *
7434 nft_obj_type_get(struct net *net, u32 objtype, u8 family)
7435 {
7436         const struct nft_object_type *type;
7437
7438         type = __nft_obj_type_get(objtype, family);
7439         if (type != NULL && try_module_get(type->owner))
7440                 return type;
7441
7442         lockdep_nfnl_nft_mutex_not_held();
7443 #ifdef CONFIG_MODULES
7444         if (type == NULL) {
7445                 if (nft_request_module(net, "nft-obj-%u", objtype) == -EAGAIN)
7446                         return ERR_PTR(-EAGAIN);
7447         }
7448 #endif
7449         return ERR_PTR(-ENOENT);
7450 }
7451
7452 static int nf_tables_updobj(const struct nft_ctx *ctx,
7453                             const struct nft_object_type *type,
7454                             const struct nlattr *attr,
7455                             struct nft_object *obj)
7456 {
7457         struct nft_object *newobj;
7458         struct nft_trans *trans;
7459         int err = -ENOMEM;
7460
7461         if (!try_module_get(type->owner))
7462                 return -ENOENT;
7463
7464         trans = nft_trans_alloc(ctx, NFT_MSG_NEWOBJ,
7465                                 sizeof(struct nft_trans_obj));
7466         if (!trans)
7467                 goto err_trans;
7468
7469         newobj = nft_obj_init(ctx, type, attr);
7470         if (IS_ERR(newobj)) {
7471                 err = PTR_ERR(newobj);
7472                 goto err_free_trans;
7473         }
7474
7475         nft_trans_obj(trans) = obj;
7476         nft_trans_obj_update(trans) = true;
7477         nft_trans_obj_newobj(trans) = newobj;
7478         nft_trans_commit_list_add_tail(ctx->net, trans);
7479
7480         return 0;
7481
7482 err_free_trans:
7483         kfree(trans);
7484 err_trans:
7485         module_put(type->owner);
7486         return err;
7487 }
7488
7489 static int nf_tables_newobj(struct sk_buff *skb, const struct nfnl_info *info,
7490                             const struct nlattr * const nla[])
7491 {
7492         struct netlink_ext_ack *extack = info->extack;
7493         u8 genmask = nft_genmask_next(info->net);
7494         u8 family = info->nfmsg->nfgen_family;
7495         const struct nft_object_type *type;
7496         struct net *net = info->net;
7497         struct nft_table *table;
7498         struct nft_object *obj;
7499         struct nft_ctx ctx;
7500         u32 objtype;
7501         int err;
7502
7503         if (!nla[NFTA_OBJ_TYPE] ||
7504             !nla[NFTA_OBJ_NAME] ||
7505             !nla[NFTA_OBJ_DATA])
7506                 return -EINVAL;
7507
7508         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
7509                                  NETLINK_CB(skb).portid);
7510         if (IS_ERR(table)) {
7511                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7512                 return PTR_ERR(table);
7513         }
7514
7515         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7516         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
7517         if (IS_ERR(obj)) {
7518                 err = PTR_ERR(obj);
7519                 if (err != -ENOENT) {
7520                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7521                         return err;
7522                 }
7523         } else {
7524                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
7525                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7526                         return -EEXIST;
7527                 }
7528                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
7529                         return -EOPNOTSUPP;
7530
7531                 type = __nft_obj_type_get(objtype, family);
7532                 if (WARN_ON_ONCE(!type))
7533                         return -ENOENT;
7534
7535                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7536
7537                 return nf_tables_updobj(&ctx, type, nla[NFTA_OBJ_DATA], obj);
7538         }
7539
7540         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7541
7542         if (!nft_use_inc(&table->use))
7543                 return -EMFILE;
7544
7545         type = nft_obj_type_get(net, objtype, family);
7546         if (IS_ERR(type)) {
7547                 err = PTR_ERR(type);
7548                 goto err_type;
7549         }
7550
7551         obj = nft_obj_init(&ctx, type, nla[NFTA_OBJ_DATA]);
7552         if (IS_ERR(obj)) {
7553                 err = PTR_ERR(obj);
7554                 goto err_init;
7555         }
7556         obj->key.table = table;
7557         obj->handle = nf_tables_alloc_handle(table);
7558
7559         obj->key.name = nla_strdup(nla[NFTA_OBJ_NAME], GFP_KERNEL_ACCOUNT);
7560         if (!obj->key.name) {
7561                 err = -ENOMEM;
7562                 goto err_strdup;
7563         }
7564
7565         if (nla[NFTA_OBJ_USERDATA]) {
7566                 obj->udata = nla_memdup(nla[NFTA_OBJ_USERDATA], GFP_KERNEL_ACCOUNT);
7567                 if (obj->udata == NULL)
7568                         goto err_userdata;
7569
7570                 obj->udlen = nla_len(nla[NFTA_OBJ_USERDATA]);
7571         }
7572
7573         err = nft_trans_obj_add(&ctx, NFT_MSG_NEWOBJ, obj);
7574         if (err < 0)
7575                 goto err_trans;
7576
7577         err = rhltable_insert(&nft_objname_ht, &obj->rhlhead,
7578                               nft_objname_ht_params);
7579         if (err < 0)
7580                 goto err_obj_ht;
7581
7582         list_add_tail_rcu(&obj->list, &table->objects);
7583
7584         return 0;
7585 err_obj_ht:
7586         /* queued in transaction log */
7587         INIT_LIST_HEAD(&obj->list);
7588         return err;
7589 err_trans:
7590         kfree(obj->udata);
7591 err_userdata:
7592         kfree(obj->key.name);
7593 err_strdup:
7594         if (obj->ops->destroy)
7595                 obj->ops->destroy(&ctx, obj);
7596         kfree(obj);
7597 err_init:
7598         module_put(type->owner);
7599 err_type:
7600         nft_use_dec_restore(&table->use);
7601
7602         return err;
7603 }
7604
7605 static int nf_tables_fill_obj_info(struct sk_buff *skb, struct net *net,
7606                                    u32 portid, u32 seq, int event, u32 flags,
7607                                    int family, const struct nft_table *table,
7608                                    struct nft_object *obj, bool reset)
7609 {
7610         struct nlmsghdr *nlh;
7611
7612         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
7613         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
7614                            NFNETLINK_V0, nft_base_seq(net));
7615         if (!nlh)
7616                 goto nla_put_failure;
7617
7618         if (nla_put_string(skb, NFTA_OBJ_TABLE, table->name) ||
7619             nla_put_string(skb, NFTA_OBJ_NAME, obj->key.name) ||
7620             nla_put_be64(skb, NFTA_OBJ_HANDLE, cpu_to_be64(obj->handle),
7621                          NFTA_OBJ_PAD))
7622                 goto nla_put_failure;
7623
7624         if (event == NFT_MSG_DELOBJ) {
7625                 nlmsg_end(skb, nlh);
7626                 return 0;
7627         }
7628
7629         if (nla_put_be32(skb, NFTA_OBJ_TYPE, htonl(obj->ops->type->type)) ||
7630             nla_put_be32(skb, NFTA_OBJ_USE, htonl(obj->use)) ||
7631             nft_object_dump(skb, NFTA_OBJ_DATA, obj, reset))
7632                 goto nla_put_failure;
7633
7634         if (obj->udata &&
7635             nla_put(skb, NFTA_OBJ_USERDATA, obj->udlen, obj->udata))
7636                 goto nla_put_failure;
7637
7638         nlmsg_end(skb, nlh);
7639         return 0;
7640
7641 nla_put_failure:
7642         nlmsg_trim(skb, nlh);
7643         return -1;
7644 }
7645
7646 static void audit_log_obj_reset(const struct nft_table *table,
7647                                 unsigned int base_seq, unsigned int nentries)
7648 {
7649         char *buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, base_seq);
7650
7651         audit_log_nfcfg(buf, table->family, nentries,
7652                         AUDIT_NFT_OP_OBJ_RESET, GFP_ATOMIC);
7653         kfree(buf);
7654 }
7655
7656 struct nft_obj_filter {
7657         char            *table;
7658         u32             type;
7659 };
7660
7661 static int nf_tables_dump_obj(struct sk_buff *skb, struct netlink_callback *cb)
7662 {
7663         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
7664         const struct nft_table *table;
7665         unsigned int idx = 0, s_idx = cb->args[0];
7666         struct nft_obj_filter *filter = cb->data;
7667         struct net *net = sock_net(skb->sk);
7668         int family = nfmsg->nfgen_family;
7669         struct nftables_pernet *nft_net;
7670         unsigned int entries = 0;
7671         struct nft_object *obj;
7672         bool reset = false;
7673         int rc = 0;
7674
7675         if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
7676                 reset = true;
7677
7678         rcu_read_lock();
7679         nft_net = nft_pernet(net);
7680         cb->seq = READ_ONCE(nft_net->base_seq);
7681
7682         list_for_each_entry_rcu(table, &nft_net->tables, list) {
7683                 if (family != NFPROTO_UNSPEC && family != table->family)
7684                         continue;
7685
7686                 entries = 0;
7687                 list_for_each_entry_rcu(obj, &table->objects, list) {
7688                         if (!nft_is_active(net, obj))
7689                                 goto cont;
7690                         if (idx < s_idx)
7691                                 goto cont;
7692                         if (idx > s_idx)
7693                                 memset(&cb->args[1], 0,
7694                                        sizeof(cb->args) - sizeof(cb->args[0]));
7695                         if (filter && filter->table &&
7696                             strcmp(filter->table, table->name))
7697                                 goto cont;
7698                         if (filter &&
7699                             filter->type != NFT_OBJECT_UNSPEC &&
7700                             obj->ops->type->type != filter->type)
7701                                 goto cont;
7702
7703                         rc = nf_tables_fill_obj_info(skb, net,
7704                                                      NETLINK_CB(cb->skb).portid,
7705                                                      cb->nlh->nlmsg_seq,
7706                                                      NFT_MSG_NEWOBJ,
7707                                                      NLM_F_MULTI | NLM_F_APPEND,
7708                                                      table->family, table,
7709                                                      obj, reset);
7710                         if (rc < 0)
7711                                 break;
7712
7713                         entries++;
7714                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
7715 cont:
7716                         idx++;
7717                 }
7718                 if (reset && entries)
7719                         audit_log_obj_reset(table, nft_net->base_seq, entries);
7720                 if (rc < 0)
7721                         break;
7722         }
7723         rcu_read_unlock();
7724
7725         cb->args[0] = idx;
7726         return skb->len;
7727 }
7728
7729 static int nf_tables_dump_obj_start(struct netlink_callback *cb)
7730 {
7731         const struct nlattr * const *nla = cb->data;
7732         struct nft_obj_filter *filter = NULL;
7733
7734         if (nla[NFTA_OBJ_TABLE] || nla[NFTA_OBJ_TYPE]) {
7735                 filter = kzalloc(sizeof(*filter), GFP_ATOMIC);
7736                 if (!filter)
7737                         return -ENOMEM;
7738
7739                 if (nla[NFTA_OBJ_TABLE]) {
7740                         filter->table = nla_strdup(nla[NFTA_OBJ_TABLE], GFP_ATOMIC);
7741                         if (!filter->table) {
7742                                 kfree(filter);
7743                                 return -ENOMEM;
7744                         }
7745                 }
7746
7747                 if (nla[NFTA_OBJ_TYPE])
7748                         filter->type = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7749         }
7750
7751         cb->data = filter;
7752         return 0;
7753 }
7754
7755 static int nf_tables_dump_obj_done(struct netlink_callback *cb)
7756 {
7757         struct nft_obj_filter *filter = cb->data;
7758
7759         if (filter) {
7760                 kfree(filter->table);
7761                 kfree(filter);
7762         }
7763
7764         return 0;
7765 }
7766
7767 /* called with rcu_read_lock held */
7768 static int nf_tables_getobj(struct sk_buff *skb, const struct nfnl_info *info,
7769                             const struct nlattr * const nla[])
7770 {
7771         struct netlink_ext_ack *extack = info->extack;
7772         u8 genmask = nft_genmask_cur(info->net);
7773         u8 family = info->nfmsg->nfgen_family;
7774         const struct nft_table *table;
7775         struct net *net = info->net;
7776         struct nft_object *obj;
7777         struct sk_buff *skb2;
7778         bool reset = false;
7779         u32 objtype;
7780         int err;
7781
7782         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
7783                 struct netlink_dump_control c = {
7784                         .start = nf_tables_dump_obj_start,
7785                         .dump = nf_tables_dump_obj,
7786                         .done = nf_tables_dump_obj_done,
7787                         .module = THIS_MODULE,
7788                         .data = (void *)nla,
7789                 };
7790
7791                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
7792         }
7793
7794         if (!nla[NFTA_OBJ_NAME] ||
7795             !nla[NFTA_OBJ_TYPE])
7796                 return -EINVAL;
7797
7798         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask, 0);
7799         if (IS_ERR(table)) {
7800                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7801                 return PTR_ERR(table);
7802         }
7803
7804         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7805         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
7806         if (IS_ERR(obj)) {
7807                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7808                 return PTR_ERR(obj);
7809         }
7810
7811         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
7812         if (!skb2)
7813                 return -ENOMEM;
7814
7815         if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
7816                 reset = true;
7817
7818         if (reset) {
7819                 const struct nftables_pernet *nft_net;
7820                 char *buf;
7821
7822                 nft_net = nft_pernet(net);
7823                 buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, nft_net->base_seq);
7824
7825                 audit_log_nfcfg(buf,
7826                                 family,
7827                                 1,
7828                                 AUDIT_NFT_OP_OBJ_RESET,
7829                                 GFP_ATOMIC);
7830                 kfree(buf);
7831         }
7832
7833         err = nf_tables_fill_obj_info(skb2, net, NETLINK_CB(skb).portid,
7834                                       info->nlh->nlmsg_seq, NFT_MSG_NEWOBJ, 0,
7835                                       family, table, obj, reset);
7836         if (err < 0)
7837                 goto err_fill_obj_info;
7838
7839         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
7840
7841 err_fill_obj_info:
7842         kfree_skb(skb2);
7843         return err;
7844 }
7845
7846 static void nft_obj_destroy(const struct nft_ctx *ctx, struct nft_object *obj)
7847 {
7848         if (obj->ops->destroy)
7849                 obj->ops->destroy(ctx, obj);
7850
7851         module_put(obj->ops->type->owner);
7852         kfree(obj->key.name);
7853         kfree(obj->udata);
7854         kfree(obj);
7855 }
7856
7857 static int nf_tables_delobj(struct sk_buff *skb, const struct nfnl_info *info,
7858                             const struct nlattr * const nla[])
7859 {
7860         struct netlink_ext_ack *extack = info->extack;
7861         u8 genmask = nft_genmask_next(info->net);
7862         u8 family = info->nfmsg->nfgen_family;
7863         struct net *net = info->net;
7864         const struct nlattr *attr;
7865         struct nft_table *table;
7866         struct nft_object *obj;
7867         struct nft_ctx ctx;
7868         u32 objtype;
7869
7870         if (!nla[NFTA_OBJ_TYPE] ||
7871             (!nla[NFTA_OBJ_NAME] && !nla[NFTA_OBJ_HANDLE]))
7872                 return -EINVAL;
7873
7874         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
7875                                  NETLINK_CB(skb).portid);
7876         if (IS_ERR(table)) {
7877                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7878                 return PTR_ERR(table);
7879         }
7880
7881         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7882         if (nla[NFTA_OBJ_HANDLE]) {
7883                 attr = nla[NFTA_OBJ_HANDLE];
7884                 obj = nft_obj_lookup_byhandle(table, attr, objtype, genmask);
7885         } else {
7886                 attr = nla[NFTA_OBJ_NAME];
7887                 obj = nft_obj_lookup(net, table, attr, objtype, genmask);
7888         }
7889
7890         if (IS_ERR(obj)) {
7891                 if (PTR_ERR(obj) == -ENOENT &&
7892                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYOBJ)
7893                         return 0;
7894
7895                 NL_SET_BAD_ATTR(extack, attr);
7896                 return PTR_ERR(obj);
7897         }
7898         if (obj->use > 0) {
7899                 NL_SET_BAD_ATTR(extack, attr);
7900                 return -EBUSY;
7901         }
7902
7903         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7904
7905         return nft_delobj(&ctx, obj);
7906 }
7907
7908 static void
7909 __nft_obj_notify(struct net *net, const struct nft_table *table,
7910                  struct nft_object *obj, u32 portid, u32 seq, int event,
7911                  u16 flags, int family, int report, gfp_t gfp)
7912 {
7913         struct nftables_pernet *nft_net = nft_pernet(net);
7914         struct sk_buff *skb;
7915         int err;
7916
7917         if (!report &&
7918             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
7919                 return;
7920
7921         skb = nlmsg_new(NLMSG_GOODSIZE, gfp);
7922         if (skb == NULL)
7923                 goto err;
7924
7925         err = nf_tables_fill_obj_info(skb, net, portid, seq, event,
7926                                       flags & (NLM_F_CREATE | NLM_F_EXCL),
7927                                       family, table, obj, false);
7928         if (err < 0) {
7929                 kfree_skb(skb);
7930                 goto err;
7931         }
7932
7933         nft_notify_enqueue(skb, report, &nft_net->notify_list);
7934         return;
7935 err:
7936         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
7937 }
7938
7939 void nft_obj_notify(struct net *net, const struct nft_table *table,
7940                     struct nft_object *obj, u32 portid, u32 seq, int event,
7941                     u16 flags, int family, int report, gfp_t gfp)
7942 {
7943         struct nftables_pernet *nft_net = nft_pernet(net);
7944         char *buf = kasprintf(gfp, "%s:%u",
7945                               table->name, nft_net->base_seq);
7946
7947         audit_log_nfcfg(buf,
7948                         family,
7949                         obj->handle,
7950                         event == NFT_MSG_NEWOBJ ?
7951                                  AUDIT_NFT_OP_OBJ_REGISTER :
7952                                  AUDIT_NFT_OP_OBJ_UNREGISTER,
7953                         gfp);
7954         kfree(buf);
7955
7956         __nft_obj_notify(net, table, obj, portid, seq, event,
7957                          flags, family, report, gfp);
7958 }
7959 EXPORT_SYMBOL_GPL(nft_obj_notify);
7960
7961 static void nf_tables_obj_notify(const struct nft_ctx *ctx,
7962                                  struct nft_object *obj, int event)
7963 {
7964         __nft_obj_notify(ctx->net, ctx->table, obj, ctx->portid,
7965                          ctx->seq, event, ctx->flags, ctx->family,
7966                          ctx->report, GFP_KERNEL);
7967 }
7968
7969 /*
7970  * Flow tables
7971  */
7972 void nft_register_flowtable_type(struct nf_flowtable_type *type)
7973 {
7974         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7975         list_add_tail_rcu(&type->list, &nf_tables_flowtables);
7976         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7977 }
7978 EXPORT_SYMBOL_GPL(nft_register_flowtable_type);
7979
7980 void nft_unregister_flowtable_type(struct nf_flowtable_type *type)
7981 {
7982         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7983         list_del_rcu(&type->list);
7984         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7985 }
7986 EXPORT_SYMBOL_GPL(nft_unregister_flowtable_type);
7987
7988 static const struct nla_policy nft_flowtable_policy[NFTA_FLOWTABLE_MAX + 1] = {
7989         [NFTA_FLOWTABLE_TABLE]          = { .type = NLA_STRING,
7990                                             .len = NFT_NAME_MAXLEN - 1 },
7991         [NFTA_FLOWTABLE_NAME]           = { .type = NLA_STRING,
7992                                             .len = NFT_NAME_MAXLEN - 1 },
7993         [NFTA_FLOWTABLE_HOOK]           = { .type = NLA_NESTED },
7994         [NFTA_FLOWTABLE_HANDLE]         = { .type = NLA_U64 },
7995         [NFTA_FLOWTABLE_FLAGS]          = { .type = NLA_U32 },
7996 };
7997
7998 struct nft_flowtable *nft_flowtable_lookup(const struct nft_table *table,
7999                                            const struct nlattr *nla, u8 genmask)
8000 {
8001         struct nft_flowtable *flowtable;
8002
8003         list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
8004                 if (!nla_strcmp(nla, flowtable->name) &&
8005                     nft_active_genmask(flowtable, genmask))
8006                         return flowtable;
8007         }
8008         return ERR_PTR(-ENOENT);
8009 }
8010 EXPORT_SYMBOL_GPL(nft_flowtable_lookup);
8011
8012 void nf_tables_deactivate_flowtable(const struct nft_ctx *ctx,
8013                                     struct nft_flowtable *flowtable,
8014                                     enum nft_trans_phase phase)
8015 {
8016         switch (phase) {
8017         case NFT_TRANS_PREPARE_ERROR:
8018         case NFT_TRANS_PREPARE:
8019         case NFT_TRANS_ABORT:
8020         case NFT_TRANS_RELEASE:
8021                 nft_use_dec(&flowtable->use);
8022                 fallthrough;
8023         default:
8024                 return;
8025         }
8026 }
8027 EXPORT_SYMBOL_GPL(nf_tables_deactivate_flowtable);
8028
8029 static struct nft_flowtable *
8030 nft_flowtable_lookup_byhandle(const struct nft_table *table,
8031                               const struct nlattr *nla, u8 genmask)
8032 {
8033        struct nft_flowtable *flowtable;
8034
8035        list_for_each_entry(flowtable, &table->flowtables, list) {
8036                if (be64_to_cpu(nla_get_be64(nla)) == flowtable->handle &&
8037                    nft_active_genmask(flowtable, genmask))
8038                        return flowtable;
8039        }
8040        return ERR_PTR(-ENOENT);
8041 }
8042
8043 struct nft_flowtable_hook {
8044         u32                     num;
8045         int                     priority;
8046         struct list_head        list;
8047 };
8048
8049 static const struct nla_policy nft_flowtable_hook_policy[NFTA_FLOWTABLE_HOOK_MAX + 1] = {
8050         [NFTA_FLOWTABLE_HOOK_NUM]       = { .type = NLA_U32 },
8051         [NFTA_FLOWTABLE_HOOK_PRIORITY]  = { .type = NLA_U32 },
8052         [NFTA_FLOWTABLE_HOOK_DEVS]      = { .type = NLA_NESTED },
8053 };
8054
8055 static int nft_flowtable_parse_hook(const struct nft_ctx *ctx,
8056                                     const struct nlattr * const nla[],
8057                                     struct nft_flowtable_hook *flowtable_hook,
8058                                     struct nft_flowtable *flowtable,
8059                                     struct netlink_ext_ack *extack, bool add)
8060 {
8061         struct nlattr *tb[NFTA_FLOWTABLE_HOOK_MAX + 1];
8062         struct nft_hook *hook;
8063         int hooknum, priority;
8064         int err;
8065
8066         INIT_LIST_HEAD(&flowtable_hook->list);
8067
8068         err = nla_parse_nested_deprecated(tb, NFTA_FLOWTABLE_HOOK_MAX,
8069                                           nla[NFTA_FLOWTABLE_HOOK],
8070                                           nft_flowtable_hook_policy, NULL);
8071         if (err < 0)
8072                 return err;
8073
8074         if (add) {
8075                 if (!tb[NFTA_FLOWTABLE_HOOK_NUM] ||
8076                     !tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
8077                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8078                         return -ENOENT;
8079                 }
8080
8081                 hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
8082                 if (hooknum != NF_NETDEV_INGRESS)
8083                         return -EOPNOTSUPP;
8084
8085                 priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
8086
8087                 flowtable_hook->priority        = priority;
8088                 flowtable_hook->num             = hooknum;
8089         } else {
8090                 if (tb[NFTA_FLOWTABLE_HOOK_NUM]) {
8091                         hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
8092                         if (hooknum != flowtable->hooknum)
8093                                 return -EOPNOTSUPP;
8094                 }
8095
8096                 if (tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
8097                         priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
8098                         if (priority != flowtable->data.priority)
8099                                 return -EOPNOTSUPP;
8100                 }
8101
8102                 flowtable_hook->priority        = flowtable->data.priority;
8103                 flowtable_hook->num             = flowtable->hooknum;
8104         }
8105
8106         if (tb[NFTA_FLOWTABLE_HOOK_DEVS]) {
8107                 err = nf_tables_parse_netdev_hooks(ctx->net,
8108                                                    tb[NFTA_FLOWTABLE_HOOK_DEVS],
8109                                                    &flowtable_hook->list,
8110                                                    extack);
8111                 if (err < 0)
8112                         return err;
8113         }
8114
8115         list_for_each_entry(hook, &flowtable_hook->list, list) {
8116                 hook->ops.pf            = NFPROTO_NETDEV;
8117                 hook->ops.hooknum       = flowtable_hook->num;
8118                 hook->ops.priority      = flowtable_hook->priority;
8119                 hook->ops.priv          = &flowtable->data;
8120                 hook->ops.hook          = flowtable->data.type->hook;
8121         }
8122
8123         return err;
8124 }
8125
8126 static const struct nf_flowtable_type *__nft_flowtable_type_get(u8 family)
8127 {
8128         const struct nf_flowtable_type *type;
8129
8130         list_for_each_entry(type, &nf_tables_flowtables, list) {
8131                 if (family == type->family)
8132                         return type;
8133         }
8134         return NULL;
8135 }
8136
8137 static const struct nf_flowtable_type *
8138 nft_flowtable_type_get(struct net *net, u8 family)
8139 {
8140         const struct nf_flowtable_type *type;
8141
8142         type = __nft_flowtable_type_get(family);
8143         if (type != NULL && try_module_get(type->owner))
8144                 return type;
8145
8146         lockdep_nfnl_nft_mutex_not_held();
8147 #ifdef CONFIG_MODULES
8148         if (type == NULL) {
8149                 if (nft_request_module(net, "nf-flowtable-%u", family) == -EAGAIN)
8150                         return ERR_PTR(-EAGAIN);
8151         }
8152 #endif
8153         return ERR_PTR(-ENOENT);
8154 }
8155
8156 /* Only called from error and netdev event paths. */
8157 static void nft_unregister_flowtable_hook(struct net *net,
8158                                           struct nft_flowtable *flowtable,
8159                                           struct nft_hook *hook)
8160 {
8161         nf_unregister_net_hook(net, &hook->ops);
8162         flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
8163                                     FLOW_BLOCK_UNBIND);
8164 }
8165
8166 static void __nft_unregister_flowtable_net_hooks(struct net *net,
8167                                                  struct list_head *hook_list,
8168                                                  bool release_netdev)
8169 {
8170         struct nft_hook *hook, *next;
8171
8172         list_for_each_entry_safe(hook, next, hook_list, list) {
8173                 nf_unregister_net_hook(net, &hook->ops);
8174                 if (release_netdev) {
8175                         list_del(&hook->list);
8176                         kfree_rcu(hook, rcu);
8177                 }
8178         }
8179 }
8180
8181 static void nft_unregister_flowtable_net_hooks(struct net *net,
8182                                                struct list_head *hook_list)
8183 {
8184         __nft_unregister_flowtable_net_hooks(net, hook_list, false);
8185 }
8186
8187 static int nft_register_flowtable_net_hooks(struct net *net,
8188                                             struct nft_table *table,
8189                                             struct list_head *hook_list,
8190                                             struct nft_flowtable *flowtable)
8191 {
8192         struct nft_hook *hook, *hook2, *next;
8193         struct nft_flowtable *ft;
8194         int err, i = 0;
8195
8196         list_for_each_entry(hook, hook_list, list) {
8197                 list_for_each_entry(ft, &table->flowtables, list) {
8198                         if (!nft_is_active_next(net, ft))
8199                                 continue;
8200
8201                         list_for_each_entry(hook2, &ft->hook_list, list) {
8202                                 if (hook->ops.dev == hook2->ops.dev &&
8203                                     hook->ops.pf == hook2->ops.pf) {
8204                                         err = -EEXIST;
8205                                         goto err_unregister_net_hooks;
8206                                 }
8207                         }
8208                 }
8209
8210                 err = flowtable->data.type->setup(&flowtable->data,
8211                                                   hook->ops.dev,
8212                                                   FLOW_BLOCK_BIND);
8213                 if (err < 0)
8214                         goto err_unregister_net_hooks;
8215
8216                 err = nf_register_net_hook(net, &hook->ops);
8217                 if (err < 0) {
8218                         flowtable->data.type->setup(&flowtable->data,
8219                                                     hook->ops.dev,
8220                                                     FLOW_BLOCK_UNBIND);
8221                         goto err_unregister_net_hooks;
8222                 }
8223
8224                 i++;
8225         }
8226
8227         return 0;
8228
8229 err_unregister_net_hooks:
8230         list_for_each_entry_safe(hook, next, hook_list, list) {
8231                 if (i-- <= 0)
8232                         break;
8233
8234                 nft_unregister_flowtable_hook(net, flowtable, hook);
8235                 list_del_rcu(&hook->list);
8236                 kfree_rcu(hook, rcu);
8237         }
8238
8239         return err;
8240 }
8241
8242 static void nft_hooks_destroy(struct list_head *hook_list)
8243 {
8244         struct nft_hook *hook, *next;
8245
8246         list_for_each_entry_safe(hook, next, hook_list, list) {
8247                 list_del_rcu(&hook->list);
8248                 kfree_rcu(hook, rcu);
8249         }
8250 }
8251
8252 static int nft_flowtable_update(struct nft_ctx *ctx, const struct nlmsghdr *nlh,
8253                                 struct nft_flowtable *flowtable,
8254                                 struct netlink_ext_ack *extack)
8255 {
8256         const struct nlattr * const *nla = ctx->nla;
8257         struct nft_flowtable_hook flowtable_hook;
8258         struct nft_hook *hook, *next;
8259         struct nft_trans *trans;
8260         bool unregister = false;
8261         u32 flags;
8262         int err;
8263
8264         err = nft_flowtable_parse_hook(ctx, nla, &flowtable_hook, flowtable,
8265                                        extack, false);
8266         if (err < 0)
8267                 return err;
8268
8269         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
8270                 if (nft_hook_list_find(&flowtable->hook_list, hook)) {
8271                         list_del(&hook->list);
8272                         kfree(hook);
8273                 }
8274         }
8275
8276         if (nla[NFTA_FLOWTABLE_FLAGS]) {
8277                 flags = ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
8278                 if (flags & ~NFT_FLOWTABLE_MASK) {
8279                         err = -EOPNOTSUPP;
8280                         goto err_flowtable_update_hook;
8281                 }
8282                 if ((flowtable->data.flags & NFT_FLOWTABLE_HW_OFFLOAD) ^
8283                     (flags & NFT_FLOWTABLE_HW_OFFLOAD)) {
8284                         err = -EOPNOTSUPP;
8285                         goto err_flowtable_update_hook;
8286                 }
8287         } else {
8288                 flags = flowtable->data.flags;
8289         }
8290
8291         err = nft_register_flowtable_net_hooks(ctx->net, ctx->table,
8292                                                &flowtable_hook.list, flowtable);
8293         if (err < 0)
8294                 goto err_flowtable_update_hook;
8295
8296         trans = nft_trans_alloc(ctx, NFT_MSG_NEWFLOWTABLE,
8297                                 sizeof(struct nft_trans_flowtable));
8298         if (!trans) {
8299                 unregister = true;
8300                 err = -ENOMEM;
8301                 goto err_flowtable_update_hook;
8302         }
8303
8304         nft_trans_flowtable_flags(trans) = flags;
8305         nft_trans_flowtable(trans) = flowtable;
8306         nft_trans_flowtable_update(trans) = true;
8307         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
8308         list_splice(&flowtable_hook.list, &nft_trans_flowtable_hooks(trans));
8309
8310         nft_trans_commit_list_add_tail(ctx->net, trans);
8311
8312         return 0;
8313
8314 err_flowtable_update_hook:
8315         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
8316                 if (unregister)
8317                         nft_unregister_flowtable_hook(ctx->net, flowtable, hook);
8318                 list_del_rcu(&hook->list);
8319                 kfree_rcu(hook, rcu);
8320         }
8321
8322         return err;
8323
8324 }
8325
8326 static int nf_tables_newflowtable(struct sk_buff *skb,
8327                                   const struct nfnl_info *info,
8328                                   const struct nlattr * const nla[])
8329 {
8330         struct netlink_ext_ack *extack = info->extack;
8331         struct nft_flowtable_hook flowtable_hook;
8332         u8 genmask = nft_genmask_next(info->net);
8333         u8 family = info->nfmsg->nfgen_family;
8334         const struct nf_flowtable_type *type;
8335         struct nft_flowtable *flowtable;
8336         struct nft_hook *hook, *next;
8337         struct net *net = info->net;
8338         struct nft_table *table;
8339         struct nft_ctx ctx;
8340         int err;
8341
8342         if (!nla[NFTA_FLOWTABLE_TABLE] ||
8343             !nla[NFTA_FLOWTABLE_NAME] ||
8344             !nla[NFTA_FLOWTABLE_HOOK])
8345                 return -EINVAL;
8346
8347         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8348                                  genmask, NETLINK_CB(skb).portid);
8349         if (IS_ERR(table)) {
8350                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
8351                 return PTR_ERR(table);
8352         }
8353
8354         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
8355                                          genmask);
8356         if (IS_ERR(flowtable)) {
8357                 err = PTR_ERR(flowtable);
8358                 if (err != -ENOENT) {
8359                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8360                         return err;
8361                 }
8362         } else {
8363                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
8364                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8365                         return -EEXIST;
8366                 }
8367
8368                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8369
8370                 return nft_flowtable_update(&ctx, info->nlh, flowtable, extack);
8371         }
8372
8373         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8374
8375         if (!nft_use_inc(&table->use))
8376                 return -EMFILE;
8377
8378         flowtable = kzalloc(sizeof(*flowtable), GFP_KERNEL_ACCOUNT);
8379         if (!flowtable) {
8380                 err = -ENOMEM;
8381                 goto flowtable_alloc;
8382         }
8383
8384         flowtable->table = table;
8385         flowtable->handle = nf_tables_alloc_handle(table);
8386         INIT_LIST_HEAD(&flowtable->hook_list);
8387
8388         flowtable->name = nla_strdup(nla[NFTA_FLOWTABLE_NAME], GFP_KERNEL_ACCOUNT);
8389         if (!flowtable->name) {
8390                 err = -ENOMEM;
8391                 goto err1;
8392         }
8393
8394         type = nft_flowtable_type_get(net, family);
8395         if (IS_ERR(type)) {
8396                 err = PTR_ERR(type);
8397                 goto err2;
8398         }
8399
8400         if (nla[NFTA_FLOWTABLE_FLAGS]) {
8401                 flowtable->data.flags =
8402                         ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
8403                 if (flowtable->data.flags & ~NFT_FLOWTABLE_MASK) {
8404                         err = -EOPNOTSUPP;
8405                         goto err3;
8406                 }
8407         }
8408
8409         write_pnet(&flowtable->data.net, net);
8410         flowtable->data.type = type;
8411         err = type->init(&flowtable->data);
8412         if (err < 0)
8413                 goto err3;
8414
8415         err = nft_flowtable_parse_hook(&ctx, nla, &flowtable_hook, flowtable,
8416                                        extack, true);
8417         if (err < 0)
8418                 goto err4;
8419
8420         list_splice(&flowtable_hook.list, &flowtable->hook_list);
8421         flowtable->data.priority = flowtable_hook.priority;
8422         flowtable->hooknum = flowtable_hook.num;
8423
8424         err = nft_register_flowtable_net_hooks(ctx.net, table,
8425                                                &flowtable->hook_list,
8426                                                flowtable);
8427         if (err < 0) {
8428                 nft_hooks_destroy(&flowtable->hook_list);
8429                 goto err4;
8430         }
8431
8432         err = nft_trans_flowtable_add(&ctx, NFT_MSG_NEWFLOWTABLE, flowtable);
8433         if (err < 0)
8434                 goto err5;
8435
8436         list_add_tail_rcu(&flowtable->list, &table->flowtables);
8437
8438         return 0;
8439 err5:
8440         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
8441                 nft_unregister_flowtable_hook(net, flowtable, hook);
8442                 list_del_rcu(&hook->list);
8443                 kfree_rcu(hook, rcu);
8444         }
8445 err4:
8446         flowtable->data.type->free(&flowtable->data);
8447 err3:
8448         module_put(type->owner);
8449 err2:
8450         kfree(flowtable->name);
8451 err1:
8452         kfree(flowtable);
8453 flowtable_alloc:
8454         nft_use_dec_restore(&table->use);
8455
8456         return err;
8457 }
8458
8459 static void nft_flowtable_hook_release(struct nft_flowtable_hook *flowtable_hook)
8460 {
8461         struct nft_hook *this, *next;
8462
8463         list_for_each_entry_safe(this, next, &flowtable_hook->list, list) {
8464                 list_del(&this->list);
8465                 kfree(this);
8466         }
8467 }
8468
8469 static int nft_delflowtable_hook(struct nft_ctx *ctx,
8470                                  struct nft_flowtable *flowtable,
8471                                  struct netlink_ext_ack *extack)
8472 {
8473         const struct nlattr * const *nla = ctx->nla;
8474         struct nft_flowtable_hook flowtable_hook;
8475         LIST_HEAD(flowtable_del_list);
8476         struct nft_hook *this, *hook;
8477         struct nft_trans *trans;
8478         int err;
8479
8480         err = nft_flowtable_parse_hook(ctx, nla, &flowtable_hook, flowtable,
8481                                        extack, false);
8482         if (err < 0)
8483                 return err;
8484
8485         list_for_each_entry(this, &flowtable_hook.list, list) {
8486                 hook = nft_hook_list_find(&flowtable->hook_list, this);
8487                 if (!hook) {
8488                         err = -ENOENT;
8489                         goto err_flowtable_del_hook;
8490                 }
8491                 list_move(&hook->list, &flowtable_del_list);
8492         }
8493
8494         trans = nft_trans_alloc(ctx, NFT_MSG_DELFLOWTABLE,
8495                                 sizeof(struct nft_trans_flowtable));
8496         if (!trans) {
8497                 err = -ENOMEM;
8498                 goto err_flowtable_del_hook;
8499         }
8500
8501         nft_trans_flowtable(trans) = flowtable;
8502         nft_trans_flowtable_update(trans) = true;
8503         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
8504         list_splice(&flowtable_del_list, &nft_trans_flowtable_hooks(trans));
8505         nft_flowtable_hook_release(&flowtable_hook);
8506
8507         nft_trans_commit_list_add_tail(ctx->net, trans);
8508
8509         return 0;
8510
8511 err_flowtable_del_hook:
8512         list_splice(&flowtable_del_list, &flowtable->hook_list);
8513         nft_flowtable_hook_release(&flowtable_hook);
8514
8515         return err;
8516 }
8517
8518 static int nf_tables_delflowtable(struct sk_buff *skb,
8519                                   const struct nfnl_info *info,
8520                                   const struct nlattr * const nla[])
8521 {
8522         struct netlink_ext_ack *extack = info->extack;
8523         u8 genmask = nft_genmask_next(info->net);
8524         u8 family = info->nfmsg->nfgen_family;
8525         struct nft_flowtable *flowtable;
8526         struct net *net = info->net;
8527         const struct nlattr *attr;
8528         struct nft_table *table;
8529         struct nft_ctx ctx;
8530
8531         if (!nla[NFTA_FLOWTABLE_TABLE] ||
8532             (!nla[NFTA_FLOWTABLE_NAME] &&
8533              !nla[NFTA_FLOWTABLE_HANDLE]))
8534                 return -EINVAL;
8535
8536         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8537                                  genmask, NETLINK_CB(skb).portid);
8538         if (IS_ERR(table)) {
8539                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
8540                 return PTR_ERR(table);
8541         }
8542
8543         if (nla[NFTA_FLOWTABLE_HANDLE]) {
8544                 attr = nla[NFTA_FLOWTABLE_HANDLE];
8545                 flowtable = nft_flowtable_lookup_byhandle(table, attr, genmask);
8546         } else {
8547                 attr = nla[NFTA_FLOWTABLE_NAME];
8548                 flowtable = nft_flowtable_lookup(table, attr, genmask);
8549         }
8550
8551         if (IS_ERR(flowtable)) {
8552                 if (PTR_ERR(flowtable) == -ENOENT &&
8553                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYFLOWTABLE)
8554                         return 0;
8555
8556                 NL_SET_BAD_ATTR(extack, attr);
8557                 return PTR_ERR(flowtable);
8558         }
8559
8560         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8561
8562         if (nla[NFTA_FLOWTABLE_HOOK])
8563                 return nft_delflowtable_hook(&ctx, flowtable, extack);
8564
8565         if (flowtable->use > 0) {
8566                 NL_SET_BAD_ATTR(extack, attr);
8567                 return -EBUSY;
8568         }
8569
8570         return nft_delflowtable(&ctx, flowtable);
8571 }
8572
8573 static int nf_tables_fill_flowtable_info(struct sk_buff *skb, struct net *net,
8574                                          u32 portid, u32 seq, int event,
8575                                          u32 flags, int family,
8576                                          struct nft_flowtable *flowtable,
8577                                          struct list_head *hook_list)
8578 {
8579         struct nlattr *nest, *nest_devs;
8580         struct nft_hook *hook;
8581         struct nlmsghdr *nlh;
8582
8583         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
8584         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
8585                            NFNETLINK_V0, nft_base_seq(net));
8586         if (!nlh)
8587                 goto nla_put_failure;
8588
8589         if (nla_put_string(skb, NFTA_FLOWTABLE_TABLE, flowtable->table->name) ||
8590             nla_put_string(skb, NFTA_FLOWTABLE_NAME, flowtable->name) ||
8591             nla_put_be64(skb, NFTA_FLOWTABLE_HANDLE, cpu_to_be64(flowtable->handle),
8592                          NFTA_FLOWTABLE_PAD))
8593                 goto nla_put_failure;
8594
8595         if (event == NFT_MSG_DELFLOWTABLE && !hook_list) {
8596                 nlmsg_end(skb, nlh);
8597                 return 0;
8598         }
8599
8600         if (nla_put_be32(skb, NFTA_FLOWTABLE_USE, htonl(flowtable->use)) ||
8601             nla_put_be32(skb, NFTA_FLOWTABLE_FLAGS, htonl(flowtable->data.flags)))
8602                 goto nla_put_failure;
8603
8604         nest = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK);
8605         if (!nest)
8606                 goto nla_put_failure;
8607         if (nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_NUM, htonl(flowtable->hooknum)) ||
8608             nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_PRIORITY, htonl(flowtable->data.priority)))
8609                 goto nla_put_failure;
8610
8611         nest_devs = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK_DEVS);
8612         if (!nest_devs)
8613                 goto nla_put_failure;
8614
8615         if (!hook_list)
8616                 hook_list = &flowtable->hook_list;
8617
8618         list_for_each_entry_rcu(hook, hook_list, list) {
8619                 if (nla_put_string(skb, NFTA_DEVICE_NAME, hook->ops.dev->name))
8620                         goto nla_put_failure;
8621         }
8622         nla_nest_end(skb, nest_devs);
8623         nla_nest_end(skb, nest);
8624
8625         nlmsg_end(skb, nlh);
8626         return 0;
8627
8628 nla_put_failure:
8629         nlmsg_trim(skb, nlh);
8630         return -1;
8631 }
8632
8633 struct nft_flowtable_filter {
8634         char            *table;
8635 };
8636
8637 static int nf_tables_dump_flowtable(struct sk_buff *skb,
8638                                     struct netlink_callback *cb)
8639 {
8640         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
8641         struct nft_flowtable_filter *filter = cb->data;
8642         unsigned int idx = 0, s_idx = cb->args[0];
8643         struct net *net = sock_net(skb->sk);
8644         int family = nfmsg->nfgen_family;
8645         struct nft_flowtable *flowtable;
8646         struct nftables_pernet *nft_net;
8647         const struct nft_table *table;
8648
8649         rcu_read_lock();
8650         nft_net = nft_pernet(net);
8651         cb->seq = READ_ONCE(nft_net->base_seq);
8652
8653         list_for_each_entry_rcu(table, &nft_net->tables, list) {
8654                 if (family != NFPROTO_UNSPEC && family != table->family)
8655                         continue;
8656
8657                 list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
8658                         if (!nft_is_active(net, flowtable))
8659                                 goto cont;
8660                         if (idx < s_idx)
8661                                 goto cont;
8662                         if (idx > s_idx)
8663                                 memset(&cb->args[1], 0,
8664                                        sizeof(cb->args) - sizeof(cb->args[0]));
8665                         if (filter && filter->table &&
8666                             strcmp(filter->table, table->name))
8667                                 goto cont;
8668
8669                         if (nf_tables_fill_flowtable_info(skb, net, NETLINK_CB(cb->skb).portid,
8670                                                           cb->nlh->nlmsg_seq,
8671                                                           NFT_MSG_NEWFLOWTABLE,
8672                                                           NLM_F_MULTI | NLM_F_APPEND,
8673                                                           table->family,
8674                                                           flowtable, NULL) < 0)
8675                                 goto done;
8676
8677                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
8678 cont:
8679                         idx++;
8680                 }
8681         }
8682 done:
8683         rcu_read_unlock();
8684
8685         cb->args[0] = idx;
8686         return skb->len;
8687 }
8688
8689 static int nf_tables_dump_flowtable_start(struct netlink_callback *cb)
8690 {
8691         const struct nlattr * const *nla = cb->data;
8692         struct nft_flowtable_filter *filter = NULL;
8693
8694         if (nla[NFTA_FLOWTABLE_TABLE]) {
8695                 filter = kzalloc(sizeof(*filter), GFP_ATOMIC);
8696                 if (!filter)
8697                         return -ENOMEM;
8698
8699                 filter->table = nla_strdup(nla[NFTA_FLOWTABLE_TABLE],
8700                                            GFP_ATOMIC);
8701                 if (!filter->table) {
8702                         kfree(filter);
8703                         return -ENOMEM;
8704                 }
8705         }
8706
8707         cb->data = filter;
8708         return 0;
8709 }
8710
8711 static int nf_tables_dump_flowtable_done(struct netlink_callback *cb)
8712 {
8713         struct nft_flowtable_filter *filter = cb->data;
8714
8715         if (!filter)
8716                 return 0;
8717
8718         kfree(filter->table);
8719         kfree(filter);
8720
8721         return 0;
8722 }
8723
8724 /* called with rcu_read_lock held */
8725 static int nf_tables_getflowtable(struct sk_buff *skb,
8726                                   const struct nfnl_info *info,
8727                                   const struct nlattr * const nla[])
8728 {
8729         u8 genmask = nft_genmask_cur(info->net);
8730         u8 family = info->nfmsg->nfgen_family;
8731         struct nft_flowtable *flowtable;
8732         const struct nft_table *table;
8733         struct net *net = info->net;
8734         struct sk_buff *skb2;
8735         int err;
8736
8737         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
8738                 struct netlink_dump_control c = {
8739                         .start = nf_tables_dump_flowtable_start,
8740                         .dump = nf_tables_dump_flowtable,
8741                         .done = nf_tables_dump_flowtable_done,
8742                         .module = THIS_MODULE,
8743                         .data = (void *)nla,
8744                 };
8745
8746                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
8747         }
8748
8749         if (!nla[NFTA_FLOWTABLE_NAME])
8750                 return -EINVAL;
8751
8752         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8753                                  genmask, 0);
8754         if (IS_ERR(table))
8755                 return PTR_ERR(table);
8756
8757         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
8758                                          genmask);
8759         if (IS_ERR(flowtable))
8760                 return PTR_ERR(flowtable);
8761
8762         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
8763         if (!skb2)
8764                 return -ENOMEM;
8765
8766         err = nf_tables_fill_flowtable_info(skb2, net, NETLINK_CB(skb).portid,
8767                                             info->nlh->nlmsg_seq,
8768                                             NFT_MSG_NEWFLOWTABLE, 0, family,
8769                                             flowtable, NULL);
8770         if (err < 0)
8771                 goto err_fill_flowtable_info;
8772
8773         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
8774
8775 err_fill_flowtable_info:
8776         kfree_skb(skb2);
8777         return err;
8778 }
8779
8780 static void nf_tables_flowtable_notify(struct nft_ctx *ctx,
8781                                        struct nft_flowtable *flowtable,
8782                                        struct list_head *hook_list, int event)
8783 {
8784         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
8785         struct sk_buff *skb;
8786         u16 flags = 0;
8787         int err;
8788
8789         if (!ctx->report &&
8790             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
8791                 return;
8792
8793         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
8794         if (skb == NULL)
8795                 goto err;
8796
8797         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
8798                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
8799
8800         err = nf_tables_fill_flowtable_info(skb, ctx->net, ctx->portid,
8801                                             ctx->seq, event, flags,
8802                                             ctx->family, flowtable, hook_list);
8803         if (err < 0) {
8804                 kfree_skb(skb);
8805                 goto err;
8806         }
8807
8808         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
8809         return;
8810 err:
8811         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
8812 }
8813
8814 static void nf_tables_flowtable_destroy(struct nft_flowtable *flowtable)
8815 {
8816         struct nft_hook *hook, *next;
8817
8818         flowtable->data.type->free(&flowtable->data);
8819         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
8820                 flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
8821                                             FLOW_BLOCK_UNBIND);
8822                 list_del_rcu(&hook->list);
8823                 kfree(hook);
8824         }
8825         kfree(flowtable->name);
8826         module_put(flowtable->data.type->owner);
8827         kfree(flowtable);
8828 }
8829
8830 static int nf_tables_fill_gen_info(struct sk_buff *skb, struct net *net,
8831                                    u32 portid, u32 seq)
8832 {
8833         struct nftables_pernet *nft_net = nft_pernet(net);
8834         struct nlmsghdr *nlh;
8835         char buf[TASK_COMM_LEN];
8836         int event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWGEN);
8837
8838         nlh = nfnl_msg_put(skb, portid, seq, event, 0, AF_UNSPEC,
8839                            NFNETLINK_V0, nft_base_seq(net));
8840         if (!nlh)
8841                 goto nla_put_failure;
8842
8843         if (nla_put_be32(skb, NFTA_GEN_ID, htonl(nft_net->base_seq)) ||
8844             nla_put_be32(skb, NFTA_GEN_PROC_PID, htonl(task_pid_nr(current))) ||
8845             nla_put_string(skb, NFTA_GEN_PROC_NAME, get_task_comm(buf, current)))
8846                 goto nla_put_failure;
8847
8848         nlmsg_end(skb, nlh);
8849         return 0;
8850
8851 nla_put_failure:
8852         nlmsg_trim(skb, nlh);
8853         return -EMSGSIZE;
8854 }
8855
8856 static void nft_flowtable_event(unsigned long event, struct net_device *dev,
8857                                 struct nft_flowtable *flowtable)
8858 {
8859         struct nft_hook *hook;
8860
8861         list_for_each_entry(hook, &flowtable->hook_list, list) {
8862                 if (hook->ops.dev != dev)
8863                         continue;
8864
8865                 /* flow_offload_netdev_event() cleans up entries for us. */
8866                 nft_unregister_flowtable_hook(dev_net(dev), flowtable, hook);
8867                 list_del_rcu(&hook->list);
8868                 kfree_rcu(hook, rcu);
8869                 break;
8870         }
8871 }
8872
8873 static int nf_tables_flowtable_event(struct notifier_block *this,
8874                                      unsigned long event, void *ptr)
8875 {
8876         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
8877         struct nft_flowtable *flowtable;
8878         struct nftables_pernet *nft_net;
8879         struct nft_table *table;
8880         struct net *net;
8881
8882         if (event != NETDEV_UNREGISTER)
8883                 return 0;
8884
8885         net = dev_net(dev);
8886         nft_net = nft_pernet(net);
8887         mutex_lock(&nft_net->commit_mutex);
8888         list_for_each_entry(table, &nft_net->tables, list) {
8889                 list_for_each_entry(flowtable, &table->flowtables, list) {
8890                         nft_flowtable_event(event, dev, flowtable);
8891                 }
8892         }
8893         mutex_unlock(&nft_net->commit_mutex);
8894
8895         return NOTIFY_DONE;
8896 }
8897
8898 static struct notifier_block nf_tables_flowtable_notifier = {
8899         .notifier_call  = nf_tables_flowtable_event,
8900 };
8901
8902 static void nf_tables_gen_notify(struct net *net, struct sk_buff *skb,
8903                                  int event)
8904 {
8905         struct nlmsghdr *nlh = nlmsg_hdr(skb);
8906         struct sk_buff *skb2;
8907         int err;
8908
8909         if (!nlmsg_report(nlh) &&
8910             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
8911                 return;
8912
8913         skb2 = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
8914         if (skb2 == NULL)
8915                 goto err;
8916
8917         err = nf_tables_fill_gen_info(skb2, net, NETLINK_CB(skb).portid,
8918                                       nlh->nlmsg_seq);
8919         if (err < 0) {
8920                 kfree_skb(skb2);
8921                 goto err;
8922         }
8923
8924         nfnetlink_send(skb2, net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
8925                        nlmsg_report(nlh), GFP_KERNEL);
8926         return;
8927 err:
8928         nfnetlink_set_err(net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
8929                           -ENOBUFS);
8930 }
8931
8932 static int nf_tables_getgen(struct sk_buff *skb, const struct nfnl_info *info,
8933                             const struct nlattr * const nla[])
8934 {
8935         struct sk_buff *skb2;
8936         int err;
8937
8938         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
8939         if (skb2 == NULL)
8940                 return -ENOMEM;
8941
8942         err = nf_tables_fill_gen_info(skb2, info->net, NETLINK_CB(skb).portid,
8943                                       info->nlh->nlmsg_seq);
8944         if (err < 0)
8945                 goto err_fill_gen_info;
8946
8947         return nfnetlink_unicast(skb2, info->net, NETLINK_CB(skb).portid);
8948
8949 err_fill_gen_info:
8950         kfree_skb(skb2);
8951         return err;
8952 }
8953
8954 static const struct nfnl_callback nf_tables_cb[NFT_MSG_MAX] = {
8955         [NFT_MSG_NEWTABLE] = {
8956                 .call           = nf_tables_newtable,
8957                 .type           = NFNL_CB_BATCH,
8958                 .attr_count     = NFTA_TABLE_MAX,
8959                 .policy         = nft_table_policy,
8960         },
8961         [NFT_MSG_GETTABLE] = {
8962                 .call           = nf_tables_gettable,
8963                 .type           = NFNL_CB_RCU,
8964                 .attr_count     = NFTA_TABLE_MAX,
8965                 .policy         = nft_table_policy,
8966         },
8967         [NFT_MSG_DELTABLE] = {
8968                 .call           = nf_tables_deltable,
8969                 .type           = NFNL_CB_BATCH,
8970                 .attr_count     = NFTA_TABLE_MAX,
8971                 .policy         = nft_table_policy,
8972         },
8973         [NFT_MSG_DESTROYTABLE] = {
8974                 .call           = nf_tables_deltable,
8975                 .type           = NFNL_CB_BATCH,
8976                 .attr_count     = NFTA_TABLE_MAX,
8977                 .policy         = nft_table_policy,
8978         },
8979         [NFT_MSG_NEWCHAIN] = {
8980                 .call           = nf_tables_newchain,
8981                 .type           = NFNL_CB_BATCH,
8982                 .attr_count     = NFTA_CHAIN_MAX,
8983                 .policy         = nft_chain_policy,
8984         },
8985         [NFT_MSG_GETCHAIN] = {
8986                 .call           = nf_tables_getchain,
8987                 .type           = NFNL_CB_RCU,
8988                 .attr_count     = NFTA_CHAIN_MAX,
8989                 .policy         = nft_chain_policy,
8990         },
8991         [NFT_MSG_DELCHAIN] = {
8992                 .call           = nf_tables_delchain,
8993                 .type           = NFNL_CB_BATCH,
8994                 .attr_count     = NFTA_CHAIN_MAX,
8995                 .policy         = nft_chain_policy,
8996         },
8997         [NFT_MSG_DESTROYCHAIN] = {
8998                 .call           = nf_tables_delchain,
8999                 .type           = NFNL_CB_BATCH,
9000                 .attr_count     = NFTA_CHAIN_MAX,
9001                 .policy         = nft_chain_policy,
9002         },
9003         [NFT_MSG_NEWRULE] = {
9004                 .call           = nf_tables_newrule,
9005                 .type           = NFNL_CB_BATCH,
9006                 .attr_count     = NFTA_RULE_MAX,
9007                 .policy         = nft_rule_policy,
9008         },
9009         [NFT_MSG_GETRULE] = {
9010                 .call           = nf_tables_getrule,
9011                 .type           = NFNL_CB_RCU,
9012                 .attr_count     = NFTA_RULE_MAX,
9013                 .policy         = nft_rule_policy,
9014         },
9015         [NFT_MSG_GETRULE_RESET] = {
9016                 .call           = nf_tables_getrule,
9017                 .type           = NFNL_CB_RCU,
9018                 .attr_count     = NFTA_RULE_MAX,
9019                 .policy         = nft_rule_policy,
9020         },
9021         [NFT_MSG_DELRULE] = {
9022                 .call           = nf_tables_delrule,
9023                 .type           = NFNL_CB_BATCH,
9024                 .attr_count     = NFTA_RULE_MAX,
9025                 .policy         = nft_rule_policy,
9026         },
9027         [NFT_MSG_DESTROYRULE] = {
9028                 .call           = nf_tables_delrule,
9029                 .type           = NFNL_CB_BATCH,
9030                 .attr_count     = NFTA_RULE_MAX,
9031                 .policy         = nft_rule_policy,
9032         },
9033         [NFT_MSG_NEWSET] = {
9034                 .call           = nf_tables_newset,
9035                 .type           = NFNL_CB_BATCH,
9036                 .attr_count     = NFTA_SET_MAX,
9037                 .policy         = nft_set_policy,
9038         },
9039         [NFT_MSG_GETSET] = {
9040                 .call           = nf_tables_getset,
9041                 .type           = NFNL_CB_RCU,
9042                 .attr_count     = NFTA_SET_MAX,
9043                 .policy         = nft_set_policy,
9044         },
9045         [NFT_MSG_DELSET] = {
9046                 .call           = nf_tables_delset,
9047                 .type           = NFNL_CB_BATCH,
9048                 .attr_count     = NFTA_SET_MAX,
9049                 .policy         = nft_set_policy,
9050         },
9051         [NFT_MSG_DESTROYSET] = {
9052                 .call           = nf_tables_delset,
9053                 .type           = NFNL_CB_BATCH,
9054                 .attr_count     = NFTA_SET_MAX,
9055                 .policy         = nft_set_policy,
9056         },
9057         [NFT_MSG_NEWSETELEM] = {
9058                 .call           = nf_tables_newsetelem,
9059                 .type           = NFNL_CB_BATCH,
9060                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9061                 .policy         = nft_set_elem_list_policy,
9062         },
9063         [NFT_MSG_GETSETELEM] = {
9064                 .call           = nf_tables_getsetelem,
9065                 .type           = NFNL_CB_RCU,
9066                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9067                 .policy         = nft_set_elem_list_policy,
9068         },
9069         [NFT_MSG_GETSETELEM_RESET] = {
9070                 .call           = nf_tables_getsetelem,
9071                 .type           = NFNL_CB_RCU,
9072                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9073                 .policy         = nft_set_elem_list_policy,
9074         },
9075         [NFT_MSG_DELSETELEM] = {
9076                 .call           = nf_tables_delsetelem,
9077                 .type           = NFNL_CB_BATCH,
9078                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9079                 .policy         = nft_set_elem_list_policy,
9080         },
9081         [NFT_MSG_DESTROYSETELEM] = {
9082                 .call           = nf_tables_delsetelem,
9083                 .type           = NFNL_CB_BATCH,
9084                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9085                 .policy         = nft_set_elem_list_policy,
9086         },
9087         [NFT_MSG_GETGEN] = {
9088                 .call           = nf_tables_getgen,
9089                 .type           = NFNL_CB_RCU,
9090         },
9091         [NFT_MSG_NEWOBJ] = {
9092                 .call           = nf_tables_newobj,
9093                 .type           = NFNL_CB_BATCH,
9094                 .attr_count     = NFTA_OBJ_MAX,
9095                 .policy         = nft_obj_policy,
9096         },
9097         [NFT_MSG_GETOBJ] = {
9098                 .call           = nf_tables_getobj,
9099                 .type           = NFNL_CB_RCU,
9100                 .attr_count     = NFTA_OBJ_MAX,
9101                 .policy         = nft_obj_policy,
9102         },
9103         [NFT_MSG_DELOBJ] = {
9104                 .call           = nf_tables_delobj,
9105                 .type           = NFNL_CB_BATCH,
9106                 .attr_count     = NFTA_OBJ_MAX,
9107                 .policy         = nft_obj_policy,
9108         },
9109         [NFT_MSG_DESTROYOBJ] = {
9110                 .call           = nf_tables_delobj,
9111                 .type           = NFNL_CB_BATCH,
9112                 .attr_count     = NFTA_OBJ_MAX,
9113                 .policy         = nft_obj_policy,
9114         },
9115         [NFT_MSG_GETOBJ_RESET] = {
9116                 .call           = nf_tables_getobj,
9117                 .type           = NFNL_CB_RCU,
9118                 .attr_count     = NFTA_OBJ_MAX,
9119                 .policy         = nft_obj_policy,
9120         },
9121         [NFT_MSG_NEWFLOWTABLE] = {
9122                 .call           = nf_tables_newflowtable,
9123                 .type           = NFNL_CB_BATCH,
9124                 .attr_count     = NFTA_FLOWTABLE_MAX,
9125                 .policy         = nft_flowtable_policy,
9126         },
9127         [NFT_MSG_GETFLOWTABLE] = {
9128                 .call           = nf_tables_getflowtable,
9129                 .type           = NFNL_CB_RCU,
9130                 .attr_count     = NFTA_FLOWTABLE_MAX,
9131                 .policy         = nft_flowtable_policy,
9132         },
9133         [NFT_MSG_DELFLOWTABLE] = {
9134                 .call           = nf_tables_delflowtable,
9135                 .type           = NFNL_CB_BATCH,
9136                 .attr_count     = NFTA_FLOWTABLE_MAX,
9137                 .policy         = nft_flowtable_policy,
9138         },
9139         [NFT_MSG_DESTROYFLOWTABLE] = {
9140                 .call           = nf_tables_delflowtable,
9141                 .type           = NFNL_CB_BATCH,
9142                 .attr_count     = NFTA_FLOWTABLE_MAX,
9143                 .policy         = nft_flowtable_policy,
9144         },
9145 };
9146
9147 static int nf_tables_validate(struct net *net)
9148 {
9149         struct nftables_pernet *nft_net = nft_pernet(net);
9150         struct nft_table *table;
9151
9152         list_for_each_entry(table, &nft_net->tables, list) {
9153                 switch (table->validate_state) {
9154                 case NFT_VALIDATE_SKIP:
9155                         continue;
9156                 case NFT_VALIDATE_NEED:
9157                         nft_validate_state_update(table, NFT_VALIDATE_DO);
9158                         fallthrough;
9159                 case NFT_VALIDATE_DO:
9160                         if (nft_table_validate(net, table) < 0)
9161                                 return -EAGAIN;
9162
9163                         nft_validate_state_update(table, NFT_VALIDATE_SKIP);
9164                         break;
9165                 }
9166         }
9167
9168         return 0;
9169 }
9170
9171 /* a drop policy has to be deferred until all rules have been activated,
9172  * otherwise a large ruleset that contains a drop-policy base chain will
9173  * cause all packets to get dropped until the full transaction has been
9174  * processed.
9175  *
9176  * We defer the drop policy until the transaction has been finalized.
9177  */
9178 static void nft_chain_commit_drop_policy(struct nft_trans *trans)
9179 {
9180         struct nft_base_chain *basechain;
9181
9182         if (nft_trans_chain_policy(trans) != NF_DROP)
9183                 return;
9184
9185         if (!nft_is_base_chain(trans->ctx.chain))
9186                 return;
9187
9188         basechain = nft_base_chain(trans->ctx.chain);
9189         basechain->policy = NF_DROP;
9190 }
9191
9192 static void nft_chain_commit_update(struct nft_trans *trans)
9193 {
9194         struct nft_base_chain *basechain;
9195
9196         if (nft_trans_chain_name(trans)) {
9197                 rhltable_remove(&trans->ctx.table->chains_ht,
9198                                 &trans->ctx.chain->rhlhead,
9199                                 nft_chain_ht_params);
9200                 swap(trans->ctx.chain->name, nft_trans_chain_name(trans));
9201                 rhltable_insert_key(&trans->ctx.table->chains_ht,
9202                                     trans->ctx.chain->name,
9203                                     &trans->ctx.chain->rhlhead,
9204                                     nft_chain_ht_params);
9205         }
9206
9207         if (!nft_is_base_chain(trans->ctx.chain))
9208                 return;
9209
9210         nft_chain_stats_replace(trans);
9211
9212         basechain = nft_base_chain(trans->ctx.chain);
9213
9214         switch (nft_trans_chain_policy(trans)) {
9215         case NF_DROP:
9216         case NF_ACCEPT:
9217                 basechain->policy = nft_trans_chain_policy(trans);
9218                 break;
9219         }
9220 }
9221
9222 static void nft_obj_commit_update(struct nft_trans *trans)
9223 {
9224         struct nft_object *newobj;
9225         struct nft_object *obj;
9226
9227         obj = nft_trans_obj(trans);
9228         newobj = nft_trans_obj_newobj(trans);
9229
9230         if (obj->ops->update)
9231                 obj->ops->update(obj, newobj);
9232
9233         nft_obj_destroy(&trans->ctx, newobj);
9234 }
9235
9236 static void nft_commit_release(struct nft_trans *trans)
9237 {
9238         switch (trans->msg_type) {
9239         case NFT_MSG_DELTABLE:
9240         case NFT_MSG_DESTROYTABLE:
9241                 nf_tables_table_destroy(&trans->ctx);
9242                 break;
9243         case NFT_MSG_NEWCHAIN:
9244                 free_percpu(nft_trans_chain_stats(trans));
9245                 kfree(nft_trans_chain_name(trans));
9246                 break;
9247         case NFT_MSG_DELCHAIN:
9248         case NFT_MSG_DESTROYCHAIN:
9249                 if (nft_trans_chain_update(trans))
9250                         nft_hooks_destroy(&nft_trans_chain_hooks(trans));
9251                 else
9252                         nf_tables_chain_destroy(&trans->ctx);
9253                 break;
9254         case NFT_MSG_DELRULE:
9255         case NFT_MSG_DESTROYRULE:
9256                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
9257                 break;
9258         case NFT_MSG_DELSET:
9259         case NFT_MSG_DESTROYSET:
9260                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
9261                 break;
9262         case NFT_MSG_DELSETELEM:
9263         case NFT_MSG_DESTROYSETELEM:
9264                 nf_tables_set_elem_destroy(&trans->ctx,
9265                                            nft_trans_elem_set(trans),
9266                                            nft_trans_elem(trans).priv);
9267                 break;
9268         case NFT_MSG_DELOBJ:
9269         case NFT_MSG_DESTROYOBJ:
9270                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
9271                 break;
9272         case NFT_MSG_DELFLOWTABLE:
9273         case NFT_MSG_DESTROYFLOWTABLE:
9274                 if (nft_trans_flowtable_update(trans))
9275                         nft_hooks_destroy(&nft_trans_flowtable_hooks(trans));
9276                 else
9277                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
9278                 break;
9279         }
9280
9281         if (trans->put_net)
9282                 put_net(trans->ctx.net);
9283
9284         kfree(trans);
9285 }
9286
9287 static void nf_tables_trans_destroy_work(struct work_struct *w)
9288 {
9289         struct nft_trans *trans, *next;
9290         LIST_HEAD(head);
9291
9292         spin_lock(&nf_tables_destroy_list_lock);
9293         list_splice_init(&nf_tables_destroy_list, &head);
9294         spin_unlock(&nf_tables_destroy_list_lock);
9295
9296         if (list_empty(&head))
9297                 return;
9298
9299         synchronize_rcu();
9300
9301         list_for_each_entry_safe(trans, next, &head, list) {
9302                 nft_trans_list_del(trans);
9303                 nft_commit_release(trans);
9304         }
9305 }
9306
9307 void nf_tables_trans_destroy_flush_work(void)
9308 {
9309         flush_work(&trans_destroy_work);
9310 }
9311 EXPORT_SYMBOL_GPL(nf_tables_trans_destroy_flush_work);
9312
9313 static bool nft_expr_reduce(struct nft_regs_track *track,
9314                             const struct nft_expr *expr)
9315 {
9316         return false;
9317 }
9318
9319 static int nf_tables_commit_chain_prepare(struct net *net, struct nft_chain *chain)
9320 {
9321         const struct nft_expr *expr, *last;
9322         struct nft_regs_track track = {};
9323         unsigned int size, data_size;
9324         void *data, *data_boundary;
9325         struct nft_rule_dp *prule;
9326         struct nft_rule *rule;
9327
9328         /* already handled or inactive chain? */
9329         if (chain->blob_next || !nft_is_active_next(net, chain))
9330                 return 0;
9331
9332         data_size = 0;
9333         list_for_each_entry(rule, &chain->rules, list) {
9334                 if (nft_is_active_next(net, rule)) {
9335                         data_size += sizeof(*prule) + rule->dlen;
9336                         if (data_size > INT_MAX)
9337                                 return -ENOMEM;
9338                 }
9339         }
9340
9341         chain->blob_next = nf_tables_chain_alloc_rules(chain, data_size);
9342         if (!chain->blob_next)
9343                 return -ENOMEM;
9344
9345         data = (void *)chain->blob_next->data;
9346         data_boundary = data + data_size;
9347         size = 0;
9348
9349         list_for_each_entry(rule, &chain->rules, list) {
9350                 if (!nft_is_active_next(net, rule))
9351                         continue;
9352
9353                 prule = (struct nft_rule_dp *)data;
9354                 data += offsetof(struct nft_rule_dp, data);
9355                 if (WARN_ON_ONCE(data > data_boundary))
9356                         return -ENOMEM;
9357
9358                 size = 0;
9359                 track.last = nft_expr_last(rule);
9360                 nft_rule_for_each_expr(expr, last, rule) {
9361                         track.cur = expr;
9362
9363                         if (nft_expr_reduce(&track, expr)) {
9364                                 expr = track.cur;
9365                                 continue;
9366                         }
9367
9368                         if (WARN_ON_ONCE(data + size + expr->ops->size > data_boundary))
9369                                 return -ENOMEM;
9370
9371                         memcpy(data + size, expr, expr->ops->size);
9372                         size += expr->ops->size;
9373                 }
9374                 if (WARN_ON_ONCE(size >= 1 << 12))
9375                         return -ENOMEM;
9376
9377                 prule->handle = rule->handle;
9378                 prule->dlen = size;
9379                 prule->is_last = 0;
9380
9381                 data += size;
9382                 size = 0;
9383                 chain->blob_next->size += (unsigned long)(data - (void *)prule);
9384         }
9385
9386         if (WARN_ON_ONCE(data > data_boundary))
9387                 return -ENOMEM;
9388
9389         prule = (struct nft_rule_dp *)data;
9390         nft_last_rule(chain, prule);
9391
9392         return 0;
9393 }
9394
9395 static void nf_tables_commit_chain_prepare_cancel(struct net *net)
9396 {
9397         struct nftables_pernet *nft_net = nft_pernet(net);
9398         struct nft_trans *trans, *next;
9399
9400         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
9401                 struct nft_chain *chain = trans->ctx.chain;
9402
9403                 if (trans->msg_type == NFT_MSG_NEWRULE ||
9404                     trans->msg_type == NFT_MSG_DELRULE) {
9405                         kvfree(chain->blob_next);
9406                         chain->blob_next = NULL;
9407                 }
9408         }
9409 }
9410
9411 static void __nf_tables_commit_chain_free_rules(struct rcu_head *h)
9412 {
9413         struct nft_rule_dp_last *l = container_of(h, struct nft_rule_dp_last, h);
9414
9415         kvfree(l->blob);
9416 }
9417
9418 static void nf_tables_commit_chain_free_rules_old(struct nft_rule_blob *blob)
9419 {
9420         struct nft_rule_dp_last *last;
9421
9422         /* last rule trailer is after end marker */
9423         last = (void *)blob + sizeof(*blob) + blob->size;
9424         last->blob = blob;
9425
9426         call_rcu(&last->h, __nf_tables_commit_chain_free_rules);
9427 }
9428
9429 static void nf_tables_commit_chain(struct net *net, struct nft_chain *chain)
9430 {
9431         struct nft_rule_blob *g0, *g1;
9432         bool next_genbit;
9433
9434         next_genbit = nft_gencursor_next(net);
9435
9436         g0 = rcu_dereference_protected(chain->blob_gen_0,
9437                                        lockdep_commit_lock_is_held(net));
9438         g1 = rcu_dereference_protected(chain->blob_gen_1,
9439                                        lockdep_commit_lock_is_held(net));
9440
9441         /* No changes to this chain? */
9442         if (chain->blob_next == NULL) {
9443                 /* chain had no change in last or next generation */
9444                 if (g0 == g1)
9445                         return;
9446                 /*
9447                  * chain had no change in this generation; make sure next
9448                  * one uses same rules as current generation.
9449                  */
9450                 if (next_genbit) {
9451                         rcu_assign_pointer(chain->blob_gen_1, g0);
9452                         nf_tables_commit_chain_free_rules_old(g1);
9453                 } else {
9454                         rcu_assign_pointer(chain->blob_gen_0, g1);
9455                         nf_tables_commit_chain_free_rules_old(g0);
9456                 }
9457
9458                 return;
9459         }
9460
9461         if (next_genbit)
9462                 rcu_assign_pointer(chain->blob_gen_1, chain->blob_next);
9463         else
9464                 rcu_assign_pointer(chain->blob_gen_0, chain->blob_next);
9465
9466         chain->blob_next = NULL;
9467
9468         if (g0 == g1)
9469                 return;
9470
9471         if (next_genbit)
9472                 nf_tables_commit_chain_free_rules_old(g1);
9473         else
9474                 nf_tables_commit_chain_free_rules_old(g0);
9475 }
9476
9477 static void nft_obj_del(struct nft_object *obj)
9478 {
9479         rhltable_remove(&nft_objname_ht, &obj->rhlhead, nft_objname_ht_params);
9480         list_del_rcu(&obj->list);
9481 }
9482
9483 void nft_chain_del(struct nft_chain *chain)
9484 {
9485         struct nft_table *table = chain->table;
9486
9487         WARN_ON_ONCE(rhltable_remove(&table->chains_ht, &chain->rhlhead,
9488                                      nft_chain_ht_params));
9489         list_del_rcu(&chain->list);
9490 }
9491
9492 static void nft_trans_gc_setelem_remove(struct nft_ctx *ctx,
9493                                         struct nft_trans_gc *trans)
9494 {
9495         void **priv = trans->priv;
9496         unsigned int i;
9497
9498         for (i = 0; i < trans->count; i++) {
9499                 struct nft_set_elem elem = {
9500                         .priv = priv[i],
9501                 };
9502
9503                 nft_setelem_data_deactivate(ctx->net, trans->set, &elem);
9504                 nft_setelem_remove(ctx->net, trans->set, &elem);
9505         }
9506 }
9507
9508 void nft_trans_gc_destroy(struct nft_trans_gc *trans)
9509 {
9510         nft_set_put(trans->set);
9511         put_net(trans->net);
9512         kfree(trans);
9513 }
9514
9515 static void nft_trans_gc_trans_free(struct rcu_head *rcu)
9516 {
9517         struct nft_set_elem elem = {};
9518         struct nft_trans_gc *trans;
9519         struct nft_ctx ctx = {};
9520         unsigned int i;
9521
9522         trans = container_of(rcu, struct nft_trans_gc, rcu);
9523         ctx.net = read_pnet(&trans->set->net);
9524
9525         for (i = 0; i < trans->count; i++) {
9526                 elem.priv = trans->priv[i];
9527                 if (!nft_setelem_is_catchall(trans->set, &elem))
9528                         atomic_dec(&trans->set->nelems);
9529
9530                 nf_tables_set_elem_destroy(&ctx, trans->set, elem.priv);
9531         }
9532
9533         nft_trans_gc_destroy(trans);
9534 }
9535
9536 static bool nft_trans_gc_work_done(struct nft_trans_gc *trans)
9537 {
9538         struct nftables_pernet *nft_net;
9539         struct nft_ctx ctx = {};
9540
9541         nft_net = nft_pernet(trans->net);
9542
9543         mutex_lock(&nft_net->commit_mutex);
9544
9545         /* Check for race with transaction, otherwise this batch refers to
9546          * stale objects that might not be there anymore. Skip transaction if
9547          * set has been destroyed from control plane transaction in case gc
9548          * worker loses race.
9549          */
9550         if (READ_ONCE(nft_net->gc_seq) != trans->seq || trans->set->dead) {
9551                 mutex_unlock(&nft_net->commit_mutex);
9552                 return false;
9553         }
9554
9555         ctx.net = trans->net;
9556         ctx.table = trans->set->table;
9557
9558         nft_trans_gc_setelem_remove(&ctx, trans);
9559         mutex_unlock(&nft_net->commit_mutex);
9560
9561         return true;
9562 }
9563
9564 static void nft_trans_gc_work(struct work_struct *work)
9565 {
9566         struct nft_trans_gc *trans, *next;
9567         LIST_HEAD(trans_gc_list);
9568
9569         spin_lock(&nf_tables_gc_list_lock);
9570         list_splice_init(&nf_tables_gc_list, &trans_gc_list);
9571         spin_unlock(&nf_tables_gc_list_lock);
9572
9573         list_for_each_entry_safe(trans, next, &trans_gc_list, list) {
9574                 list_del(&trans->list);
9575                 if (!nft_trans_gc_work_done(trans)) {
9576                         nft_trans_gc_destroy(trans);
9577                         continue;
9578                 }
9579                 call_rcu(&trans->rcu, nft_trans_gc_trans_free);
9580         }
9581 }
9582
9583 struct nft_trans_gc *nft_trans_gc_alloc(struct nft_set *set,
9584                                         unsigned int gc_seq, gfp_t gfp)
9585 {
9586         struct net *net = read_pnet(&set->net);
9587         struct nft_trans_gc *trans;
9588
9589         trans = kzalloc(sizeof(*trans), gfp);
9590         if (!trans)
9591                 return NULL;
9592
9593         trans->net = maybe_get_net(net);
9594         if (!trans->net) {
9595                 kfree(trans);
9596                 return NULL;
9597         }
9598
9599         refcount_inc(&set->refs);
9600         trans->set = set;
9601         trans->seq = gc_seq;
9602
9603         return trans;
9604 }
9605
9606 void nft_trans_gc_elem_add(struct nft_trans_gc *trans, void *priv)
9607 {
9608         trans->priv[trans->count++] = priv;
9609 }
9610
9611 static void nft_trans_gc_queue_work(struct nft_trans_gc *trans)
9612 {
9613         spin_lock(&nf_tables_gc_list_lock);
9614         list_add_tail(&trans->list, &nf_tables_gc_list);
9615         spin_unlock(&nf_tables_gc_list_lock);
9616
9617         schedule_work(&trans_gc_work);
9618 }
9619
9620 static int nft_trans_gc_space(struct nft_trans_gc *trans)
9621 {
9622         return NFT_TRANS_GC_BATCHCOUNT - trans->count;
9623 }
9624
9625 struct nft_trans_gc *nft_trans_gc_queue_async(struct nft_trans_gc *gc,
9626                                               unsigned int gc_seq, gfp_t gfp)
9627 {
9628         struct nft_set *set;
9629
9630         if (nft_trans_gc_space(gc))
9631                 return gc;
9632
9633         set = gc->set;
9634         nft_trans_gc_queue_work(gc);
9635
9636         return nft_trans_gc_alloc(set, gc_seq, gfp);
9637 }
9638
9639 void nft_trans_gc_queue_async_done(struct nft_trans_gc *trans)
9640 {
9641         if (trans->count == 0) {
9642                 nft_trans_gc_destroy(trans);
9643                 return;
9644         }
9645
9646         nft_trans_gc_queue_work(trans);
9647 }
9648
9649 struct nft_trans_gc *nft_trans_gc_queue_sync(struct nft_trans_gc *gc, gfp_t gfp)
9650 {
9651         struct nft_set *set;
9652
9653         if (WARN_ON_ONCE(!lockdep_commit_lock_is_held(gc->net)))
9654                 return NULL;
9655
9656         if (nft_trans_gc_space(gc))
9657                 return gc;
9658
9659         set = gc->set;
9660         call_rcu(&gc->rcu, nft_trans_gc_trans_free);
9661
9662         return nft_trans_gc_alloc(set, 0, gfp);
9663 }
9664
9665 void nft_trans_gc_queue_sync_done(struct nft_trans_gc *trans)
9666 {
9667         WARN_ON_ONCE(!lockdep_commit_lock_is_held(trans->net));
9668
9669         if (trans->count == 0) {
9670                 nft_trans_gc_destroy(trans);
9671                 return;
9672         }
9673
9674         call_rcu(&trans->rcu, nft_trans_gc_trans_free);
9675 }
9676
9677 struct nft_trans_gc *nft_trans_gc_catchall_async(struct nft_trans_gc *gc,
9678                                                  unsigned int gc_seq)
9679 {
9680         struct nft_set_elem_catchall *catchall;
9681         const struct nft_set *set = gc->set;
9682         struct nft_set_ext *ext;
9683
9684         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
9685                 ext = nft_set_elem_ext(set, catchall->elem);
9686
9687                 if (!nft_set_elem_expired(ext))
9688                         continue;
9689                 if (nft_set_elem_is_dead(ext))
9690                         goto dead_elem;
9691
9692                 nft_set_elem_dead(ext);
9693 dead_elem:
9694                 gc = nft_trans_gc_queue_async(gc, gc_seq, GFP_ATOMIC);
9695                 if (!gc)
9696                         return NULL;
9697
9698                 nft_trans_gc_elem_add(gc, catchall->elem);
9699         }
9700
9701         return gc;
9702 }
9703
9704 struct nft_trans_gc *nft_trans_gc_catchall_sync(struct nft_trans_gc *gc)
9705 {
9706         struct nft_set_elem_catchall *catchall, *next;
9707         const struct nft_set *set = gc->set;
9708         struct nft_set_elem elem;
9709         struct nft_set_ext *ext;
9710
9711         WARN_ON_ONCE(!lockdep_commit_lock_is_held(gc->net));
9712
9713         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
9714                 ext = nft_set_elem_ext(set, catchall->elem);
9715
9716                 if (!nft_set_elem_expired(ext))
9717                         continue;
9718
9719                 gc = nft_trans_gc_queue_sync(gc, GFP_KERNEL);
9720                 if (!gc)
9721                         return NULL;
9722
9723                 memset(&elem, 0, sizeof(elem));
9724                 elem.priv = catchall->elem;
9725
9726                 nft_setelem_data_deactivate(gc->net, gc->set, &elem);
9727                 nft_setelem_catchall_destroy(catchall);
9728                 nft_trans_gc_elem_add(gc, elem.priv);
9729         }
9730
9731         return gc;
9732 }
9733
9734 static void nf_tables_module_autoload_cleanup(struct net *net)
9735 {
9736         struct nftables_pernet *nft_net = nft_pernet(net);
9737         struct nft_module_request *req, *next;
9738
9739         WARN_ON_ONCE(!list_empty(&nft_net->commit_list));
9740         list_for_each_entry_safe(req, next, &nft_net->module_list, list) {
9741                 WARN_ON_ONCE(!req->done);
9742                 list_del(&req->list);
9743                 kfree(req);
9744         }
9745 }
9746
9747 static void nf_tables_commit_release(struct net *net)
9748 {
9749         struct nftables_pernet *nft_net = nft_pernet(net);
9750         struct nft_trans *trans;
9751
9752         /* all side effects have to be made visible.
9753          * For example, if a chain named 'foo' has been deleted, a
9754          * new transaction must not find it anymore.
9755          *
9756          * Memory reclaim happens asynchronously from work queue
9757          * to prevent expensive synchronize_rcu() in commit phase.
9758          */
9759         if (list_empty(&nft_net->commit_list)) {
9760                 nf_tables_module_autoload_cleanup(net);
9761                 mutex_unlock(&nft_net->commit_mutex);
9762                 return;
9763         }
9764
9765         trans = list_last_entry(&nft_net->commit_list,
9766                                 struct nft_trans, list);
9767         get_net(trans->ctx.net);
9768         WARN_ON_ONCE(trans->put_net);
9769
9770         trans->put_net = true;
9771         spin_lock(&nf_tables_destroy_list_lock);
9772         list_splice_tail_init(&nft_net->commit_list, &nf_tables_destroy_list);
9773         spin_unlock(&nf_tables_destroy_list_lock);
9774
9775         nf_tables_module_autoload_cleanup(net);
9776         schedule_work(&trans_destroy_work);
9777
9778         mutex_unlock(&nft_net->commit_mutex);
9779 }
9780
9781 static void nft_commit_notify(struct net *net, u32 portid)
9782 {
9783         struct nftables_pernet *nft_net = nft_pernet(net);
9784         struct sk_buff *batch_skb = NULL, *nskb, *skb;
9785         unsigned char *data;
9786         int len;
9787
9788         list_for_each_entry_safe(skb, nskb, &nft_net->notify_list, list) {
9789                 if (!batch_skb) {
9790 new_batch:
9791                         batch_skb = skb;
9792                         len = NLMSG_GOODSIZE - skb->len;
9793                         list_del(&skb->list);
9794                         continue;
9795                 }
9796                 len -= skb->len;
9797                 if (len > 0 && NFT_CB(skb).report == NFT_CB(batch_skb).report) {
9798                         data = skb_put(batch_skb, skb->len);
9799                         memcpy(data, skb->data, skb->len);
9800                         list_del(&skb->list);
9801                         kfree_skb(skb);
9802                         continue;
9803                 }
9804                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
9805                                NFT_CB(batch_skb).report, GFP_KERNEL);
9806                 goto new_batch;
9807         }
9808
9809         if (batch_skb) {
9810                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
9811                                NFT_CB(batch_skb).report, GFP_KERNEL);
9812         }
9813
9814         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
9815 }
9816
9817 static int nf_tables_commit_audit_alloc(struct list_head *adl,
9818                                         struct nft_table *table)
9819 {
9820         struct nft_audit_data *adp;
9821
9822         list_for_each_entry(adp, adl, list) {
9823                 if (adp->table == table)
9824                         return 0;
9825         }
9826         adp = kzalloc(sizeof(*adp), GFP_KERNEL);
9827         if (!adp)
9828                 return -ENOMEM;
9829         adp->table = table;
9830         list_add(&adp->list, adl);
9831         return 0;
9832 }
9833
9834 static void nf_tables_commit_audit_free(struct list_head *adl)
9835 {
9836         struct nft_audit_data *adp, *adn;
9837
9838         list_for_each_entry_safe(adp, adn, adl, list) {
9839                 list_del(&adp->list);
9840                 kfree(adp);
9841         }
9842 }
9843
9844 static void nf_tables_commit_audit_collect(struct list_head *adl,
9845                                            struct nft_table *table, u32 op)
9846 {
9847         struct nft_audit_data *adp;
9848
9849         list_for_each_entry(adp, adl, list) {
9850                 if (adp->table == table)
9851                         goto found;
9852         }
9853         WARN_ONCE(1, "table=%s not expected in commit list", table->name);
9854         return;
9855 found:
9856         adp->entries++;
9857         if (!adp->op || adp->op > op)
9858                 adp->op = op;
9859 }
9860
9861 #define AUNFTABLENAMELEN (NFT_TABLE_MAXNAMELEN + 22)
9862
9863 static void nf_tables_commit_audit_log(struct list_head *adl, u32 generation)
9864 {
9865         struct nft_audit_data *adp, *adn;
9866         char aubuf[AUNFTABLENAMELEN];
9867
9868         list_for_each_entry_safe(adp, adn, adl, list) {
9869                 snprintf(aubuf, AUNFTABLENAMELEN, "%s:%u", adp->table->name,
9870                          generation);
9871                 audit_log_nfcfg(aubuf, adp->table->family, adp->entries,
9872                                 nft2audit_op[adp->op], GFP_KERNEL);
9873                 list_del(&adp->list);
9874                 kfree(adp);
9875         }
9876 }
9877
9878 static void nft_set_commit_update(struct list_head *set_update_list)
9879 {
9880         struct nft_set *set, *next;
9881
9882         list_for_each_entry_safe(set, next, set_update_list, pending_update) {
9883                 list_del_init(&set->pending_update);
9884
9885                 if (!set->ops->commit || set->dead)
9886                         continue;
9887
9888                 set->ops->commit(set);
9889         }
9890 }
9891
9892 static unsigned int nft_gc_seq_begin(struct nftables_pernet *nft_net)
9893 {
9894         unsigned int gc_seq;
9895
9896         /* Bump gc counter, it becomes odd, this is the busy mark. */
9897         gc_seq = READ_ONCE(nft_net->gc_seq);
9898         WRITE_ONCE(nft_net->gc_seq, ++gc_seq);
9899
9900         return gc_seq;
9901 }
9902
9903 static void nft_gc_seq_end(struct nftables_pernet *nft_net, unsigned int gc_seq)
9904 {
9905         WRITE_ONCE(nft_net->gc_seq, ++gc_seq);
9906 }
9907
9908 static int nf_tables_commit(struct net *net, struct sk_buff *skb)
9909 {
9910         struct nftables_pernet *nft_net = nft_pernet(net);
9911         struct nft_trans *trans, *next;
9912         unsigned int base_seq, gc_seq;
9913         LIST_HEAD(set_update_list);
9914         struct nft_trans_elem *te;
9915         struct nft_chain *chain;
9916         struct nft_table *table;
9917         LIST_HEAD(adl);
9918         int err;
9919
9920         if (list_empty(&nft_net->commit_list)) {
9921                 mutex_unlock(&nft_net->commit_mutex);
9922                 return 0;
9923         }
9924
9925         list_for_each_entry(trans, &nft_net->binding_list, binding_list) {
9926                 switch (trans->msg_type) {
9927                 case NFT_MSG_NEWSET:
9928                         if (!nft_trans_set_update(trans) &&
9929                             nft_set_is_anonymous(nft_trans_set(trans)) &&
9930                             !nft_trans_set_bound(trans)) {
9931                                 pr_warn_once("nftables ruleset with unbound set\n");
9932                                 return -EINVAL;
9933                         }
9934                         break;
9935                 case NFT_MSG_NEWCHAIN:
9936                         if (!nft_trans_chain_update(trans) &&
9937                             nft_chain_binding(nft_trans_chain(trans)) &&
9938                             !nft_trans_chain_bound(trans)) {
9939                                 pr_warn_once("nftables ruleset with unbound chain\n");
9940                                 return -EINVAL;
9941                         }
9942                         break;
9943                 }
9944         }
9945
9946         /* 0. Validate ruleset, otherwise roll back for error reporting. */
9947         if (nf_tables_validate(net) < 0) {
9948                 nft_net->validate_state = NFT_VALIDATE_DO;
9949                 return -EAGAIN;
9950         }
9951
9952         err = nft_flow_rule_offload_commit(net);
9953         if (err < 0)
9954                 return err;
9955
9956         /* 1.  Allocate space for next generation rules_gen_X[] */
9957         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
9958                 int ret;
9959
9960                 ret = nf_tables_commit_audit_alloc(&adl, trans->ctx.table);
9961                 if (ret) {
9962                         nf_tables_commit_chain_prepare_cancel(net);
9963                         nf_tables_commit_audit_free(&adl);
9964                         return ret;
9965                 }
9966                 if (trans->msg_type == NFT_MSG_NEWRULE ||
9967                     trans->msg_type == NFT_MSG_DELRULE) {
9968                         chain = trans->ctx.chain;
9969
9970                         ret = nf_tables_commit_chain_prepare(net, chain);
9971                         if (ret < 0) {
9972                                 nf_tables_commit_chain_prepare_cancel(net);
9973                                 nf_tables_commit_audit_free(&adl);
9974                                 return ret;
9975                         }
9976                 }
9977         }
9978
9979         /* step 2.  Make rules_gen_X visible to packet path */
9980         list_for_each_entry(table, &nft_net->tables, list) {
9981                 list_for_each_entry(chain, &table->chains, list)
9982                         nf_tables_commit_chain(net, chain);
9983         }
9984
9985         /*
9986          * Bump generation counter, invalidate any dump in progress.
9987          * Cannot fail after this point.
9988          */
9989         base_seq = READ_ONCE(nft_net->base_seq);
9990         while (++base_seq == 0)
9991                 ;
9992
9993         WRITE_ONCE(nft_net->base_seq, base_seq);
9994
9995         gc_seq = nft_gc_seq_begin(nft_net);
9996
9997         /* step 3. Start new generation, rules_gen_X now in use. */
9998         net->nft.gencursor = nft_gencursor_next(net);
9999
10000         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
10001                 nf_tables_commit_audit_collect(&adl, trans->ctx.table,
10002                                                trans->msg_type);
10003                 switch (trans->msg_type) {
10004                 case NFT_MSG_NEWTABLE:
10005                         if (nft_trans_table_update(trans)) {
10006                                 if (!(trans->ctx.table->flags & __NFT_TABLE_F_UPDATE)) {
10007                                         nft_trans_destroy(trans);
10008                                         break;
10009                                 }
10010                                 if (trans->ctx.table->flags & NFT_TABLE_F_DORMANT)
10011                                         nf_tables_table_disable(net, trans->ctx.table);
10012
10013                                 trans->ctx.table->flags &= ~__NFT_TABLE_F_UPDATE;
10014                         } else {
10015                                 nft_clear(net, trans->ctx.table);
10016                         }
10017                         nf_tables_table_notify(&trans->ctx, NFT_MSG_NEWTABLE);
10018                         nft_trans_destroy(trans);
10019                         break;
10020                 case NFT_MSG_DELTABLE:
10021                 case NFT_MSG_DESTROYTABLE:
10022                         list_del_rcu(&trans->ctx.table->list);
10023                         nf_tables_table_notify(&trans->ctx, trans->msg_type);
10024                         break;
10025                 case NFT_MSG_NEWCHAIN:
10026                         if (nft_trans_chain_update(trans)) {
10027                                 nft_chain_commit_update(trans);
10028                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN,
10029                                                        &nft_trans_chain_hooks(trans));
10030                                 list_splice(&nft_trans_chain_hooks(trans),
10031                                             &nft_trans_basechain(trans)->hook_list);
10032                                 /* trans destroyed after rcu grace period */
10033                         } else {
10034                                 nft_chain_commit_drop_policy(trans);
10035                                 nft_clear(net, trans->ctx.chain);
10036                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN, NULL);
10037                                 nft_trans_destroy(trans);
10038                         }
10039                         break;
10040                 case NFT_MSG_DELCHAIN:
10041                 case NFT_MSG_DESTROYCHAIN:
10042                         if (nft_trans_chain_update(trans)) {
10043                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_DELCHAIN,
10044                                                        &nft_trans_chain_hooks(trans));
10045                                 nft_netdev_unregister_hooks(net,
10046                                                             &nft_trans_chain_hooks(trans),
10047                                                             true);
10048                         } else {
10049                                 nft_chain_del(trans->ctx.chain);
10050                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_DELCHAIN,
10051                                                        NULL);
10052                                 nf_tables_unregister_hook(trans->ctx.net,
10053                                                           trans->ctx.table,
10054                                                           trans->ctx.chain);
10055                         }
10056                         break;
10057                 case NFT_MSG_NEWRULE:
10058                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
10059                         nf_tables_rule_notify(&trans->ctx,
10060                                               nft_trans_rule(trans),
10061                                               NFT_MSG_NEWRULE);
10062                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
10063                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10064
10065                         nft_trans_destroy(trans);
10066                         break;
10067                 case NFT_MSG_DELRULE:
10068                 case NFT_MSG_DESTROYRULE:
10069                         list_del_rcu(&nft_trans_rule(trans)->list);
10070                         nf_tables_rule_notify(&trans->ctx,
10071                                               nft_trans_rule(trans),
10072                                               trans->msg_type);
10073                         nft_rule_expr_deactivate(&trans->ctx,
10074                                                  nft_trans_rule(trans),
10075                                                  NFT_TRANS_COMMIT);
10076
10077                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
10078                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10079                         break;
10080                 case NFT_MSG_NEWSET:
10081                         if (nft_trans_set_update(trans)) {
10082                                 struct nft_set *set = nft_trans_set(trans);
10083
10084                                 WRITE_ONCE(set->timeout, nft_trans_set_timeout(trans));
10085                                 WRITE_ONCE(set->gc_int, nft_trans_set_gc_int(trans));
10086
10087                                 if (nft_trans_set_size(trans))
10088                                         WRITE_ONCE(set->size, nft_trans_set_size(trans));
10089                         } else {
10090                                 nft_clear(net, nft_trans_set(trans));
10091                                 /* This avoids hitting -EBUSY when deleting the table
10092                                  * from the transaction.
10093                                  */
10094                                 if (nft_set_is_anonymous(nft_trans_set(trans)) &&
10095                                     !list_empty(&nft_trans_set(trans)->bindings))
10096                                         nft_use_dec(&trans->ctx.table->use);
10097                         }
10098                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
10099                                              NFT_MSG_NEWSET, GFP_KERNEL);
10100                         nft_trans_destroy(trans);
10101                         break;
10102                 case NFT_MSG_DELSET:
10103                 case NFT_MSG_DESTROYSET:
10104                         nft_trans_set(trans)->dead = 1;
10105                         list_del_rcu(&nft_trans_set(trans)->list);
10106                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
10107                                              trans->msg_type, GFP_KERNEL);
10108                         break;
10109                 case NFT_MSG_NEWSETELEM:
10110                         te = (struct nft_trans_elem *)trans->data;
10111
10112                         nft_setelem_activate(net, te->set, &te->elem);
10113                         nf_tables_setelem_notify(&trans->ctx, te->set,
10114                                                  &te->elem,
10115                                                  NFT_MSG_NEWSETELEM);
10116                         if (te->set->ops->commit &&
10117                             list_empty(&te->set->pending_update)) {
10118                                 list_add_tail(&te->set->pending_update,
10119                                               &set_update_list);
10120                         }
10121                         nft_trans_destroy(trans);
10122                         break;
10123                 case NFT_MSG_DELSETELEM:
10124                 case NFT_MSG_DESTROYSETELEM:
10125                         te = (struct nft_trans_elem *)trans->data;
10126
10127                         nf_tables_setelem_notify(&trans->ctx, te->set,
10128                                                  &te->elem,
10129                                                  trans->msg_type);
10130                         nft_setelem_remove(net, te->set, &te->elem);
10131                         if (!nft_setelem_is_catchall(te->set, &te->elem)) {
10132                                 atomic_dec(&te->set->nelems);
10133                                 te->set->ndeact--;
10134                         }
10135                         if (te->set->ops->commit &&
10136                             list_empty(&te->set->pending_update)) {
10137                                 list_add_tail(&te->set->pending_update,
10138                                               &set_update_list);
10139                         }
10140                         break;
10141                 case NFT_MSG_NEWOBJ:
10142                         if (nft_trans_obj_update(trans)) {
10143                                 nft_obj_commit_update(trans);
10144                                 nf_tables_obj_notify(&trans->ctx,
10145                                                      nft_trans_obj(trans),
10146                                                      NFT_MSG_NEWOBJ);
10147                         } else {
10148                                 nft_clear(net, nft_trans_obj(trans));
10149                                 nf_tables_obj_notify(&trans->ctx,
10150                                                      nft_trans_obj(trans),
10151                                                      NFT_MSG_NEWOBJ);
10152                                 nft_trans_destroy(trans);
10153                         }
10154                         break;
10155                 case NFT_MSG_DELOBJ:
10156                 case NFT_MSG_DESTROYOBJ:
10157                         nft_obj_del(nft_trans_obj(trans));
10158                         nf_tables_obj_notify(&trans->ctx, nft_trans_obj(trans),
10159                                              trans->msg_type);
10160                         break;
10161                 case NFT_MSG_NEWFLOWTABLE:
10162                         if (nft_trans_flowtable_update(trans)) {
10163                                 nft_trans_flowtable(trans)->data.flags =
10164                                         nft_trans_flowtable_flags(trans);
10165                                 nf_tables_flowtable_notify(&trans->ctx,
10166                                                            nft_trans_flowtable(trans),
10167                                                            &nft_trans_flowtable_hooks(trans),
10168                                                            NFT_MSG_NEWFLOWTABLE);
10169                                 list_splice(&nft_trans_flowtable_hooks(trans),
10170                                             &nft_trans_flowtable(trans)->hook_list);
10171                         } else {
10172                                 nft_clear(net, nft_trans_flowtable(trans));
10173                                 nf_tables_flowtable_notify(&trans->ctx,
10174                                                            nft_trans_flowtable(trans),
10175                                                            NULL,
10176                                                            NFT_MSG_NEWFLOWTABLE);
10177                         }
10178                         nft_trans_destroy(trans);
10179                         break;
10180                 case NFT_MSG_DELFLOWTABLE:
10181                 case NFT_MSG_DESTROYFLOWTABLE:
10182                         if (nft_trans_flowtable_update(trans)) {
10183                                 nf_tables_flowtable_notify(&trans->ctx,
10184                                                            nft_trans_flowtable(trans),
10185                                                            &nft_trans_flowtable_hooks(trans),
10186                                                            trans->msg_type);
10187                                 nft_unregister_flowtable_net_hooks(net,
10188                                                                    &nft_trans_flowtable_hooks(trans));
10189                         } else {
10190                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
10191                                 nf_tables_flowtable_notify(&trans->ctx,
10192                                                            nft_trans_flowtable(trans),
10193                                                            NULL,
10194                                                            trans->msg_type);
10195                                 nft_unregister_flowtable_net_hooks(net,
10196                                                 &nft_trans_flowtable(trans)->hook_list);
10197                         }
10198                         break;
10199                 }
10200         }
10201
10202         nft_set_commit_update(&set_update_list);
10203
10204         nft_commit_notify(net, NETLINK_CB(skb).portid);
10205         nf_tables_gen_notify(net, skb, NFT_MSG_NEWGEN);
10206         nf_tables_commit_audit_log(&adl, nft_net->base_seq);
10207
10208         nft_gc_seq_end(nft_net, gc_seq);
10209         nft_net->validate_state = NFT_VALIDATE_SKIP;
10210         nf_tables_commit_release(net);
10211
10212         return 0;
10213 }
10214
10215 static void nf_tables_module_autoload(struct net *net)
10216 {
10217         struct nftables_pernet *nft_net = nft_pernet(net);
10218         struct nft_module_request *req, *next;
10219         LIST_HEAD(module_list);
10220
10221         list_splice_init(&nft_net->module_list, &module_list);
10222         mutex_unlock(&nft_net->commit_mutex);
10223         list_for_each_entry_safe(req, next, &module_list, list) {
10224                 request_module("%s", req->module);
10225                 req->done = true;
10226         }
10227         mutex_lock(&nft_net->commit_mutex);
10228         list_splice(&module_list, &nft_net->module_list);
10229 }
10230
10231 static void nf_tables_abort_release(struct nft_trans *trans)
10232 {
10233         switch (trans->msg_type) {
10234         case NFT_MSG_NEWTABLE:
10235                 nf_tables_table_destroy(&trans->ctx);
10236                 break;
10237         case NFT_MSG_NEWCHAIN:
10238                 if (nft_trans_chain_update(trans))
10239                         nft_hooks_destroy(&nft_trans_chain_hooks(trans));
10240                 else
10241                         nf_tables_chain_destroy(&trans->ctx);
10242                 break;
10243         case NFT_MSG_NEWRULE:
10244                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
10245                 break;
10246         case NFT_MSG_NEWSET:
10247                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
10248                 break;
10249         case NFT_MSG_NEWSETELEM:
10250                 nft_set_elem_destroy(nft_trans_elem_set(trans),
10251                                      nft_trans_elem(trans).priv, true);
10252                 break;
10253         case NFT_MSG_NEWOBJ:
10254                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
10255                 break;
10256         case NFT_MSG_NEWFLOWTABLE:
10257                 if (nft_trans_flowtable_update(trans))
10258                         nft_hooks_destroy(&nft_trans_flowtable_hooks(trans));
10259                 else
10260                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
10261                 break;
10262         }
10263         kfree(trans);
10264 }
10265
10266 static void nft_set_abort_update(struct list_head *set_update_list)
10267 {
10268         struct nft_set *set, *next;
10269
10270         list_for_each_entry_safe(set, next, set_update_list, pending_update) {
10271                 list_del_init(&set->pending_update);
10272
10273                 if (!set->ops->abort)
10274                         continue;
10275
10276                 set->ops->abort(set);
10277         }
10278 }
10279
10280 static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action)
10281 {
10282         struct nftables_pernet *nft_net = nft_pernet(net);
10283         struct nft_trans *trans, *next;
10284         LIST_HEAD(set_update_list);
10285         struct nft_trans_elem *te;
10286
10287         if (action == NFNL_ABORT_VALIDATE &&
10288             nf_tables_validate(net) < 0)
10289                 return -EAGAIN;
10290
10291         list_for_each_entry_safe_reverse(trans, next, &nft_net->commit_list,
10292                                          list) {
10293                 switch (trans->msg_type) {
10294                 case NFT_MSG_NEWTABLE:
10295                         if (nft_trans_table_update(trans)) {
10296                                 if (!(trans->ctx.table->flags & __NFT_TABLE_F_UPDATE)) {
10297                                         nft_trans_destroy(trans);
10298                                         break;
10299                                 }
10300                                 if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_DORMANT) {
10301                                         nf_tables_table_disable(net, trans->ctx.table);
10302                                         trans->ctx.table->flags |= NFT_TABLE_F_DORMANT;
10303                                 } else if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_AWAKEN) {
10304                                         trans->ctx.table->flags &= ~NFT_TABLE_F_DORMANT;
10305                                 }
10306                                 trans->ctx.table->flags &= ~__NFT_TABLE_F_UPDATE;
10307                                 nft_trans_destroy(trans);
10308                         } else {
10309                                 list_del_rcu(&trans->ctx.table->list);
10310                         }
10311                         break;
10312                 case NFT_MSG_DELTABLE:
10313                 case NFT_MSG_DESTROYTABLE:
10314                         nft_clear(trans->ctx.net, trans->ctx.table);
10315                         nft_trans_destroy(trans);
10316                         break;
10317                 case NFT_MSG_NEWCHAIN:
10318                         if (nft_trans_chain_update(trans)) {
10319                                 nft_netdev_unregister_hooks(net,
10320                                                             &nft_trans_chain_hooks(trans),
10321                                                             true);
10322                                 free_percpu(nft_trans_chain_stats(trans));
10323                                 kfree(nft_trans_chain_name(trans));
10324                                 nft_trans_destroy(trans);
10325                         } else {
10326                                 if (nft_trans_chain_bound(trans)) {
10327                                         nft_trans_destroy(trans);
10328                                         break;
10329                                 }
10330                                 nft_use_dec_restore(&trans->ctx.table->use);
10331                                 nft_chain_del(trans->ctx.chain);
10332                                 nf_tables_unregister_hook(trans->ctx.net,
10333                                                           trans->ctx.table,
10334                                                           trans->ctx.chain);
10335                         }
10336                         break;
10337                 case NFT_MSG_DELCHAIN:
10338                 case NFT_MSG_DESTROYCHAIN:
10339                         if (nft_trans_chain_update(trans)) {
10340                                 list_splice(&nft_trans_chain_hooks(trans),
10341                                             &nft_trans_basechain(trans)->hook_list);
10342                         } else {
10343                                 nft_use_inc_restore(&trans->ctx.table->use);
10344                                 nft_clear(trans->ctx.net, trans->ctx.chain);
10345                         }
10346                         nft_trans_destroy(trans);
10347                         break;
10348                 case NFT_MSG_NEWRULE:
10349                         if (nft_trans_rule_bound(trans)) {
10350                                 nft_trans_destroy(trans);
10351                                 break;
10352                         }
10353                         nft_use_dec_restore(&trans->ctx.chain->use);
10354                         list_del_rcu(&nft_trans_rule(trans)->list);
10355                         nft_rule_expr_deactivate(&trans->ctx,
10356                                                  nft_trans_rule(trans),
10357                                                  NFT_TRANS_ABORT);
10358                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
10359                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10360                         break;
10361                 case NFT_MSG_DELRULE:
10362                 case NFT_MSG_DESTROYRULE:
10363                         nft_use_inc_restore(&trans->ctx.chain->use);
10364                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
10365                         nft_rule_expr_activate(&trans->ctx, nft_trans_rule(trans));
10366                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
10367                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10368
10369                         nft_trans_destroy(trans);
10370                         break;
10371                 case NFT_MSG_NEWSET:
10372                         if (nft_trans_set_update(trans)) {
10373                                 nft_trans_destroy(trans);
10374                                 break;
10375                         }
10376                         nft_use_dec_restore(&trans->ctx.table->use);
10377                         if (nft_trans_set_bound(trans)) {
10378                                 nft_trans_destroy(trans);
10379                                 break;
10380                         }
10381                         nft_trans_set(trans)->dead = 1;
10382                         list_del_rcu(&nft_trans_set(trans)->list);
10383                         break;
10384                 case NFT_MSG_DELSET:
10385                 case NFT_MSG_DESTROYSET:
10386                         nft_use_inc_restore(&trans->ctx.table->use);
10387                         nft_clear(trans->ctx.net, nft_trans_set(trans));
10388                         if (nft_trans_set(trans)->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
10389                                 nft_map_activate(&trans->ctx, nft_trans_set(trans));
10390
10391                         nft_trans_destroy(trans);
10392                         break;
10393                 case NFT_MSG_NEWSETELEM:
10394                         if (nft_trans_elem_set_bound(trans)) {
10395                                 nft_trans_destroy(trans);
10396                                 break;
10397                         }
10398                         te = (struct nft_trans_elem *)trans->data;
10399                         nft_setelem_remove(net, te->set, &te->elem);
10400                         if (!nft_setelem_is_catchall(te->set, &te->elem))
10401                                 atomic_dec(&te->set->nelems);
10402
10403                         if (te->set->ops->abort &&
10404                             list_empty(&te->set->pending_update)) {
10405                                 list_add_tail(&te->set->pending_update,
10406                                               &set_update_list);
10407                         }
10408                         break;
10409                 case NFT_MSG_DELSETELEM:
10410                 case NFT_MSG_DESTROYSETELEM:
10411                         te = (struct nft_trans_elem *)trans->data;
10412
10413                         nft_setelem_data_activate(net, te->set, &te->elem);
10414                         nft_setelem_activate(net, te->set, &te->elem);
10415                         if (!nft_setelem_is_catchall(te->set, &te->elem))
10416                                 te->set->ndeact--;
10417
10418                         if (te->set->ops->abort &&
10419                             list_empty(&te->set->pending_update)) {
10420                                 list_add_tail(&te->set->pending_update,
10421                                               &set_update_list);
10422                         }
10423                         nft_trans_destroy(trans);
10424                         break;
10425                 case NFT_MSG_NEWOBJ:
10426                         if (nft_trans_obj_update(trans)) {
10427                                 nft_obj_destroy(&trans->ctx, nft_trans_obj_newobj(trans));
10428                                 nft_trans_destroy(trans);
10429                         } else {
10430                                 nft_use_dec_restore(&trans->ctx.table->use);
10431                                 nft_obj_del(nft_trans_obj(trans));
10432                         }
10433                         break;
10434                 case NFT_MSG_DELOBJ:
10435                 case NFT_MSG_DESTROYOBJ:
10436                         nft_use_inc_restore(&trans->ctx.table->use);
10437                         nft_clear(trans->ctx.net, nft_trans_obj(trans));
10438                         nft_trans_destroy(trans);
10439                         break;
10440                 case NFT_MSG_NEWFLOWTABLE:
10441                         if (nft_trans_flowtable_update(trans)) {
10442                                 nft_unregister_flowtable_net_hooks(net,
10443                                                 &nft_trans_flowtable_hooks(trans));
10444                         } else {
10445                                 nft_use_dec_restore(&trans->ctx.table->use);
10446                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
10447                                 nft_unregister_flowtable_net_hooks(net,
10448                                                 &nft_trans_flowtable(trans)->hook_list);
10449                         }
10450                         break;
10451                 case NFT_MSG_DELFLOWTABLE:
10452                 case NFT_MSG_DESTROYFLOWTABLE:
10453                         if (nft_trans_flowtable_update(trans)) {
10454                                 list_splice(&nft_trans_flowtable_hooks(trans),
10455                                             &nft_trans_flowtable(trans)->hook_list);
10456                         } else {
10457                                 nft_use_inc_restore(&trans->ctx.table->use);
10458                                 nft_clear(trans->ctx.net, nft_trans_flowtable(trans));
10459                         }
10460                         nft_trans_destroy(trans);
10461                         break;
10462                 }
10463         }
10464
10465         nft_set_abort_update(&set_update_list);
10466
10467         synchronize_rcu();
10468
10469         list_for_each_entry_safe_reverse(trans, next,
10470                                          &nft_net->commit_list, list) {
10471                 nft_trans_list_del(trans);
10472                 nf_tables_abort_release(trans);
10473         }
10474
10475         if (action == NFNL_ABORT_AUTOLOAD)
10476                 nf_tables_module_autoload(net);
10477         else
10478                 nf_tables_module_autoload_cleanup(net);
10479
10480         return 0;
10481 }
10482
10483 static int nf_tables_abort(struct net *net, struct sk_buff *skb,
10484                            enum nfnl_abort_action action)
10485 {
10486         struct nftables_pernet *nft_net = nft_pernet(net);
10487         unsigned int gc_seq;
10488         int ret;
10489
10490         gc_seq = nft_gc_seq_begin(nft_net);
10491         ret = __nf_tables_abort(net, action);
10492         nft_gc_seq_end(nft_net, gc_seq);
10493         mutex_unlock(&nft_net->commit_mutex);
10494
10495         return ret;
10496 }
10497
10498 static bool nf_tables_valid_genid(struct net *net, u32 genid)
10499 {
10500         struct nftables_pernet *nft_net = nft_pernet(net);
10501         bool genid_ok;
10502
10503         mutex_lock(&nft_net->commit_mutex);
10504
10505         genid_ok = genid == 0 || nft_net->base_seq == genid;
10506         if (!genid_ok)
10507                 mutex_unlock(&nft_net->commit_mutex);
10508
10509         /* else, commit mutex has to be released by commit or abort function */
10510         return genid_ok;
10511 }
10512
10513 static const struct nfnetlink_subsystem nf_tables_subsys = {
10514         .name           = "nf_tables",
10515         .subsys_id      = NFNL_SUBSYS_NFTABLES,
10516         .cb_count       = NFT_MSG_MAX,
10517         .cb             = nf_tables_cb,
10518         .commit         = nf_tables_commit,
10519         .abort          = nf_tables_abort,
10520         .valid_genid    = nf_tables_valid_genid,
10521         .owner          = THIS_MODULE,
10522 };
10523
10524 int nft_chain_validate_dependency(const struct nft_chain *chain,
10525                                   enum nft_chain_types type)
10526 {
10527         const struct nft_base_chain *basechain;
10528
10529         if (nft_is_base_chain(chain)) {
10530                 basechain = nft_base_chain(chain);
10531                 if (basechain->type->type != type)
10532                         return -EOPNOTSUPP;
10533         }
10534         return 0;
10535 }
10536 EXPORT_SYMBOL_GPL(nft_chain_validate_dependency);
10537
10538 int nft_chain_validate_hooks(const struct nft_chain *chain,
10539                              unsigned int hook_flags)
10540 {
10541         struct nft_base_chain *basechain;
10542
10543         if (nft_is_base_chain(chain)) {
10544                 basechain = nft_base_chain(chain);
10545
10546                 if ((1 << basechain->ops.hooknum) & hook_flags)
10547                         return 0;
10548
10549                 return -EOPNOTSUPP;
10550         }
10551
10552         return 0;
10553 }
10554 EXPORT_SYMBOL_GPL(nft_chain_validate_hooks);
10555
10556 /*
10557  * Loop detection - walk through the ruleset beginning at the destination chain
10558  * of a new jump until either the source chain is reached (loop) or all
10559  * reachable chains have been traversed.
10560  *
10561  * The loop check is performed whenever a new jump verdict is added to an
10562  * expression or verdict map or a verdict map is bound to a new chain.
10563  */
10564
10565 static int nf_tables_check_loops(const struct nft_ctx *ctx,
10566                                  const struct nft_chain *chain);
10567
10568 static int nft_check_loops(const struct nft_ctx *ctx,
10569                            const struct nft_set_ext *ext)
10570 {
10571         const struct nft_data *data;
10572         int ret;
10573
10574         data = nft_set_ext_data(ext);
10575         switch (data->verdict.code) {
10576         case NFT_JUMP:
10577         case NFT_GOTO:
10578                 ret = nf_tables_check_loops(ctx, data->verdict.chain);
10579                 break;
10580         default:
10581                 ret = 0;
10582                 break;
10583         }
10584
10585         return ret;
10586 }
10587
10588 static int nf_tables_loop_check_setelem(const struct nft_ctx *ctx,
10589                                         struct nft_set *set,
10590                                         const struct nft_set_iter *iter,
10591                                         struct nft_set_elem *elem)
10592 {
10593         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
10594
10595         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
10596             *nft_set_ext_flags(ext) & NFT_SET_ELEM_INTERVAL_END)
10597                 return 0;
10598
10599         return nft_check_loops(ctx, ext);
10600 }
10601
10602 static int nft_set_catchall_loops(const struct nft_ctx *ctx,
10603                                   struct nft_set *set)
10604 {
10605         u8 genmask = nft_genmask_next(ctx->net);
10606         struct nft_set_elem_catchall *catchall;
10607         struct nft_set_ext *ext;
10608         int ret = 0;
10609
10610         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
10611                 ext = nft_set_elem_ext(set, catchall->elem);
10612                 if (!nft_set_elem_active(ext, genmask))
10613                         continue;
10614
10615                 ret = nft_check_loops(ctx, ext);
10616                 if (ret < 0)
10617                         return ret;
10618         }
10619
10620         return ret;
10621 }
10622
10623 static int nf_tables_check_loops(const struct nft_ctx *ctx,
10624                                  const struct nft_chain *chain)
10625 {
10626         const struct nft_rule *rule;
10627         const struct nft_expr *expr, *last;
10628         struct nft_set *set;
10629         struct nft_set_binding *binding;
10630         struct nft_set_iter iter;
10631
10632         if (ctx->chain == chain)
10633                 return -ELOOP;
10634
10635         if (fatal_signal_pending(current))
10636                 return -EINTR;
10637
10638         list_for_each_entry(rule, &chain->rules, list) {
10639                 nft_rule_for_each_expr(expr, last, rule) {
10640                         struct nft_immediate_expr *priv;
10641                         const struct nft_data *data;
10642                         int err;
10643
10644                         if (strcmp(expr->ops->type->name, "immediate"))
10645                                 continue;
10646
10647                         priv = nft_expr_priv(expr);
10648                         if (priv->dreg != NFT_REG_VERDICT)
10649                                 continue;
10650
10651                         data = &priv->data;
10652                         switch (data->verdict.code) {
10653                         case NFT_JUMP:
10654                         case NFT_GOTO:
10655                                 err = nf_tables_check_loops(ctx,
10656                                                         data->verdict.chain);
10657                                 if (err < 0)
10658                                         return err;
10659                                 break;
10660                         default:
10661                                 break;
10662                         }
10663                 }
10664         }
10665
10666         list_for_each_entry(set, &ctx->table->sets, list) {
10667                 if (!nft_is_active_next(ctx->net, set))
10668                         continue;
10669                 if (!(set->flags & NFT_SET_MAP) ||
10670                     set->dtype != NFT_DATA_VERDICT)
10671                         continue;
10672
10673                 list_for_each_entry(binding, &set->bindings, list) {
10674                         if (!(binding->flags & NFT_SET_MAP) ||
10675                             binding->chain != chain)
10676                                 continue;
10677
10678                         iter.genmask    = nft_genmask_next(ctx->net);
10679                         iter.skip       = 0;
10680                         iter.count      = 0;
10681                         iter.err        = 0;
10682                         iter.fn         = nf_tables_loop_check_setelem;
10683
10684                         set->ops->walk(ctx, set, &iter);
10685                         if (!iter.err)
10686                                 iter.err = nft_set_catchall_loops(ctx, set);
10687
10688                         if (iter.err < 0)
10689                                 return iter.err;
10690                 }
10691         }
10692
10693         return 0;
10694 }
10695
10696 /**
10697  *      nft_parse_u32_check - fetch u32 attribute and check for maximum value
10698  *
10699  *      @attr: netlink attribute to fetch value from
10700  *      @max: maximum value to be stored in dest
10701  *      @dest: pointer to the variable
10702  *
10703  *      Parse, check and store a given u32 netlink attribute into variable.
10704  *      This function returns -ERANGE if the value goes over maximum value.
10705  *      Otherwise a 0 is returned and the attribute value is stored in the
10706  *      destination variable.
10707  */
10708 int nft_parse_u32_check(const struct nlattr *attr, int max, u32 *dest)
10709 {
10710         u32 val;
10711
10712         val = ntohl(nla_get_be32(attr));
10713         if (val > max)
10714                 return -ERANGE;
10715
10716         *dest = val;
10717         return 0;
10718 }
10719 EXPORT_SYMBOL_GPL(nft_parse_u32_check);
10720
10721 static int nft_parse_register(const struct nlattr *attr, u32 *preg)
10722 {
10723         unsigned int reg;
10724
10725         reg = ntohl(nla_get_be32(attr));
10726         switch (reg) {
10727         case NFT_REG_VERDICT...NFT_REG_4:
10728                 *preg = reg * NFT_REG_SIZE / NFT_REG32_SIZE;
10729                 break;
10730         case NFT_REG32_00...NFT_REG32_15:
10731                 *preg = reg + NFT_REG_SIZE / NFT_REG32_SIZE - NFT_REG32_00;
10732                 break;
10733         default:
10734                 return -ERANGE;
10735         }
10736
10737         return 0;
10738 }
10739
10740 /**
10741  *      nft_dump_register - dump a register value to a netlink attribute
10742  *
10743  *      @skb: socket buffer
10744  *      @attr: attribute number
10745  *      @reg: register number
10746  *
10747  *      Construct a netlink attribute containing the register number. For
10748  *      compatibility reasons, register numbers being a multiple of 4 are
10749  *      translated to the corresponding 128 bit register numbers.
10750  */
10751 int nft_dump_register(struct sk_buff *skb, unsigned int attr, unsigned int reg)
10752 {
10753         if (reg % (NFT_REG_SIZE / NFT_REG32_SIZE) == 0)
10754                 reg = reg / (NFT_REG_SIZE / NFT_REG32_SIZE);
10755         else
10756                 reg = reg - NFT_REG_SIZE / NFT_REG32_SIZE + NFT_REG32_00;
10757
10758         return nla_put_be32(skb, attr, htonl(reg));
10759 }
10760 EXPORT_SYMBOL_GPL(nft_dump_register);
10761
10762 static int nft_validate_register_load(enum nft_registers reg, unsigned int len)
10763 {
10764         if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
10765                 return -EINVAL;
10766         if (len == 0)
10767                 return -EINVAL;
10768         if (reg * NFT_REG32_SIZE + len > sizeof_field(struct nft_regs, data))
10769                 return -ERANGE;
10770
10771         return 0;
10772 }
10773
10774 int nft_parse_register_load(const struct nlattr *attr, u8 *sreg, u32 len)
10775 {
10776         u32 reg;
10777         int err;
10778
10779         err = nft_parse_register(attr, &reg);
10780         if (err < 0)
10781                 return err;
10782
10783         err = nft_validate_register_load(reg, len);
10784         if (err < 0)
10785                 return err;
10786
10787         *sreg = reg;
10788         return 0;
10789 }
10790 EXPORT_SYMBOL_GPL(nft_parse_register_load);
10791
10792 static int nft_validate_register_store(const struct nft_ctx *ctx,
10793                                        enum nft_registers reg,
10794                                        const struct nft_data *data,
10795                                        enum nft_data_types type,
10796                                        unsigned int len)
10797 {
10798         int err;
10799
10800         switch (reg) {
10801         case NFT_REG_VERDICT:
10802                 if (type != NFT_DATA_VERDICT)
10803                         return -EINVAL;
10804
10805                 if (data != NULL &&
10806                     (data->verdict.code == NFT_GOTO ||
10807                      data->verdict.code == NFT_JUMP)) {
10808                         err = nf_tables_check_loops(ctx, data->verdict.chain);
10809                         if (err < 0)
10810                                 return err;
10811                 }
10812
10813                 return 0;
10814         default:
10815                 if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
10816                         return -EINVAL;
10817                 if (len == 0)
10818                         return -EINVAL;
10819                 if (reg * NFT_REG32_SIZE + len >
10820                     sizeof_field(struct nft_regs, data))
10821                         return -ERANGE;
10822
10823                 if (data != NULL && type != NFT_DATA_VALUE)
10824                         return -EINVAL;
10825                 return 0;
10826         }
10827 }
10828
10829 int nft_parse_register_store(const struct nft_ctx *ctx,
10830                              const struct nlattr *attr, u8 *dreg,
10831                              const struct nft_data *data,
10832                              enum nft_data_types type, unsigned int len)
10833 {
10834         int err;
10835         u32 reg;
10836
10837         err = nft_parse_register(attr, &reg);
10838         if (err < 0)
10839                 return err;
10840
10841         err = nft_validate_register_store(ctx, reg, data, type, len);
10842         if (err < 0)
10843                 return err;
10844
10845         *dreg = reg;
10846         return 0;
10847 }
10848 EXPORT_SYMBOL_GPL(nft_parse_register_store);
10849
10850 static const struct nla_policy nft_verdict_policy[NFTA_VERDICT_MAX + 1] = {
10851         [NFTA_VERDICT_CODE]     = { .type = NLA_U32 },
10852         [NFTA_VERDICT_CHAIN]    = { .type = NLA_STRING,
10853                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
10854         [NFTA_VERDICT_CHAIN_ID] = { .type = NLA_U32 },
10855 };
10856
10857 static int nft_verdict_init(const struct nft_ctx *ctx, struct nft_data *data,
10858                             struct nft_data_desc *desc, const struct nlattr *nla)
10859 {
10860         u8 genmask = nft_genmask_next(ctx->net);
10861         struct nlattr *tb[NFTA_VERDICT_MAX + 1];
10862         struct nft_chain *chain;
10863         int err;
10864
10865         err = nla_parse_nested_deprecated(tb, NFTA_VERDICT_MAX, nla,
10866                                           nft_verdict_policy, NULL);
10867         if (err < 0)
10868                 return err;
10869
10870         if (!tb[NFTA_VERDICT_CODE])
10871                 return -EINVAL;
10872
10873         /* zero padding hole for memcmp */
10874         memset(data, 0, sizeof(*data));
10875         data->verdict.code = ntohl(nla_get_be32(tb[NFTA_VERDICT_CODE]));
10876
10877         switch (data->verdict.code) {
10878         case NF_ACCEPT:
10879         case NF_DROP:
10880         case NF_QUEUE:
10881                 break;
10882         case NFT_CONTINUE:
10883         case NFT_BREAK:
10884         case NFT_RETURN:
10885                 break;
10886         case NFT_JUMP:
10887         case NFT_GOTO:
10888                 if (tb[NFTA_VERDICT_CHAIN]) {
10889                         chain = nft_chain_lookup(ctx->net, ctx->table,
10890                                                  tb[NFTA_VERDICT_CHAIN],
10891                                                  genmask);
10892                 } else if (tb[NFTA_VERDICT_CHAIN_ID]) {
10893                         chain = nft_chain_lookup_byid(ctx->net, ctx->table,
10894                                                       tb[NFTA_VERDICT_CHAIN_ID],
10895                                                       genmask);
10896                         if (IS_ERR(chain))
10897                                 return PTR_ERR(chain);
10898                 } else {
10899                         return -EINVAL;
10900                 }
10901
10902                 if (IS_ERR(chain))
10903                         return PTR_ERR(chain);
10904                 if (nft_is_base_chain(chain))
10905                         return -EOPNOTSUPP;
10906                 if (nft_chain_is_bound(chain))
10907                         return -EINVAL;
10908                 if (desc->flags & NFT_DATA_DESC_SETELEM &&
10909                     chain->flags & NFT_CHAIN_BINDING)
10910                         return -EINVAL;
10911                 if (!nft_use_inc(&chain->use))
10912                         return -EMFILE;
10913
10914                 data->verdict.chain = chain;
10915                 break;
10916         default:
10917                 return -EINVAL;
10918         }
10919
10920         desc->len = sizeof(data->verdict);
10921
10922         return 0;
10923 }
10924
10925 static void nft_verdict_uninit(const struct nft_data *data)
10926 {
10927         struct nft_chain *chain;
10928
10929         switch (data->verdict.code) {
10930         case NFT_JUMP:
10931         case NFT_GOTO:
10932                 chain = data->verdict.chain;
10933                 nft_use_dec(&chain->use);
10934                 break;
10935         }
10936 }
10937
10938 int nft_verdict_dump(struct sk_buff *skb, int type, const struct nft_verdict *v)
10939 {
10940         struct nlattr *nest;
10941
10942         nest = nla_nest_start_noflag(skb, type);
10943         if (!nest)
10944                 goto nla_put_failure;
10945
10946         if (nla_put_be32(skb, NFTA_VERDICT_CODE, htonl(v->code)))
10947                 goto nla_put_failure;
10948
10949         switch (v->code) {
10950         case NFT_JUMP:
10951         case NFT_GOTO:
10952                 if (nla_put_string(skb, NFTA_VERDICT_CHAIN,
10953                                    v->chain->name))
10954                         goto nla_put_failure;
10955         }
10956         nla_nest_end(skb, nest);
10957         return 0;
10958
10959 nla_put_failure:
10960         return -1;
10961 }
10962
10963 static int nft_value_init(const struct nft_ctx *ctx,
10964                           struct nft_data *data, struct nft_data_desc *desc,
10965                           const struct nlattr *nla)
10966 {
10967         unsigned int len;
10968
10969         len = nla_len(nla);
10970         if (len == 0)
10971                 return -EINVAL;
10972         if (len > desc->size)
10973                 return -EOVERFLOW;
10974         if (desc->len) {
10975                 if (len != desc->len)
10976                         return -EINVAL;
10977         } else {
10978                 desc->len = len;
10979         }
10980
10981         nla_memcpy(data->data, nla, len);
10982
10983         return 0;
10984 }
10985
10986 static int nft_value_dump(struct sk_buff *skb, const struct nft_data *data,
10987                           unsigned int len)
10988 {
10989         return nla_put(skb, NFTA_DATA_VALUE, len, data->data);
10990 }
10991
10992 static const struct nla_policy nft_data_policy[NFTA_DATA_MAX + 1] = {
10993         [NFTA_DATA_VALUE]       = { .type = NLA_BINARY },
10994         [NFTA_DATA_VERDICT]     = { .type = NLA_NESTED },
10995 };
10996
10997 /**
10998  *      nft_data_init - parse nf_tables data netlink attributes
10999  *
11000  *      @ctx: context of the expression using the data
11001  *      @data: destination struct nft_data
11002  *      @desc: data description
11003  *      @nla: netlink attribute containing data
11004  *
11005  *      Parse the netlink data attributes and initialize a struct nft_data.
11006  *      The type and length of data are returned in the data description.
11007  *
11008  *      The caller can indicate that it only wants to accept data of type
11009  *      NFT_DATA_VALUE by passing NULL for the ctx argument.
11010  */
11011 int nft_data_init(const struct nft_ctx *ctx, struct nft_data *data,
11012                   struct nft_data_desc *desc, const struct nlattr *nla)
11013 {
11014         struct nlattr *tb[NFTA_DATA_MAX + 1];
11015         int err;
11016
11017         if (WARN_ON_ONCE(!desc->size))
11018                 return -EINVAL;
11019
11020         err = nla_parse_nested_deprecated(tb, NFTA_DATA_MAX, nla,
11021                                           nft_data_policy, NULL);
11022         if (err < 0)
11023                 return err;
11024
11025         if (tb[NFTA_DATA_VALUE]) {
11026                 if (desc->type != NFT_DATA_VALUE)
11027                         return -EINVAL;
11028
11029                 err = nft_value_init(ctx, data, desc, tb[NFTA_DATA_VALUE]);
11030         } else if (tb[NFTA_DATA_VERDICT] && ctx != NULL) {
11031                 if (desc->type != NFT_DATA_VERDICT)
11032                         return -EINVAL;
11033
11034                 err = nft_verdict_init(ctx, data, desc, tb[NFTA_DATA_VERDICT]);
11035         } else {
11036                 err = -EINVAL;
11037         }
11038
11039         return err;
11040 }
11041 EXPORT_SYMBOL_GPL(nft_data_init);
11042
11043 /**
11044  *      nft_data_release - release a nft_data item
11045  *
11046  *      @data: struct nft_data to release
11047  *      @type: type of data
11048  *
11049  *      Release a nft_data item. NFT_DATA_VALUE types can be silently discarded,
11050  *      all others need to be released by calling this function.
11051  */
11052 void nft_data_release(const struct nft_data *data, enum nft_data_types type)
11053 {
11054         if (type < NFT_DATA_VERDICT)
11055                 return;
11056         switch (type) {
11057         case NFT_DATA_VERDICT:
11058                 return nft_verdict_uninit(data);
11059         default:
11060                 WARN_ON(1);
11061         }
11062 }
11063 EXPORT_SYMBOL_GPL(nft_data_release);
11064
11065 int nft_data_dump(struct sk_buff *skb, int attr, const struct nft_data *data,
11066                   enum nft_data_types type, unsigned int len)
11067 {
11068         struct nlattr *nest;
11069         int err;
11070
11071         nest = nla_nest_start_noflag(skb, attr);
11072         if (nest == NULL)
11073                 return -1;
11074
11075         switch (type) {
11076         case NFT_DATA_VALUE:
11077                 err = nft_value_dump(skb, data, len);
11078                 break;
11079         case NFT_DATA_VERDICT:
11080                 err = nft_verdict_dump(skb, NFTA_DATA_VERDICT, &data->verdict);
11081                 break;
11082         default:
11083                 err = -EINVAL;
11084                 WARN_ON(1);
11085         }
11086
11087         nla_nest_end(skb, nest);
11088         return err;
11089 }
11090 EXPORT_SYMBOL_GPL(nft_data_dump);
11091
11092 int __nft_release_basechain(struct nft_ctx *ctx)
11093 {
11094         struct nft_rule *rule, *nr;
11095
11096         if (WARN_ON(!nft_is_base_chain(ctx->chain)))
11097                 return 0;
11098
11099         nf_tables_unregister_hook(ctx->net, ctx->chain->table, ctx->chain);
11100         list_for_each_entry_safe(rule, nr, &ctx->chain->rules, list) {
11101                 list_del(&rule->list);
11102                 nft_use_dec(&ctx->chain->use);
11103                 nf_tables_rule_release(ctx, rule);
11104         }
11105         nft_chain_del(ctx->chain);
11106         nft_use_dec(&ctx->table->use);
11107         nf_tables_chain_destroy(ctx);
11108
11109         return 0;
11110 }
11111 EXPORT_SYMBOL_GPL(__nft_release_basechain);
11112
11113 static void __nft_release_hook(struct net *net, struct nft_table *table)
11114 {
11115         struct nft_flowtable *flowtable;
11116         struct nft_chain *chain;
11117
11118         list_for_each_entry(chain, &table->chains, list)
11119                 __nf_tables_unregister_hook(net, table, chain, true);
11120         list_for_each_entry(flowtable, &table->flowtables, list)
11121                 __nft_unregister_flowtable_net_hooks(net, &flowtable->hook_list,
11122                                                      true);
11123 }
11124
11125 static void __nft_release_hooks(struct net *net)
11126 {
11127         struct nftables_pernet *nft_net = nft_pernet(net);
11128         struct nft_table *table;
11129
11130         list_for_each_entry(table, &nft_net->tables, list) {
11131                 if (nft_table_has_owner(table))
11132                         continue;
11133
11134                 __nft_release_hook(net, table);
11135         }
11136 }
11137
11138 static void __nft_release_table(struct net *net, struct nft_table *table)
11139 {
11140         struct nft_flowtable *flowtable, *nf;
11141         struct nft_chain *chain, *nc;
11142         struct nft_object *obj, *ne;
11143         struct nft_rule *rule, *nr;
11144         struct nft_set *set, *ns;
11145         struct nft_ctx ctx = {
11146                 .net    = net,
11147                 .family = NFPROTO_NETDEV,
11148         };
11149
11150         ctx.family = table->family;
11151         ctx.table = table;
11152         list_for_each_entry(chain, &table->chains, list) {
11153                 if (nft_chain_binding(chain))
11154                         continue;
11155
11156                 ctx.chain = chain;
11157                 list_for_each_entry_safe(rule, nr, &chain->rules, list) {
11158                         list_del(&rule->list);
11159                         nft_use_dec(&chain->use);
11160                         nf_tables_rule_release(&ctx, rule);
11161                 }
11162         }
11163         list_for_each_entry_safe(flowtable, nf, &table->flowtables, list) {
11164                 list_del(&flowtable->list);
11165                 nft_use_dec(&table->use);
11166                 nf_tables_flowtable_destroy(flowtable);
11167         }
11168         list_for_each_entry_safe(set, ns, &table->sets, list) {
11169                 list_del(&set->list);
11170                 nft_use_dec(&table->use);
11171                 if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
11172                         nft_map_deactivate(&ctx, set);
11173
11174                 nft_set_destroy(&ctx, set);
11175         }
11176         list_for_each_entry_safe(obj, ne, &table->objects, list) {
11177                 nft_obj_del(obj);
11178                 nft_use_dec(&table->use);
11179                 nft_obj_destroy(&ctx, obj);
11180         }
11181         list_for_each_entry_safe(chain, nc, &table->chains, list) {
11182                 ctx.chain = chain;
11183                 nft_chain_del(chain);
11184                 nft_use_dec(&table->use);
11185                 nf_tables_chain_destroy(&ctx);
11186         }
11187         nf_tables_table_destroy(&ctx);
11188 }
11189
11190 static void __nft_release_tables(struct net *net)
11191 {
11192         struct nftables_pernet *nft_net = nft_pernet(net);
11193         struct nft_table *table, *nt;
11194
11195         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
11196                 if (nft_table_has_owner(table))
11197                         continue;
11198
11199                 list_del(&table->list);
11200
11201                 __nft_release_table(net, table);
11202         }
11203 }
11204
11205 static int nft_rcv_nl_event(struct notifier_block *this, unsigned long event,
11206                             void *ptr)
11207 {
11208         struct nft_table *table, *to_delete[8];
11209         struct nftables_pernet *nft_net;
11210         struct netlink_notify *n = ptr;
11211         struct net *net = n->net;
11212         unsigned int deleted;
11213         bool restart = false;
11214         unsigned int gc_seq;
11215
11216         if (event != NETLINK_URELEASE || n->protocol != NETLINK_NETFILTER)
11217                 return NOTIFY_DONE;
11218
11219         nft_net = nft_pernet(net);
11220         deleted = 0;
11221         mutex_lock(&nft_net->commit_mutex);
11222
11223         gc_seq = nft_gc_seq_begin(nft_net);
11224
11225         if (!list_empty(&nf_tables_destroy_list))
11226                 nf_tables_trans_destroy_flush_work();
11227 again:
11228         list_for_each_entry(table, &nft_net->tables, list) {
11229                 if (nft_table_has_owner(table) &&
11230                     n->portid == table->nlpid) {
11231                         __nft_release_hook(net, table);
11232                         list_del_rcu(&table->list);
11233                         to_delete[deleted++] = table;
11234                         if (deleted >= ARRAY_SIZE(to_delete))
11235                                 break;
11236                 }
11237         }
11238         if (deleted) {
11239                 restart = deleted >= ARRAY_SIZE(to_delete);
11240                 synchronize_rcu();
11241                 while (deleted)
11242                         __nft_release_table(net, to_delete[--deleted]);
11243
11244                 if (restart)
11245                         goto again;
11246         }
11247         nft_gc_seq_end(nft_net, gc_seq);
11248
11249         mutex_unlock(&nft_net->commit_mutex);
11250
11251         return NOTIFY_DONE;
11252 }
11253
11254 static struct notifier_block nft_nl_notifier = {
11255         .notifier_call  = nft_rcv_nl_event,
11256 };
11257
11258 static int __net_init nf_tables_init_net(struct net *net)
11259 {
11260         struct nftables_pernet *nft_net = nft_pernet(net);
11261
11262         INIT_LIST_HEAD(&nft_net->tables);
11263         INIT_LIST_HEAD(&nft_net->commit_list);
11264         INIT_LIST_HEAD(&nft_net->binding_list);
11265         INIT_LIST_HEAD(&nft_net->module_list);
11266         INIT_LIST_HEAD(&nft_net->notify_list);
11267         mutex_init(&nft_net->commit_mutex);
11268         nft_net->base_seq = 1;
11269         nft_net->gc_seq = 0;
11270         nft_net->validate_state = NFT_VALIDATE_SKIP;
11271
11272         return 0;
11273 }
11274
11275 static void __net_exit nf_tables_pre_exit_net(struct net *net)
11276 {
11277         struct nftables_pernet *nft_net = nft_pernet(net);
11278
11279         mutex_lock(&nft_net->commit_mutex);
11280         __nft_release_hooks(net);
11281         mutex_unlock(&nft_net->commit_mutex);
11282 }
11283
11284 static void __net_exit nf_tables_exit_net(struct net *net)
11285 {
11286         struct nftables_pernet *nft_net = nft_pernet(net);
11287         unsigned int gc_seq;
11288
11289         mutex_lock(&nft_net->commit_mutex);
11290
11291         gc_seq = nft_gc_seq_begin(nft_net);
11292
11293         if (!list_empty(&nft_net->commit_list) ||
11294             !list_empty(&nft_net->module_list))
11295                 __nf_tables_abort(net, NFNL_ABORT_NONE);
11296
11297         __nft_release_tables(net);
11298
11299         nft_gc_seq_end(nft_net, gc_seq);
11300
11301         mutex_unlock(&nft_net->commit_mutex);
11302         WARN_ON_ONCE(!list_empty(&nft_net->tables));
11303         WARN_ON_ONCE(!list_empty(&nft_net->module_list));
11304         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
11305 }
11306
11307 static void nf_tables_exit_batch(struct list_head *net_exit_list)
11308 {
11309         flush_work(&trans_gc_work);
11310 }
11311
11312 static struct pernet_operations nf_tables_net_ops = {
11313         .init           = nf_tables_init_net,
11314         .pre_exit       = nf_tables_pre_exit_net,
11315         .exit           = nf_tables_exit_net,
11316         .exit_batch     = nf_tables_exit_batch,
11317         .id             = &nf_tables_net_id,
11318         .size           = sizeof(struct nftables_pernet),
11319 };
11320
11321 static int __init nf_tables_module_init(void)
11322 {
11323         int err;
11324
11325         err = register_pernet_subsys(&nf_tables_net_ops);
11326         if (err < 0)
11327                 return err;
11328
11329         err = nft_chain_filter_init();
11330         if (err < 0)
11331                 goto err_chain_filter;
11332
11333         err = nf_tables_core_module_init();
11334         if (err < 0)
11335                 goto err_core_module;
11336
11337         err = register_netdevice_notifier(&nf_tables_flowtable_notifier);
11338         if (err < 0)
11339                 goto err_netdev_notifier;
11340
11341         err = rhltable_init(&nft_objname_ht, &nft_objname_ht_params);
11342         if (err < 0)
11343                 goto err_rht_objname;
11344
11345         err = nft_offload_init();
11346         if (err < 0)
11347                 goto err_offload;
11348
11349         err = netlink_register_notifier(&nft_nl_notifier);
11350         if (err < 0)
11351                 goto err_netlink_notifier;
11352
11353         /* must be last */
11354         err = nfnetlink_subsys_register(&nf_tables_subsys);
11355         if (err < 0)
11356                 goto err_nfnl_subsys;
11357
11358         nft_chain_route_init();
11359
11360         return err;
11361
11362 err_nfnl_subsys:
11363         netlink_unregister_notifier(&nft_nl_notifier);
11364 err_netlink_notifier:
11365         nft_offload_exit();
11366 err_offload:
11367         rhltable_destroy(&nft_objname_ht);
11368 err_rht_objname:
11369         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
11370 err_netdev_notifier:
11371         nf_tables_core_module_exit();
11372 err_core_module:
11373         nft_chain_filter_fini();
11374 err_chain_filter:
11375         unregister_pernet_subsys(&nf_tables_net_ops);
11376         return err;
11377 }
11378
11379 static void __exit nf_tables_module_exit(void)
11380 {
11381         nfnetlink_subsys_unregister(&nf_tables_subsys);
11382         netlink_unregister_notifier(&nft_nl_notifier);
11383         nft_offload_exit();
11384         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
11385         nft_chain_filter_fini();
11386         nft_chain_route_fini();
11387         unregister_pernet_subsys(&nf_tables_net_ops);
11388         cancel_work_sync(&trans_gc_work);
11389         cancel_work_sync(&trans_destroy_work);
11390         rcu_barrier();
11391         rhltable_destroy(&nft_objname_ht);
11392         nf_tables_core_module_exit();
11393 }
11394
11395 module_init(nf_tables_module_init);
11396 module_exit(nf_tables_module_exit);
11397
11398 MODULE_LICENSE("GPL");
11399 MODULE_AUTHOR("Patrick McHardy <kaber@trash.net>");
11400 MODULE_ALIAS_NFNL_SUBSYS(NFNL_SUBSYS_NFTABLES);