net/netfilter/nf_tables_api.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2007-2009 Patrick McHardy <kaber@trash.net>
   4  *
   5  * Development of this code funded by Astaro AG (http://www.astaro.com/)
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/init.h>
  10 #include <linux/list.h>
  11 #include <linux/skbuff.h>
  12 #include <linux/netlink.h>
  13 #include <linux/vmalloc.h>
  14 #include <linux/rhashtable.h>
  15 #include <linux/audit.h>
  16 #include <linux/netfilter.h>
  17 #include <linux/netfilter/nfnetlink.h>
  18 #include <linux/netfilter/nf_tables.h>
  19 #include <net/netfilter/nf_flow_table.h>
  20 #include <net/netfilter/nf_tables_core.h>
  21 #include <net/netfilter/nf_tables.h>
  22 #include <net/netfilter/nf_tables_offload.h>
  23 #include <net/net_namespace.h>
  24 #include <net/sock.h>
  25
  26 #define NFT_MODULE_AUTOLOAD_LIMIT (MODULE_NAME_LEN - sizeof("nft-expr-255-"))
  27
  28 unsigned int nf_tables_net_id __read_mostly;
  29
  30 static LIST_HEAD(nf_tables_expressions);
  31 static LIST_HEAD(nf_tables_objects);
  32 static LIST_HEAD(nf_tables_flowtables);
  33 static LIST_HEAD(nf_tables_destroy_list);
  34 static DEFINE_SPINLOCK(nf_tables_destroy_list_lock);
  35
  36 enum {
  37         NFT_VALIDATE_SKIP       = 0,
  38         NFT_VALIDATE_NEED,
  39         NFT_VALIDATE_DO,
  40 };
  41
  42 static struct rhltable nft_objname_ht;
  43
  44 static u32 nft_chain_hash(const void *data, u32 len, u32 seed);
  45 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed);
  46 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *, const void *);
  47
  48 static u32 nft_objname_hash(const void *data, u32 len, u32 seed);
  49 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed);
  50 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *, const void *);
  51
  52 static const struct rhashtable_params nft_chain_ht_params = {
  53         .head_offset            = offsetof(struct nft_chain, rhlhead),
  54         .key_offset             = offsetof(struct nft_chain, name),
  55         .hashfn                 = nft_chain_hash,
  56         .obj_hashfn             = nft_chain_hash_obj,
  57         .obj_cmpfn              = nft_chain_hash_cmp,
  58         .automatic_shrinking    = true,
  59 };
  60
  61 static const struct rhashtable_params nft_objname_ht_params = {
  62         .head_offset            = offsetof(struct nft_object, rhlhead),
  63         .key_offset             = offsetof(struct nft_object, key),
  64         .hashfn                 = nft_objname_hash,
  65         .obj_hashfn             = nft_objname_hash_obj,
  66         .obj_cmpfn              = nft_objname_hash_cmp,
  67         .automatic_shrinking    = true,
  68 };
  69
  70 struct nft_audit_data {
  71         struct nft_table *table;
  72         int entries;
  73         int op;
  74         struct list_head list;
  75 };
  76
  77 static const u8 nft2audit_op[NFT_MSG_MAX] = { // enum nf_tables_msg_types
  78         [NFT_MSG_NEWTABLE]      = AUDIT_NFT_OP_TABLE_REGISTER,
  79         [NFT_MSG_GETTABLE]      = AUDIT_NFT_OP_INVALID,
  80         [NFT_MSG_DELTABLE]      = AUDIT_NFT_OP_TABLE_UNREGISTER,
  81         [NFT_MSG_NEWCHAIN]      = AUDIT_NFT_OP_CHAIN_REGISTER,
  82         [NFT_MSG_GETCHAIN]      = AUDIT_NFT_OP_INVALID,
  83         [NFT_MSG_DELCHAIN]      = AUDIT_NFT_OP_CHAIN_UNREGISTER,
  84         [NFT_MSG_NEWRULE]       = AUDIT_NFT_OP_RULE_REGISTER,
  85         [NFT_MSG_GETRULE]       = AUDIT_NFT_OP_INVALID,
  86         [NFT_MSG_DELRULE]       = AUDIT_NFT_OP_RULE_UNREGISTER,
  87         [NFT_MSG_NEWSET]        = AUDIT_NFT_OP_SET_REGISTER,
  88         [NFT_MSG_GETSET]        = AUDIT_NFT_OP_INVALID,
  89         [NFT_MSG_DELSET]        = AUDIT_NFT_OP_SET_UNREGISTER,
  90         [NFT_MSG_NEWSETELEM]    = AUDIT_NFT_OP_SETELEM_REGISTER,
  91         [NFT_MSG_GETSETELEM]    = AUDIT_NFT_OP_INVALID,
  92         [NFT_MSG_DELSETELEM]    = AUDIT_NFT_OP_SETELEM_UNREGISTER,
  93         [NFT_MSG_NEWGEN]        = AUDIT_NFT_OP_GEN_REGISTER,
  94         [NFT_MSG_GETGEN]        = AUDIT_NFT_OP_INVALID,
  95         [NFT_MSG_TRACE]         = AUDIT_NFT_OP_INVALID,
  96         [NFT_MSG_NEWOBJ]        = AUDIT_NFT_OP_OBJ_REGISTER,
  97         [NFT_MSG_GETOBJ]        = AUDIT_NFT_OP_INVALID,
  98         [NFT_MSG_DELOBJ]        = AUDIT_NFT_OP_OBJ_UNREGISTER,
  99         [NFT_MSG_GETOBJ_RESET]  = AUDIT_NFT_OP_OBJ_RESET,
 100         [NFT_MSG_NEWFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_REGISTER,
 101         [NFT_MSG_GETFLOWTABLE]  = AUDIT_NFT_OP_INVALID,
 102         [NFT_MSG_DELFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_UNREGISTER,
 103 };
 104
 105 static void nft_validate_state_update(struct nft_table *table, u8 new_validate_state)
 106 {
 107         switch (table->validate_state) {
 108         case NFT_VALIDATE_SKIP:
 109                 WARN_ON_ONCE(new_validate_state == NFT_VALIDATE_DO);
 110                 break;
 111         case NFT_VALIDATE_NEED:
 112                 break;
 113         case NFT_VALIDATE_DO:
 114                 if (new_validate_state == NFT_VALIDATE_NEED)
 115                         return;
 116         }
 117
 118         table->validate_state = new_validate_state;
 119 }
 120 static void nf_tables_trans_destroy_work(struct work_struct *w);
 121 static DECLARE_WORK(trans_destroy_work, nf_tables_trans_destroy_work);
 122
 123 static void nft_ctx_init(struct nft_ctx *ctx,
 124                          struct net *net,
 125                          const struct sk_buff *skb,
 126                          const struct nlmsghdr *nlh,
 127                          u8 family,
 128                          struct nft_table *table,
 129                          struct nft_chain *chain,
 130                          const struct nlattr * const *nla)
 131 {
 132         ctx->net        = net;
 133         ctx->family     = family;
 134         ctx->level      = 0;
 135         ctx->table      = table;
 136         ctx->chain      = chain;
 137         ctx->nla        = nla;
 138         ctx->portid     = NETLINK_CB(skb).portid;
 139         ctx->report     = nlmsg_report(nlh);
 140         ctx->flags      = nlh->nlmsg_flags;
 141         ctx->seq        = nlh->nlmsg_seq;
 142 }
 143
 144 static struct nft_trans *nft_trans_alloc_gfp(const struct nft_ctx *ctx,
 145                                              int msg_type, u32 size, gfp_t gfp)
 146 {
 147         struct nft_trans *trans;
 148
 149         trans = kzalloc(sizeof(struct nft_trans) + size, gfp);
 150         if (trans == NULL)
 151                 return NULL;
 152
 153         INIT_LIST_HEAD(&trans->list);
 154         INIT_LIST_HEAD(&trans->binding_list);
 155         trans->msg_type = msg_type;
 156         trans->ctx      = *ctx;
 157
 158         return trans;
 159 }
 160
 161 static struct nft_trans *nft_trans_alloc(const struct nft_ctx *ctx,
 162                                          int msg_type, u32 size)
 163 {
 164         return nft_trans_alloc_gfp(ctx, msg_type, size, GFP_KERNEL);
 165 }
 166
 167 static void nft_trans_list_del(struct nft_trans *trans)
 168 {
 169         list_del(&trans->list);
 170         list_del(&trans->binding_list);
 171 }
 172
 173 static void nft_trans_destroy(struct nft_trans *trans)
 174 {
 175         nft_trans_list_del(trans);
 176         kfree(trans);
 177 }
 178
 179 static void __nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set,
 180                                  bool bind)
 181 {
 182         struct nftables_pernet *nft_net;
 183         struct net *net = ctx->net;
 184         struct nft_trans *trans;
 185
 186         if (!nft_set_is_anonymous(set))
 187                 return;
 188
 189         nft_net = nft_pernet(net);
 190         list_for_each_entry_reverse(trans, &nft_net->commit_list, list) {
 191                 switch (trans->msg_type) {
 192                 case NFT_MSG_NEWSET:
 193                         if (nft_trans_set(trans) == set)
 194                                 nft_trans_set_bound(trans) = bind;
 195                         break;
 196                 case NFT_MSG_NEWSETELEM:
 197                         if (nft_trans_elem_set(trans) == set)
 198                                 nft_trans_elem_set_bound(trans) = bind;
 199                         break;
 200                 }
 201         }
 202 }
 203
 204 static void nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set)
 205 {
 206         return __nft_set_trans_bind(ctx, set, true);
 207 }
 208
 209 static void nft_set_trans_unbind(const struct nft_ctx *ctx, struct nft_set *set)
 210 {
 211         return __nft_set_trans_bind(ctx, set, false);
 212 }
 213
 214 static void __nft_chain_trans_bind(const struct nft_ctx *ctx,
 215                                    struct nft_chain *chain, bool bind)
 216 {
 217         struct nftables_pernet *nft_net;
 218         struct net *net = ctx->net;
 219         struct nft_trans *trans;
 220
 221         if (!nft_chain_binding(chain))
 222                 return;
 223
 224         nft_net = nft_pernet(net);
 225         list_for_each_entry_reverse(trans, &nft_net->commit_list, list) {
 226                 switch (trans->msg_type) {
 227                 case NFT_MSG_NEWCHAIN:
 228                         if (nft_trans_chain(trans) == chain)
 229                                 nft_trans_chain_bound(trans) = bind;
 230                         break;
 231                 case NFT_MSG_NEWRULE:
 232                         if (trans->ctx.chain == chain)
 233                                 nft_trans_rule_bound(trans) = bind;
 234                         break;
 235                 }
 236         }
 237 }
 238
 239 static void nft_chain_trans_bind(const struct nft_ctx *ctx,
 240                                  struct nft_chain *chain)
 241 {
 242         __nft_chain_trans_bind(ctx, chain, true);
 243 }
 244
 245 int nf_tables_bind_chain(const struct nft_ctx *ctx, struct nft_chain *chain)
 246 {
 247         if (!nft_chain_binding(chain))
 248                 return 0;
 249
 250         if (nft_chain_binding(ctx->chain))
 251                 return -EOPNOTSUPP;
 252
 253         if (chain->bound)
 254                 return -EBUSY;
 255
 256         chain->bound = true;
 257         chain->use++;
 258         nft_chain_trans_bind(ctx, chain);
 259
 260         return 0;
 261 }
 262
 263 void nf_tables_unbind_chain(const struct nft_ctx *ctx, struct nft_chain *chain)
 264 {
 265         __nft_chain_trans_bind(ctx, chain, false);
 266 }
 267
 268 static int nft_netdev_register_hooks(struct net *net,
 269                                      struct list_head *hook_list)
 270 {
 271         struct nft_hook *hook;
 272         int err, j;
 273
 274         j = 0;
 275         list_for_each_entry(hook, hook_list, list) {
 276                 err = nf_register_net_hook(net, &hook->ops);
 277                 if (err < 0)
 278                         goto err_register;
 279
 280                 j++;
 281         }
 282         return 0;
 283
 284 err_register:
 285         list_for_each_entry(hook, hook_list, list) {
 286                 if (j-- <= 0)
 287                         break;
 288
 289                 nf_unregister_net_hook(net, &hook->ops);
 290         }
 291         return err;
 292 }
 293
 294 static void nft_netdev_unregister_hooks(struct net *net,
 295                                         struct list_head *hook_list,
 296                                         bool release_netdev)
 297 {
 298         struct nft_hook *hook, *next;
 299
 300         list_for_each_entry_safe(hook, next, hook_list, list) {
 301                 nf_unregister_net_hook(net, &hook->ops);
 302                 if (release_netdev) {
 303                         list_del(&hook->list);
 304                         kfree_rcu(hook, rcu);
 305                 }
 306         }
 307 }
 308
 309 static int nf_tables_register_hook(struct net *net,
 310                                    const struct nft_table *table,
 311                                    struct nft_chain *chain)
 312 {
 313         struct nft_base_chain *basechain;
 314         const struct nf_hook_ops *ops;
 315
 316         if (table->flags & NFT_TABLE_F_DORMANT ||
 317             !nft_is_base_chain(chain))
 318                 return 0;
 319
 320         basechain = nft_base_chain(chain);
 321         ops = &basechain->ops;
 322
 323         if (basechain->type->ops_register)
 324                 return basechain->type->ops_register(net, ops);
 325
 326         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 327                 return nft_netdev_register_hooks(net, &basechain->hook_list);
 328
 329         return nf_register_net_hook(net, &basechain->ops);
 330 }
 331
 332 static void __nf_tables_unregister_hook(struct net *net,
 333                                         const struct nft_table *table,
 334                                         struct nft_chain *chain,
 335                                         bool release_netdev)
 336 {
 337         struct nft_base_chain *basechain;
 338         const struct nf_hook_ops *ops;
 339
 340         if (table->flags & NFT_TABLE_F_DORMANT ||
 341             !nft_is_base_chain(chain))
 342                 return;
 343         basechain = nft_base_chain(chain);
 344         ops = &basechain->ops;
 345
 346         if (basechain->type->ops_unregister)
 347                 return basechain->type->ops_unregister(net, ops);
 348
 349         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 350                 nft_netdev_unregister_hooks(net, &basechain->hook_list,
 351                                             release_netdev);
 352         else
 353                 nf_unregister_net_hook(net, &basechain->ops);
 354 }
 355
 356 static void nf_tables_unregister_hook(struct net *net,
 357                                       const struct nft_table *table,
 358                                       struct nft_chain *chain)
 359 {
 360         return __nf_tables_unregister_hook(net, table, chain, false);
 361 }
 362
 363 static void nft_trans_commit_list_add_tail(struct net *net, struct nft_trans *trans)
 364 {
 365         struct nftables_pernet *nft_net = nft_pernet(net);
 366
 367         switch (trans->msg_type) {
 368         case NFT_MSG_NEWSET:
 369                 if (!nft_trans_set_update(trans) &&
 370                     nft_set_is_anonymous(nft_trans_set(trans)))
 371                         list_add_tail(&trans->binding_list, &nft_net->binding_list);
 372                 break;
 373         case NFT_MSG_NEWCHAIN:
 374                 if (!nft_trans_chain_update(trans) &&
 375                     nft_chain_binding(nft_trans_chain(trans)))
 376                         list_add_tail(&trans->binding_list, &nft_net->binding_list);
 377                 break;
 378         }
 379
 380         list_add_tail(&trans->list, &nft_net->commit_list);
 381 }
 382
 383 static int nft_trans_table_add(struct nft_ctx *ctx, int msg_type)
 384 {
 385         struct nft_trans *trans;
 386
 387         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_table));
 388         if (trans == NULL)
 389                 return -ENOMEM;
 390
 391         if (msg_type == NFT_MSG_NEWTABLE)
 392                 nft_activate_next(ctx->net, ctx->table);
 393
 394         nft_trans_commit_list_add_tail(ctx->net, trans);
 395         return 0;
 396 }
 397
 398 static int nft_deltable(struct nft_ctx *ctx)
 399 {
 400         int err;
 401
 402         err = nft_trans_table_add(ctx, NFT_MSG_DELTABLE);
 403         if (err < 0)
 404                 return err;
 405
 406         nft_deactivate_next(ctx->net, ctx->table);
 407         return err;
 408 }
 409
 410 static struct nft_trans *nft_trans_chain_add(struct nft_ctx *ctx, int msg_type)
 411 {
 412         struct nft_trans *trans;
 413
 414         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_chain));
 415         if (trans == NULL)
 416                 return ERR_PTR(-ENOMEM);
 417
 418         if (msg_type == NFT_MSG_NEWCHAIN) {
 419                 nft_activate_next(ctx->net, ctx->chain);
 420
 421                 if (ctx->nla[NFTA_CHAIN_ID]) {
 422                         nft_trans_chain_id(trans) =
 423                                 ntohl(nla_get_be32(ctx->nla[NFTA_CHAIN_ID]));
 424                 }
 425         }
 426         nft_trans_chain(trans) = ctx->chain;
 427         nft_trans_commit_list_add_tail(ctx->net, trans);
 428
 429         return trans;
 430 }
 431
 432 static int nft_delchain(struct nft_ctx *ctx)
 433 {
 434         struct nft_trans *trans;
 435
 436         trans = nft_trans_chain_add(ctx, NFT_MSG_DELCHAIN);
 437         if (IS_ERR(trans))
 438                 return PTR_ERR(trans);
 439
 440         ctx->table->use--;
 441         nft_deactivate_next(ctx->net, ctx->chain);
 442
 443         return 0;
 444 }
 445
 446 void nft_rule_expr_activate(const struct nft_ctx *ctx, struct nft_rule *rule)
 447 {
 448         struct nft_expr *expr;
 449
 450         expr = nft_expr_first(rule);
 451         while (nft_expr_more(rule, expr)) {
 452                 if (expr->ops->activate)
 453                         expr->ops->activate(ctx, expr);
 454
 455                 expr = nft_expr_next(expr);
 456         }
 457 }
 458
 459 void nft_rule_expr_deactivate(const struct nft_ctx *ctx, struct nft_rule *rule,
 460                               enum nft_trans_phase phase)
 461 {
 462         struct nft_expr *expr;
 463
 464         expr = nft_expr_first(rule);
 465         while (nft_expr_more(rule, expr)) {
 466                 if (expr->ops->deactivate)
 467                         expr->ops->deactivate(ctx, expr, phase);
 468
 469                 expr = nft_expr_next(expr);
 470         }
 471 }
 472
 473 static int
 474 nf_tables_delrule_deactivate(struct nft_ctx *ctx, struct nft_rule *rule)
 475 {
 476         /* You cannot delete the same rule twice */
 477         if (nft_is_active_next(ctx->net, rule)) {
 478                 nft_deactivate_next(ctx->net, rule);
 479                 ctx->chain->use--;
 480                 return 0;
 481         }
 482         return -ENOENT;
 483 }
 484
 485 static struct nft_trans *nft_trans_rule_add(struct nft_ctx *ctx, int msg_type,
 486                                             struct nft_rule *rule)
 487 {
 488         struct nft_trans *trans;
 489
 490         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_rule));
 491         if (trans == NULL)
 492                 return NULL;
 493
 494         if (msg_type == NFT_MSG_NEWRULE && ctx->nla[NFTA_RULE_ID] != NULL) {
 495                 nft_trans_rule_id(trans) =
 496                         ntohl(nla_get_be32(ctx->nla[NFTA_RULE_ID]));
 497         }
 498         nft_trans_rule(trans) = rule;
 499         nft_trans_commit_list_add_tail(ctx->net, trans);
 500
 501         return trans;
 502 }
 503
 504 static int nft_delrule(struct nft_ctx *ctx, struct nft_rule *rule)
 505 {
 506         struct nft_flow_rule *flow;
 507         struct nft_trans *trans;
 508         int err;
 509
 510         trans = nft_trans_rule_add(ctx, NFT_MSG_DELRULE, rule);
 511         if (trans == NULL)
 512                 return -ENOMEM;
 513
 514         if (ctx->chain->flags & NFT_CHAIN_HW_OFFLOAD) {
 515                 flow = nft_flow_rule_create(ctx->net, rule);
 516                 if (IS_ERR(flow)) {
 517                         nft_trans_destroy(trans);
 518                         return PTR_ERR(flow);
 519                 }
 520
 521                 nft_trans_flow_rule(trans) = flow;
 522         }
 523
 524         err = nf_tables_delrule_deactivate(ctx, rule);
 525         if (err < 0) {
 526                 nft_trans_destroy(trans);
 527                 return err;
 528         }
 529         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_PREPARE);
 530
 531         return 0;
 532 }
 533
 534 static int nft_delrule_by_chain(struct nft_ctx *ctx)
 535 {
 536         struct nft_rule *rule;
 537         int err;
 538
 539         list_for_each_entry(rule, &ctx->chain->rules, list) {
 540                 if (!nft_is_active_next(ctx->net, rule))
 541                         continue;
 542
 543                 err = nft_delrule(ctx, rule);
 544                 if (err < 0)
 545                         return err;
 546         }
 547         return 0;
 548 }
 549
 550 static int __nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
 551                                struct nft_set *set,
 552                                const struct nft_set_desc *desc)
 553 {
 554         struct nft_trans *trans;
 555
 556         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_set));
 557         if (trans == NULL)
 558                 return -ENOMEM;
 559
 560         if (msg_type == NFT_MSG_NEWSET && ctx->nla[NFTA_SET_ID] && !desc) {
 561                 nft_trans_set_id(trans) =
 562                         ntohl(nla_get_be32(ctx->nla[NFTA_SET_ID]));
 563                 nft_activate_next(ctx->net, set);
 564         }
 565         nft_trans_set(trans) = set;
 566         if (desc) {
 567                 nft_trans_set_update(trans) = true;
 568                 nft_trans_set_gc_int(trans) = desc->gc_int;
 569                 nft_trans_set_timeout(trans) = desc->timeout;
 570         }
 571         nft_trans_commit_list_add_tail(ctx->net, trans);
 572
 573         return 0;
 574 }
 575
 576 static int nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
 577                              struct nft_set *set)
 578 {
 579         return __nft_trans_set_add(ctx, msg_type, set, NULL);
 580 }
 581
 582 static void nft_setelem_data_deactivate(const struct net *net,
 583                                         const struct nft_set *set,
 584                                         struct nft_set_elem *elem);
 585
 586 static int nft_mapelem_deactivate(const struct nft_ctx *ctx,
 587                                   struct nft_set *set,
 588                                   const struct nft_set_iter *iter,
 589                                   struct nft_set_elem *elem)
 590 {
 591         nft_setelem_data_deactivate(ctx->net, set, elem);
 592
 593         return 0;
 594 }
 595
 596 struct nft_set_elem_catchall {
 597         struct list_head        list;
 598         struct rcu_head         rcu;
 599         void                    *elem;
 600 };
 601
 602 static void nft_map_catchall_deactivate(const struct nft_ctx *ctx,
 603                                         struct nft_set *set)
 604 {
 605         u8 genmask = nft_genmask_next(ctx->net);
 606         struct nft_set_elem_catchall *catchall;
 607         struct nft_set_elem elem;
 608         struct nft_set_ext *ext;
 609
 610         list_for_each_entry(catchall, &set->catchall_list, list) {
 611                 ext = nft_set_elem_ext(set, catchall->elem);
 612                 if (!nft_set_elem_active(ext, genmask))
 613                         continue;
 614
 615                 elem.priv = catchall->elem;
 616                 nft_setelem_data_deactivate(ctx->net, set, &elem);
 617                 break;
 618         }
 619 }
 620
 621 static void nft_map_deactivate(const struct nft_ctx *ctx, struct nft_set *set)
 622 {
 623         struct nft_set_iter iter = {
 624                 .genmask        = nft_genmask_next(ctx->net),
 625                 .fn             = nft_mapelem_deactivate,
 626         };
 627
 628         set->ops->walk(ctx, set, &iter);
 629         WARN_ON_ONCE(iter.err);
 630
 631         nft_map_catchall_deactivate(ctx, set);
 632 }
 633
 634 static int nft_delset(const struct nft_ctx *ctx, struct nft_set *set)
 635 {
 636         int err;
 637
 638         err = nft_trans_set_add(ctx, NFT_MSG_DELSET, set);
 639         if (err < 0)
 640                 return err;
 641
 642         if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
 643                 nft_map_deactivate(ctx, set);
 644
 645         nft_deactivate_next(ctx->net, set);
 646         ctx->table->use--;
 647
 648         return err;
 649 }
 650
 651 static int nft_trans_obj_add(struct nft_ctx *ctx, int msg_type,
 652                              struct nft_object *obj)
 653 {
 654         struct nft_trans *trans;
 655
 656         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_obj));
 657         if (trans == NULL)
 658                 return -ENOMEM;
 659
 660         if (msg_type == NFT_MSG_NEWOBJ)
 661                 nft_activate_next(ctx->net, obj);
 662
 663         nft_trans_obj(trans) = obj;
 664         nft_trans_commit_list_add_tail(ctx->net, trans);
 665
 666         return 0;
 667 }
 668
 669 static int nft_delobj(struct nft_ctx *ctx, struct nft_object *obj)
 670 {
 671         int err;
 672
 673         err = nft_trans_obj_add(ctx, NFT_MSG_DELOBJ, obj);
 674         if (err < 0)
 675                 return err;
 676
 677         nft_deactivate_next(ctx->net, obj);
 678         ctx->table->use--;
 679
 680         return err;
 681 }
 682
 683 static int nft_trans_flowtable_add(struct nft_ctx *ctx, int msg_type,
 684                                    struct nft_flowtable *flowtable)
 685 {
 686         struct nft_trans *trans;
 687
 688         trans = nft_trans_alloc(ctx, msg_type,
 689                                 sizeof(struct nft_trans_flowtable));
 690         if (trans == NULL)
 691                 return -ENOMEM;
 692
 693         if (msg_type == NFT_MSG_NEWFLOWTABLE)
 694                 nft_activate_next(ctx->net, flowtable);
 695
 696         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
 697         nft_trans_flowtable(trans) = flowtable;
 698         nft_trans_commit_list_add_tail(ctx->net, trans);
 699
 700         return 0;
 701 }
 702
 703 static int nft_delflowtable(struct nft_ctx *ctx,
 704                             struct nft_flowtable *flowtable)
 705 {
 706         int err;
 707
 708         err = nft_trans_flowtable_add(ctx, NFT_MSG_DELFLOWTABLE, flowtable);
 709         if (err < 0)
 710                 return err;
 711
 712         nft_deactivate_next(ctx->net, flowtable);
 713         ctx->table->use--;
 714
 715         return err;
 716 }
 717
 718 static void __nft_reg_track_clobber(struct nft_regs_track *track, u8 dreg)
 719 {
 720         int i;
 721
 722         for (i = track->regs[dreg].num_reg; i > 0; i--)
 723                 __nft_reg_track_cancel(track, dreg - i);
 724 }
 725
 726 static void __nft_reg_track_update(struct nft_regs_track *track,
 727                                    const struct nft_expr *expr,
 728                                    u8 dreg, u8 num_reg)
 729 {
 730         track->regs[dreg].selector = expr;
 731         track->regs[dreg].bitwise = NULL;
 732         track->regs[dreg].num_reg = num_reg;
 733 }
 734
 735 void nft_reg_track_update(struct nft_regs_track *track,
 736                           const struct nft_expr *expr, u8 dreg, u8 len)
 737 {
 738         unsigned int regcount;
 739         int i;
 740
 741         __nft_reg_track_clobber(track, dreg);
 742
 743         regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
 744         for (i = 0; i < regcount; i++, dreg++)
 745                 __nft_reg_track_update(track, expr, dreg, i);
 746 }
 747 EXPORT_SYMBOL_GPL(nft_reg_track_update);
 748
 749 void nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg, u8 len)
 750 {
 751         unsigned int regcount;
 752         int i;
 753
 754         __nft_reg_track_clobber(track, dreg);
 755
 756         regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
 757         for (i = 0; i < regcount; i++, dreg++)
 758                 __nft_reg_track_cancel(track, dreg);
 759 }
 760 EXPORT_SYMBOL_GPL(nft_reg_track_cancel);
 761
 762 void __nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg)
 763 {
 764         track->regs[dreg].selector = NULL;
 765         track->regs[dreg].bitwise = NULL;
 766         track->regs[dreg].num_reg = 0;
 767 }
 768 EXPORT_SYMBOL_GPL(__nft_reg_track_cancel);
 769
 770 /*
 771  * Tables
 772  */
 773
 774 static struct nft_table *nft_table_lookup(const struct net *net,
 775                                           const struct nlattr *nla,
 776                                           u8 family, u8 genmask, u32 nlpid)
 777 {
 778         struct nftables_pernet *nft_net;
 779         struct nft_table *table;
 780
 781         if (nla == NULL)
 782                 return ERR_PTR(-EINVAL);
 783
 784         nft_net = nft_pernet(net);
 785         list_for_each_entry_rcu(table, &nft_net->tables, list,
 786                                 lockdep_is_held(&nft_net->commit_mutex)) {
 787                 if (!nla_strcmp(nla, table->name) &&
 788                     table->family == family &&
 789                     nft_active_genmask(table, genmask)) {
 790                         if (nft_table_has_owner(table) &&
 791                             nlpid && table->nlpid != nlpid)
 792                                 return ERR_PTR(-EPERM);
 793
 794                         return table;
 795                 }
 796         }
 797
 798         return ERR_PTR(-ENOENT);
 799 }
 800
 801 static struct nft_table *nft_table_lookup_byhandle(const struct net *net,
 802                                                    const struct nlattr *nla,
 803                                                    u8 genmask, u32 nlpid)
 804 {
 805         struct nftables_pernet *nft_net;
 806         struct nft_table *table;
 807
 808         nft_net = nft_pernet(net);
 809         list_for_each_entry(table, &nft_net->tables, list) {
 810                 if (be64_to_cpu(nla_get_be64(nla)) == table->handle &&
 811                     nft_active_genmask(table, genmask)) {
 812                         if (nft_table_has_owner(table) &&
 813                             nlpid && table->nlpid != nlpid)
 814                                 return ERR_PTR(-EPERM);
 815
 816                         return table;
 817                 }
 818         }
 819
 820         return ERR_PTR(-ENOENT);
 821 }
 822
 823 static inline u64 nf_tables_alloc_handle(struct nft_table *table)
 824 {
 825         return ++table->hgenerator;
 826 }
 827
 828 static const struct nft_chain_type *chain_type[NFPROTO_NUMPROTO][NFT_CHAIN_T_MAX];
 829
 830 static const struct nft_chain_type *
 831 __nft_chain_type_get(u8 family, enum nft_chain_types type)
 832 {
 833         if (family >= NFPROTO_NUMPROTO ||
 834             type >= NFT_CHAIN_T_MAX)
 835                 return NULL;
 836
 837         return chain_type[family][type];
 838 }
 839
 840 static const struct nft_chain_type *
 841 __nf_tables_chain_type_lookup(const struct nlattr *nla, u8 family)
 842 {
 843         const struct nft_chain_type *type;
 844         int i;
 845
 846         for (i = 0; i < NFT_CHAIN_T_MAX; i++) {
 847                 type = __nft_chain_type_get(family, i);
 848                 if (!type)
 849                         continue;
 850                 if (!nla_strcmp(nla, type->name))
 851                         return type;
 852         }
 853         return NULL;
 854 }
 855
 856 struct nft_module_request {
 857         struct list_head        list;
 858         char                    module[MODULE_NAME_LEN];
 859         bool                    done;
 860 };
 861
 862 #ifdef CONFIG_MODULES
 863 __printf(2, 3) int nft_request_module(struct net *net, const char *fmt,
 864                                       ...)
 865 {
 866         char module_name[MODULE_NAME_LEN];
 867         struct nftables_pernet *nft_net;
 868         struct nft_module_request *req;
 869         va_list args;
 870         int ret;
 871
 872         va_start(args, fmt);
 873         ret = vsnprintf(module_name, MODULE_NAME_LEN, fmt, args);
 874         va_end(args);
 875         if (ret >= MODULE_NAME_LEN)
 876                 return 0;
 877
 878         nft_net = nft_pernet(net);
 879         list_for_each_entry(req, &nft_net->module_list, list) {
 880                 if (!strcmp(req->module, module_name)) {
 881                         if (req->done)
 882                                 return 0;
 883
 884                         /* A request to load this module already exists. */
 885                         return -EAGAIN;
 886                 }
 887         }
 888
 889         req = kmalloc(sizeof(*req), GFP_KERNEL);
 890         if (!req)
 891                 return -ENOMEM;
 892
 893         req->done = false;
 894         strscpy(req->module, module_name, MODULE_NAME_LEN);
 895         list_add_tail(&req->list, &nft_net->module_list);
 896
 897         return -EAGAIN;
 898 }
 899 EXPORT_SYMBOL_GPL(nft_request_module);
 900 #endif
 901
 902 static void lockdep_nfnl_nft_mutex_not_held(void)
 903 {
 904 #ifdef CONFIG_PROVE_LOCKING
 905         if (debug_locks)
 906                 WARN_ON_ONCE(lockdep_nfnl_is_held(NFNL_SUBSYS_NFTABLES));
 907 #endif
 908 }
 909
 910 static const struct nft_chain_type *
 911 nf_tables_chain_type_lookup(struct net *net, const struct nlattr *nla,
 912                             u8 family, bool autoload)
 913 {
 914         const struct nft_chain_type *type;
 915
 916         type = __nf_tables_chain_type_lookup(nla, family);
 917         if (type != NULL)
 918                 return type;
 919
 920         lockdep_nfnl_nft_mutex_not_held();
 921 #ifdef CONFIG_MODULES
 922         if (autoload) {
 923                 if (nft_request_module(net, "nft-chain-%u-%.*s", family,
 924                                        nla_len(nla),
 925                                        (const char *)nla_data(nla)) == -EAGAIN)
 926                         return ERR_PTR(-EAGAIN);
 927         }
 928 #endif
 929         return ERR_PTR(-ENOENT);
 930 }
 931
 932 static __be16 nft_base_seq(const struct net *net)
 933 {
 934         struct nftables_pernet *nft_net = nft_pernet(net);
 935
 936         return htons(nft_net->base_seq & 0xffff);
 937 }
 938
 939 static const struct nla_policy nft_table_policy[NFTA_TABLE_MAX + 1] = {
 940         [NFTA_TABLE_NAME]       = { .type = NLA_STRING,
 941                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
 942         [NFTA_TABLE_FLAGS]      = { .type = NLA_U32 },
 943         [NFTA_TABLE_HANDLE]     = { .type = NLA_U64 },
 944         [NFTA_TABLE_USERDATA]   = { .type = NLA_BINARY,
 945                                     .len = NFT_USERDATA_MAXLEN }
 946 };
 947
 948 static int nf_tables_fill_table_info(struct sk_buff *skb, struct net *net,
 949                                      u32 portid, u32 seq, int event, u32 flags,
 950                                      int family, const struct nft_table *table)
 951 {
 952         struct nlmsghdr *nlh;
 953
 954         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
 955         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
 956                            NFNETLINK_V0, nft_base_seq(net));
 957         if (!nlh)
 958                 goto nla_put_failure;
 959
 960         if (nla_put_string(skb, NFTA_TABLE_NAME, table->name) ||
 961             nla_put_be32(skb, NFTA_TABLE_USE, htonl(table->use)) ||
 962             nla_put_be64(skb, NFTA_TABLE_HANDLE, cpu_to_be64(table->handle),
 963                          NFTA_TABLE_PAD))
 964                 goto nla_put_failure;
 965
 966         if (event == NFT_MSG_DELTABLE) {
 967                 nlmsg_end(skb, nlh);
 968                 return 0;
 969         }
 970
 971         if (nla_put_be32(skb, NFTA_TABLE_FLAGS,
 972                          htonl(table->flags & NFT_TABLE_F_MASK)))
 973                 goto nla_put_failure;
 974
 975         if (nft_table_has_owner(table) &&
 976             nla_put_be32(skb, NFTA_TABLE_OWNER, htonl(table->nlpid)))
 977                 goto nla_put_failure;
 978
 979         if (table->udata) {
 980                 if (nla_put(skb, NFTA_TABLE_USERDATA, table->udlen, table->udata))
 981                         goto nla_put_failure;
 982         }
 983
 984         nlmsg_end(skb, nlh);
 985         return 0;
 986
 987 nla_put_failure:
 988         nlmsg_trim(skb, nlh);
 989         return -1;
 990 }
 991
 992 struct nftnl_skb_parms {
 993         bool report;
 994 };
 995 #define NFT_CB(skb)     (*(struct nftnl_skb_parms*)&((skb)->cb))
 996
 997 static void nft_notify_enqueue(struct sk_buff *skb, bool report,
 998                                struct list_head *notify_list)
 999 {
1000         NFT_CB(skb).report = report;
1001         list_add_tail(&skb->list, notify_list);
1002 }
1003
1004 static void nf_tables_table_notify(const struct nft_ctx *ctx, int event)
1005 {
1006         struct nftables_pernet *nft_net;
1007         struct sk_buff *skb;
1008         u16 flags = 0;
1009         int err;
1010
1011         if (!ctx->report &&
1012             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
1013                 return;
1014
1015         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
1016         if (skb == NULL)
1017                 goto err;
1018
1019         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
1020                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
1021
1022         err = nf_tables_fill_table_info(skb, ctx->net, ctx->portid, ctx->seq,
1023                                         event, flags, ctx->family, ctx->table);
1024         if (err < 0) {
1025                 kfree_skb(skb);
1026                 goto err;
1027         }
1028
1029         nft_net = nft_pernet(ctx->net);
1030         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
1031         return;
1032 err:
1033         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
1034 }
1035
1036 static int nf_tables_dump_tables(struct sk_buff *skb,
1037                                  struct netlink_callback *cb)
1038 {
1039         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
1040         struct nftables_pernet *nft_net;
1041         const struct nft_table *table;
1042         unsigned int idx = 0, s_idx = cb->args[0];
1043         struct net *net = sock_net(skb->sk);
1044         int family = nfmsg->nfgen_family;
1045
1046         rcu_read_lock();
1047         nft_net = nft_pernet(net);
1048         cb->seq = READ_ONCE(nft_net->base_seq);
1049
1050         list_for_each_entry_rcu(table, &nft_net->tables, list) {
1051                 if (family != NFPROTO_UNSPEC && family != table->family)
1052                         continue;
1053
1054                 if (idx < s_idx)
1055                         goto cont;
1056                 if (idx > s_idx)
1057                         memset(&cb->args[1], 0,
1058                                sizeof(cb->args) - sizeof(cb->args[0]));
1059                 if (!nft_is_active(net, table))
1060                         continue;
1061                 if (nf_tables_fill_table_info(skb, net,
1062                                               NETLINK_CB(cb->skb).portid,
1063                                               cb->nlh->nlmsg_seq,
1064                                               NFT_MSG_NEWTABLE, NLM_F_MULTI,
1065                                               table->family, table) < 0)
1066                         goto done;
1067
1068                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
1069 cont:
1070                 idx++;
1071         }
1072 done:
1073         rcu_read_unlock();
1074         cb->args[0] = idx;
1075         return skb->len;
1076 }
1077
1078 static int nft_netlink_dump_start_rcu(struct sock *nlsk, struct sk_buff *skb,
1079                                       const struct nlmsghdr *nlh,
1080                                       struct netlink_dump_control *c)
1081 {
1082         int err;
1083
1084         if (!try_module_get(THIS_MODULE))
1085                 return -EINVAL;
1086
1087         rcu_read_unlock();
1088         err = netlink_dump_start(nlsk, skb, nlh, c);
1089         rcu_read_lock();
1090         module_put(THIS_MODULE);
1091
1092         return err;
1093 }
1094
1095 /* called with rcu_read_lock held */
1096 static int nf_tables_gettable(struct sk_buff *skb, const struct nfnl_info *info,
1097                               const struct nlattr * const nla[])
1098 {
1099         struct netlink_ext_ack *extack = info->extack;
1100         u8 genmask = nft_genmask_cur(info->net);
1101         u8 family = info->nfmsg->nfgen_family;
1102         const struct nft_table *table;
1103         struct net *net = info->net;
1104         struct sk_buff *skb2;
1105         int err;
1106
1107         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
1108                 struct netlink_dump_control c = {
1109                         .dump = nf_tables_dump_tables,
1110                         .module = THIS_MODULE,
1111                 };
1112
1113                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
1114         }
1115
1116         table = nft_table_lookup(net, nla[NFTA_TABLE_NAME], family, genmask, 0);
1117         if (IS_ERR(table)) {
1118                 NL_SET_BAD_ATTR(extack, nla[NFTA_TABLE_NAME]);
1119                 return PTR_ERR(table);
1120         }
1121
1122         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
1123         if (!skb2)
1124                 return -ENOMEM;
1125
1126         err = nf_tables_fill_table_info(skb2, net, NETLINK_CB(skb).portid,
1127                                         info->nlh->nlmsg_seq, NFT_MSG_NEWTABLE,
1128                                         0, family, table);
1129         if (err < 0)
1130                 goto err_fill_table_info;
1131
1132         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
1133
1134 err_fill_table_info:
1135         kfree_skb(skb2);
1136         return err;
1137 }
1138
1139 static void nft_table_disable(struct net *net, struct nft_table *table, u32 cnt)
1140 {
1141         struct nft_chain *chain;
1142         u32 i = 0;
1143
1144         list_for_each_entry(chain, &table->chains, list) {
1145                 if (!nft_is_active_next(net, chain))
1146                         continue;
1147                 if (!nft_is_base_chain(chain))
1148                         continue;
1149
1150                 if (cnt && i++ == cnt)
1151                         break;
1152
1153                 nf_tables_unregister_hook(net, table, chain);
1154         }
1155 }
1156
1157 static int nf_tables_table_enable(struct net *net, struct nft_table *table)
1158 {
1159         struct nft_chain *chain;
1160         int err, i = 0;
1161
1162         list_for_each_entry(chain, &table->chains, list) {
1163                 if (!nft_is_active_next(net, chain))
1164                         continue;
1165                 if (!nft_is_base_chain(chain))
1166                         continue;
1167
1168                 err = nf_tables_register_hook(net, table, chain);
1169                 if (err < 0)
1170                         goto err_register_hooks;
1171
1172                 i++;
1173         }
1174         return 0;
1175
1176 err_register_hooks:
1177         if (i)
1178                 nft_table_disable(net, table, i);
1179         return err;
1180 }
1181
1182 static void nf_tables_table_disable(struct net *net, struct nft_table *table)
1183 {
1184         table->flags &= ~NFT_TABLE_F_DORMANT;
1185         nft_table_disable(net, table, 0);
1186         table->flags |= NFT_TABLE_F_DORMANT;
1187 }
1188
1189 #define __NFT_TABLE_F_INTERNAL          (NFT_TABLE_F_MASK + 1)
1190 #define __NFT_TABLE_F_WAS_DORMANT       (__NFT_TABLE_F_INTERNAL << 0)
1191 #define __NFT_TABLE_F_WAS_AWAKEN        (__NFT_TABLE_F_INTERNAL << 1)
1192 #define __NFT_TABLE_F_UPDATE            (__NFT_TABLE_F_WAS_DORMANT | \
1193                                          __NFT_TABLE_F_WAS_AWAKEN)
1194
1195 static int nf_tables_updtable(struct nft_ctx *ctx)
1196 {
1197         struct nft_trans *trans;
1198         u32 flags;
1199         int ret;
1200
1201         if (!ctx->nla[NFTA_TABLE_FLAGS])
1202                 return 0;
1203
1204         flags = ntohl(nla_get_be32(ctx->nla[NFTA_TABLE_FLAGS]));
1205         if (flags & ~NFT_TABLE_F_MASK)
1206                 return -EOPNOTSUPP;
1207
1208         if (flags == ctx->table->flags)
1209                 return 0;
1210
1211         if ((nft_table_has_owner(ctx->table) &&
1212              !(flags & NFT_TABLE_F_OWNER)) ||
1213             (!nft_table_has_owner(ctx->table) &&
1214              flags & NFT_TABLE_F_OWNER))
1215                 return -EOPNOTSUPP;
1216
1217         trans = nft_trans_alloc(ctx, NFT_MSG_NEWTABLE,
1218                                 sizeof(struct nft_trans_table));
1219         if (trans == NULL)
1220                 return -ENOMEM;
1221
1222         if ((flags & NFT_TABLE_F_DORMANT) &&
1223             !(ctx->table->flags & NFT_TABLE_F_DORMANT)) {
1224                 ctx->table->flags |= NFT_TABLE_F_DORMANT;
1225                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE))
1226                         ctx->table->flags |= __NFT_TABLE_F_WAS_AWAKEN;
1227         } else if (!(flags & NFT_TABLE_F_DORMANT) &&
1228                    ctx->table->flags & NFT_TABLE_F_DORMANT) {
1229                 ctx->table->flags &= ~NFT_TABLE_F_DORMANT;
1230                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE)) {
1231                         ret = nf_tables_table_enable(ctx->net, ctx->table);
1232                         if (ret < 0)
1233                                 goto err_register_hooks;
1234
1235                         ctx->table->flags |= __NFT_TABLE_F_WAS_DORMANT;
1236                 }
1237         }
1238
1239         nft_trans_table_update(trans) = true;
1240         nft_trans_commit_list_add_tail(ctx->net, trans);
1241
1242         return 0;
1243
1244 err_register_hooks:
1245         nft_trans_destroy(trans);
1246         return ret;
1247 }
1248
1249 static u32 nft_chain_hash(const void *data, u32 len, u32 seed)
1250 {
1251         const char *name = data;
1252
1253         return jhash(name, strlen(name), seed);
1254 }
1255
1256 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed)
1257 {
1258         const struct nft_chain *chain = data;
1259
1260         return nft_chain_hash(chain->name, 0, seed);
1261 }
1262
1263 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *arg,
1264                               const void *ptr)
1265 {
1266         const struct nft_chain *chain = ptr;
1267         const char *name = arg->key;
1268
1269         return strcmp(chain->name, name);
1270 }
1271
1272 static u32 nft_objname_hash(const void *data, u32 len, u32 seed)
1273 {
1274         const struct nft_object_hash_key *k = data;
1275
1276         seed ^= hash_ptr(k->table, 32);
1277
1278         return jhash(k->name, strlen(k->name), seed);
1279 }
1280
1281 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed)
1282 {
1283         const struct nft_object *obj = data;
1284
1285         return nft_objname_hash(&obj->key, 0, seed);
1286 }
1287
1288 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *arg,
1289                                 const void *ptr)
1290 {
1291         const struct nft_object_hash_key *k = arg->key;
1292         const struct nft_object *obj = ptr;
1293
1294         if (obj->key.table != k->table)
1295                 return -1;
1296
1297         return strcmp(obj->key.name, k->name);
1298 }
1299
1300 static bool nft_supported_family(u8 family)
1301 {
1302         return false
1303 #ifdef CONFIG_NF_TABLES_INET
1304                 || family == NFPROTO_INET
1305 #endif
1306 #ifdef CONFIG_NF_TABLES_IPV4
1307                 || family == NFPROTO_IPV4
1308 #endif
1309 #ifdef CONFIG_NF_TABLES_ARP
1310                 || family == NFPROTO_ARP
1311 #endif
1312 #ifdef CONFIG_NF_TABLES_NETDEV
1313                 || family == NFPROTO_NETDEV
1314 #endif
1315 #if IS_ENABLED(CONFIG_NF_TABLES_BRIDGE)
1316                 || family == NFPROTO_BRIDGE
1317 #endif
1318 #ifdef CONFIG_NF_TABLES_IPV6
1319                 || family == NFPROTO_IPV6
1320 #endif
1321                 ;
1322 }
1323
1324 static int nf_tables_newtable(struct sk_buff *skb, const struct nfnl_info *info,
1325                               const struct nlattr * const nla[])
1326 {
1327         struct nftables_pernet *nft_net = nft_pernet(info->net);
1328         struct netlink_ext_ack *extack = info->extack;
1329         u8 genmask = nft_genmask_next(info->net);
1330         u8 family = info->nfmsg->nfgen_family;
1331         struct net *net = info->net;
1332         const struct nlattr *attr;
1333         struct nft_table *table;
1334         struct nft_ctx ctx;
1335         u32 flags = 0;
1336         int err;
1337
1338         if (!nft_supported_family(family))
1339                 return -EOPNOTSUPP;
1340
1341         lockdep_assert_held(&nft_net->commit_mutex);
1342         attr = nla[NFTA_TABLE_NAME];
1343         table = nft_table_lookup(net, attr, family, genmask,
1344                                  NETLINK_CB(skb).portid);
1345         if (IS_ERR(table)) {
1346                 if (PTR_ERR(table) != -ENOENT)
1347                         return PTR_ERR(table);
1348         } else {
1349                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
1350                         NL_SET_BAD_ATTR(extack, attr);
1351                         return -EEXIST;
1352                 }
1353                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
1354                         return -EOPNOTSUPP;
1355
1356                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1357
1358                 return nf_tables_updtable(&ctx);
1359         }
1360
1361         if (nla[NFTA_TABLE_FLAGS]) {
1362                 flags = ntohl(nla_get_be32(nla[NFTA_TABLE_FLAGS]));
1363                 if (flags & ~NFT_TABLE_F_MASK)
1364                         return -EOPNOTSUPP;
1365         }
1366
1367         err = -ENOMEM;
1368         table = kzalloc(sizeof(*table), GFP_KERNEL_ACCOUNT);
1369         if (table == NULL)
1370                 goto err_kzalloc;
1371
1372         table->validate_state = NFT_VALIDATE_SKIP;
1373         table->name = nla_strdup(attr, GFP_KERNEL_ACCOUNT);
1374         if (table->name == NULL)
1375                 goto err_strdup;
1376
1377         if (nla[NFTA_TABLE_USERDATA]) {
1378                 table->udata = nla_memdup(nla[NFTA_TABLE_USERDATA], GFP_KERNEL_ACCOUNT);
1379                 if (table->udata == NULL)
1380                         goto err_table_udata;
1381
1382                 table->udlen = nla_len(nla[NFTA_TABLE_USERDATA]);
1383         }
1384
1385         err = rhltable_init(&table->chains_ht, &nft_chain_ht_params);
1386         if (err)
1387                 goto err_chain_ht;
1388
1389         INIT_LIST_HEAD(&table->chains);
1390         INIT_LIST_HEAD(&table->sets);
1391         INIT_LIST_HEAD(&table->objects);
1392         INIT_LIST_HEAD(&table->flowtables);
1393         table->family = family;
1394         table->flags = flags;
1395         table->handle = ++nft_net->table_handle;
1396         if (table->flags & NFT_TABLE_F_OWNER)
1397                 table->nlpid = NETLINK_CB(skb).portid;
1398
1399         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1400         err = nft_trans_table_add(&ctx, NFT_MSG_NEWTABLE);
1401         if (err < 0)
1402                 goto err_trans;
1403
1404         list_add_tail_rcu(&table->list, &nft_net->tables);
1405         return 0;
1406 err_trans:
1407         rhltable_destroy(&table->chains_ht);
1408 err_chain_ht:
1409         kfree(table->udata);
1410 err_table_udata:
1411         kfree(table->name);
1412 err_strdup:
1413         kfree(table);
1414 err_kzalloc:
1415         return err;
1416 }
1417
1418 static int nft_flush_table(struct nft_ctx *ctx)
1419 {
1420         struct nft_flowtable *flowtable, *nft;
1421         struct nft_chain *chain, *nc;
1422         struct nft_object *obj, *ne;
1423         struct nft_set *set, *ns;
1424         int err;
1425
1426         list_for_each_entry(chain, &ctx->table->chains, list) {
1427                 if (!nft_is_active_next(ctx->net, chain))
1428                         continue;
1429
1430                 if (nft_chain_is_bound(chain))
1431                         continue;
1432
1433                 ctx->chain = chain;
1434
1435                 err = nft_delrule_by_chain(ctx);
1436                 if (err < 0)
1437                         goto out;
1438         }
1439
1440         list_for_each_entry_safe(set, ns, &ctx->table->sets, list) {
1441                 if (!nft_is_active_next(ctx->net, set))
1442                         continue;
1443
1444                 if (nft_set_is_anonymous(set) &&
1445                     !list_empty(&set->bindings))
1446                         continue;
1447
1448                 err = nft_delset(ctx, set);
1449                 if (err < 0)
1450                         goto out;
1451         }
1452
1453         list_for_each_entry_safe(flowtable, nft, &ctx->table->flowtables, list) {
1454                 if (!nft_is_active_next(ctx->net, flowtable))
1455                         continue;
1456
1457                 err = nft_delflowtable(ctx, flowtable);
1458                 if (err < 0)
1459                         goto out;
1460         }
1461
1462         list_for_each_entry_safe(obj, ne, &ctx->table->objects, list) {
1463                 if (!nft_is_active_next(ctx->net, obj))
1464                         continue;
1465
1466                 err = nft_delobj(ctx, obj);
1467                 if (err < 0)
1468                         goto out;
1469         }
1470
1471         list_for_each_entry_safe(chain, nc, &ctx->table->chains, list) {
1472                 if (!nft_is_active_next(ctx->net, chain))
1473                         continue;
1474
1475                 if (nft_chain_is_bound(chain))
1476                         continue;
1477
1478                 ctx->chain = chain;
1479
1480                 err = nft_delchain(ctx);
1481                 if (err < 0)
1482                         goto out;
1483         }
1484
1485         err = nft_deltable(ctx);
1486 out:
1487         return err;
1488 }
1489
1490 static int nft_flush(struct nft_ctx *ctx, int family)
1491 {
1492         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
1493         const struct nlattr * const *nla = ctx->nla;
1494         struct nft_table *table, *nt;
1495         int err = 0;
1496
1497         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
1498                 if (family != AF_UNSPEC && table->family != family)
1499                         continue;
1500
1501                 ctx->family = table->family;
1502
1503                 if (!nft_is_active_next(ctx->net, table))
1504                         continue;
1505
1506                 if (nft_table_has_owner(table) && table->nlpid != ctx->portid)
1507                         continue;
1508
1509                 if (nla[NFTA_TABLE_NAME] &&
1510                     nla_strcmp(nla[NFTA_TABLE_NAME], table->name) != 0)
1511                         continue;
1512
1513                 ctx->table = table;
1514
1515                 err = nft_flush_table(ctx);
1516                 if (err < 0)
1517                         goto out;
1518         }
1519 out:
1520         return err;
1521 }
1522
1523 static int nf_tables_deltable(struct sk_buff *skb, const struct nfnl_info *info,
1524                               const struct nlattr * const nla[])
1525 {
1526         struct netlink_ext_ack *extack = info->extack;
1527         u8 genmask = nft_genmask_next(info->net);
1528         u8 family = info->nfmsg->nfgen_family;
1529         struct net *net = info->net;
1530         const struct nlattr *attr;
1531         struct nft_table *table;
1532         struct nft_ctx ctx;
1533
1534         nft_ctx_init(&ctx, net, skb, info->nlh, 0, NULL, NULL, nla);
1535         if (family == AF_UNSPEC ||
1536             (!nla[NFTA_TABLE_NAME] && !nla[NFTA_TABLE_HANDLE]))
1537                 return nft_flush(&ctx, family);
1538
1539         if (nla[NFTA_TABLE_HANDLE]) {
1540                 attr = nla[NFTA_TABLE_HANDLE];
1541                 table = nft_table_lookup_byhandle(net, attr, genmask,
1542                                                   NETLINK_CB(skb).portid);
1543         } else {
1544                 attr = nla[NFTA_TABLE_NAME];
1545                 table = nft_table_lookup(net, attr, family, genmask,
1546                                          NETLINK_CB(skb).portid);
1547         }
1548
1549         if (IS_ERR(table)) {
1550                 if (PTR_ERR(table) == -ENOENT &&
1551                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYTABLE)
1552                         return 0;
1553
1554                 NL_SET_BAD_ATTR(extack, attr);
1555                 return PTR_ERR(table);
1556         }
1557
1558         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
1559             table->use > 0)
1560                 return -EBUSY;
1561
1562         ctx.family = family;
1563         ctx.table = table;
1564
1565         return nft_flush_table(&ctx);
1566 }
1567
1568 static void nf_tables_table_destroy(struct nft_ctx *ctx)
1569 {
1570         if (WARN_ON(ctx->table->use > 0))
1571                 return;
1572
1573         rhltable_destroy(&ctx->table->chains_ht);
1574         kfree(ctx->table->name);
1575         kfree(ctx->table->udata);
1576         kfree(ctx->table);
1577 }
1578
1579 void nft_register_chain_type(const struct nft_chain_type *ctype)
1580 {
1581         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1582         if (WARN_ON(__nft_chain_type_get(ctype->family, ctype->type))) {
1583                 nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1584                 return;
1585         }
1586         chain_type[ctype->family][ctype->type] = ctype;
1587         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1588 }
1589 EXPORT_SYMBOL_GPL(nft_register_chain_type);
1590
1591 void nft_unregister_chain_type(const struct nft_chain_type *ctype)
1592 {
1593         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1594         chain_type[ctype->family][ctype->type] = NULL;
1595         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1596 }
1597 EXPORT_SYMBOL_GPL(nft_unregister_chain_type);
1598
1599 /*
1600  * Chains
1601  */
1602
1603 static struct nft_chain *
1604 nft_chain_lookup_byhandle(const struct nft_table *table, u64 handle, u8 genmask)
1605 {
1606         struct nft_chain *chain;
1607
1608         list_for_each_entry(chain, &table->chains, list) {
1609                 if (chain->handle == handle &&
1610                     nft_active_genmask(chain, genmask))
1611                         return chain;
1612         }
1613
1614         return ERR_PTR(-ENOENT);
1615 }
1616
1617 static bool lockdep_commit_lock_is_held(const struct net *net)
1618 {
1619 #ifdef CONFIG_PROVE_LOCKING
1620         struct nftables_pernet *nft_net = nft_pernet(net);
1621
1622         return lockdep_is_held(&nft_net->commit_mutex);
1623 #else
1624         return true;
1625 #endif
1626 }
1627
1628 static struct nft_chain *nft_chain_lookup(struct net *net,
1629                                           struct nft_table *table,
1630                                           const struct nlattr *nla, u8 genmask)
1631 {
1632         char search[NFT_CHAIN_MAXNAMELEN + 1];
1633         struct rhlist_head *tmp, *list;
1634         struct nft_chain *chain;
1635
1636         if (nla == NULL)
1637                 return ERR_PTR(-EINVAL);
1638
1639         nla_strscpy(search, nla, sizeof(search));
1640
1641         WARN_ON(!rcu_read_lock_held() &&
1642                 !lockdep_commit_lock_is_held(net));
1643
1644         chain = ERR_PTR(-ENOENT);
1645         rcu_read_lock();
1646         list = rhltable_lookup(&table->chains_ht, search, nft_chain_ht_params);
1647         if (!list)
1648                 goto out_unlock;
1649
1650         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
1651                 if (nft_active_genmask(chain, genmask))
1652                         goto out_unlock;
1653         }
1654         chain = ERR_PTR(-ENOENT);
1655 out_unlock:
1656         rcu_read_unlock();
1657         return chain;
1658 }
1659
1660 static const struct nla_policy nft_chain_policy[NFTA_CHAIN_MAX + 1] = {
1661         [NFTA_CHAIN_TABLE]      = { .type = NLA_STRING,
1662                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
1663         [NFTA_CHAIN_HANDLE]     = { .type = NLA_U64 },
1664         [NFTA_CHAIN_NAME]       = { .type = NLA_STRING,
1665                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
1666         [NFTA_CHAIN_HOOK]       = { .type = NLA_NESTED },
1667         [NFTA_CHAIN_POLICY]     = { .type = NLA_U32 },
1668         [NFTA_CHAIN_TYPE]       = { .type = NLA_STRING,
1669                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
1670         [NFTA_CHAIN_COUNTERS]   = { .type = NLA_NESTED },
1671         [NFTA_CHAIN_FLAGS]      = { .type = NLA_U32 },
1672         [NFTA_CHAIN_ID]         = { .type = NLA_U32 },
1673         [NFTA_CHAIN_USERDATA]   = { .type = NLA_BINARY,
1674                                     .len = NFT_USERDATA_MAXLEN },
1675 };
1676
1677 static const struct nla_policy nft_hook_policy[NFTA_HOOK_MAX + 1] = {
1678         [NFTA_HOOK_HOOKNUM]     = { .type = NLA_U32 },
1679         [NFTA_HOOK_PRIORITY]    = { .type = NLA_U32 },
1680         [NFTA_HOOK_DEV]         = { .type = NLA_STRING,
1681                                     .len = IFNAMSIZ - 1 },
1682 };
1683
1684 static int nft_dump_stats(struct sk_buff *skb, struct nft_stats __percpu *stats)
1685 {
1686         struct nft_stats *cpu_stats, total;
1687         struct nlattr *nest;
1688         unsigned int seq;
1689         u64 pkts, bytes;
1690         int cpu;
1691
1692         if (!stats)
1693                 return 0;
1694
1695         memset(&total, 0, sizeof(total));
1696         for_each_possible_cpu(cpu) {
1697                 cpu_stats = per_cpu_ptr(stats, cpu);
1698                 do {
1699                         seq = u64_stats_fetch_begin(&cpu_stats->syncp);
1700                         pkts = cpu_stats->pkts;
1701                         bytes = cpu_stats->bytes;
1702                 } while (u64_stats_fetch_retry(&cpu_stats->syncp, seq));
1703                 total.pkts += pkts;
1704                 total.bytes += bytes;
1705         }
1706         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_COUNTERS);
1707         if (nest == NULL)
1708                 goto nla_put_failure;
1709
1710         if (nla_put_be64(skb, NFTA_COUNTER_PACKETS, cpu_to_be64(total.pkts),
1711                          NFTA_COUNTER_PAD) ||
1712             nla_put_be64(skb, NFTA_COUNTER_BYTES, cpu_to_be64(total.bytes),
1713                          NFTA_COUNTER_PAD))
1714                 goto nla_put_failure;
1715
1716         nla_nest_end(skb, nest);
1717         return 0;
1718
1719 nla_put_failure:
1720         return -ENOSPC;
1721 }
1722
1723 static int nft_dump_basechain_hook(struct sk_buff *skb, int family,
1724                                    const struct nft_base_chain *basechain,
1725                                    const struct list_head *hook_list)
1726 {
1727         const struct nf_hook_ops *ops = &basechain->ops;
1728         struct nft_hook *hook, *first = NULL;
1729         struct nlattr *nest, *nest_devs;
1730         int n = 0;
1731
1732         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_HOOK);
1733         if (nest == NULL)
1734                 goto nla_put_failure;
1735         if (nla_put_be32(skb, NFTA_HOOK_HOOKNUM, htonl(ops->hooknum)))
1736                 goto nla_put_failure;
1737         if (nla_put_be32(skb, NFTA_HOOK_PRIORITY, htonl(ops->priority)))
1738                 goto nla_put_failure;
1739
1740         if (nft_base_chain_netdev(family, ops->hooknum)) {
1741                 nest_devs = nla_nest_start_noflag(skb, NFTA_HOOK_DEVS);
1742                 if (!nest_devs)
1743                         goto nla_put_failure;
1744
1745                 if (!hook_list)
1746                         hook_list = &basechain->hook_list;
1747
1748                 list_for_each_entry(hook, hook_list, list) {
1749                         if (!first)
1750                                 first = hook;
1751
1752                         if (nla_put_string(skb, NFTA_DEVICE_NAME,
1753                                            hook->ops.dev->name))
1754                                 goto nla_put_failure;
1755                         n++;
1756                 }
1757                 nla_nest_end(skb, nest_devs);
1758
1759                 if (n == 1 &&
1760                     nla_put_string(skb, NFTA_HOOK_DEV, first->ops.dev->name))
1761                         goto nla_put_failure;
1762         }
1763         nla_nest_end(skb, nest);
1764
1765         return 0;
1766 nla_put_failure:
1767         return -1;
1768 }
1769
1770 static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net,
1771                                      u32 portid, u32 seq, int event, u32 flags,
1772                                      int family, const struct nft_table *table,
1773                                      const struct nft_chain *chain,
1774                                      const struct list_head *hook_list)
1775 {
1776         struct nlmsghdr *nlh;
1777
1778         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
1779         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
1780                            NFNETLINK_V0, nft_base_seq(net));
1781         if (!nlh)
1782                 goto nla_put_failure;
1783
1784         if (nla_put_string(skb, NFTA_CHAIN_TABLE, table->name) ||
1785             nla_put_string(skb, NFTA_CHAIN_NAME, chain->name) ||
1786             nla_put_be64(skb, NFTA_CHAIN_HANDLE, cpu_to_be64(chain->handle),
1787                          NFTA_CHAIN_PAD))
1788                 goto nla_put_failure;
1789
1790         if (event == NFT_MSG_DELCHAIN && !hook_list) {
1791                 nlmsg_end(skb, nlh);
1792                 return 0;
1793         }
1794
1795         if (nft_is_base_chain(chain)) {
1796                 const struct nft_base_chain *basechain = nft_base_chain(chain);
1797                 struct nft_stats __percpu *stats;
1798
1799                 if (nft_dump_basechain_hook(skb, family, basechain, hook_list))
1800                         goto nla_put_failure;
1801
1802                 if (nla_put_be32(skb, NFTA_CHAIN_POLICY,
1803                                  htonl(basechain->policy)))
1804                         goto nla_put_failure;
1805
1806                 if (nla_put_string(skb, NFTA_CHAIN_TYPE, basechain->type->name))
1807                         goto nla_put_failure;
1808
1809                 stats = rcu_dereference_check(basechain->stats,
1810                                               lockdep_commit_lock_is_held(net));
1811                 if (nft_dump_stats(skb, stats))
1812                         goto nla_put_failure;
1813         }
1814
1815         if (chain->flags &&
1816             nla_put_be32(skb, NFTA_CHAIN_FLAGS, htonl(chain->flags)))
1817                 goto nla_put_failure;
1818
1819         if (nla_put_be32(skb, NFTA_CHAIN_USE, htonl(chain->use)))
1820                 goto nla_put_failure;
1821
1822         if (chain->udata &&
1823             nla_put(skb, NFTA_CHAIN_USERDATA, chain->udlen, chain->udata))
1824                 goto nla_put_failure;
1825
1826         nlmsg_end(skb, nlh);
1827         return 0;
1828
1829 nla_put_failure:
1830         nlmsg_trim(skb, nlh);
1831         return -1;
1832 }
1833
1834 static void nf_tables_chain_notify(const struct nft_ctx *ctx, int event,
1835                                    const struct list_head *hook_list)
1836 {
1837         struct nftables_pernet *nft_net;
1838         struct sk_buff *skb;
1839         u16 flags = 0;
1840         int err;
1841
1842         if (!ctx->report &&
1843             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
1844                 return;
1845
1846         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
1847         if (skb == NULL)
1848                 goto err;
1849
1850         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
1851                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
1852
1853         err = nf_tables_fill_chain_info(skb, ctx->net, ctx->portid, ctx->seq,
1854                                         event, flags, ctx->family, ctx->table,
1855                                         ctx->chain, hook_list);
1856         if (err < 0) {
1857                 kfree_skb(skb);
1858                 goto err;
1859         }
1860
1861         nft_net = nft_pernet(ctx->net);
1862         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
1863         return;
1864 err:
1865         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
1866 }
1867
1868 static int nf_tables_dump_chains(struct sk_buff *skb,
1869                                  struct netlink_callback *cb)
1870 {
1871         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
1872         unsigned int idx = 0, s_idx = cb->args[0];
1873         struct net *net = sock_net(skb->sk);
1874         int family = nfmsg->nfgen_family;
1875         struct nftables_pernet *nft_net;
1876         const struct nft_table *table;
1877         const struct nft_chain *chain;
1878
1879         rcu_read_lock();
1880         nft_net = nft_pernet(net);
1881         cb->seq = READ_ONCE(nft_net->base_seq);
1882
1883         list_for_each_entry_rcu(table, &nft_net->tables, list) {
1884                 if (family != NFPROTO_UNSPEC && family != table->family)
1885                         continue;
1886
1887                 list_for_each_entry_rcu(chain, &table->chains, list) {
1888                         if (idx < s_idx)
1889                                 goto cont;
1890                         if (idx > s_idx)
1891                                 memset(&cb->args[1], 0,
1892                                        sizeof(cb->args) - sizeof(cb->args[0]));
1893                         if (!nft_is_active(net, chain))
1894                                 continue;
1895                         if (nf_tables_fill_chain_info(skb, net,
1896                                                       NETLINK_CB(cb->skb).portid,
1897                                                       cb->nlh->nlmsg_seq,
1898                                                       NFT_MSG_NEWCHAIN,
1899                                                       NLM_F_MULTI,
1900                                                       table->family, table,
1901                                                       chain, NULL) < 0)
1902                                 goto done;
1903
1904                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
1905 cont:
1906                         idx++;
1907                 }
1908         }
1909 done:
1910         rcu_read_unlock();
1911         cb->args[0] = idx;
1912         return skb->len;
1913 }
1914
1915 /* called with rcu_read_lock held */
1916 static int nf_tables_getchain(struct sk_buff *skb, const struct nfnl_info *info,
1917                               const struct nlattr * const nla[])
1918 {
1919         struct netlink_ext_ack *extack = info->extack;
1920         u8 genmask = nft_genmask_cur(info->net);
1921         u8 family = info->nfmsg->nfgen_family;
1922         const struct nft_chain *chain;
1923         struct net *net = info->net;
1924         struct nft_table *table;
1925         struct sk_buff *skb2;
1926         int err;
1927
1928         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
1929                 struct netlink_dump_control c = {
1930                         .dump = nf_tables_dump_chains,
1931                         .module = THIS_MODULE,
1932                 };
1933
1934                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
1935         }
1936
1937         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask, 0);
1938         if (IS_ERR(table)) {
1939                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
1940                 return PTR_ERR(table);
1941         }
1942
1943         chain = nft_chain_lookup(net, table, nla[NFTA_CHAIN_NAME], genmask);
1944         if (IS_ERR(chain)) {
1945                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
1946                 return PTR_ERR(chain);
1947         }
1948
1949         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
1950         if (!skb2)
1951                 return -ENOMEM;
1952
1953         err = nf_tables_fill_chain_info(skb2, net, NETLINK_CB(skb).portid,
1954                                         info->nlh->nlmsg_seq, NFT_MSG_NEWCHAIN,
1955                                         0, family, table, chain, NULL);
1956         if (err < 0)
1957                 goto err_fill_chain_info;
1958
1959         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
1960
1961 err_fill_chain_info:
1962         kfree_skb(skb2);
1963         return err;
1964 }
1965
1966 static const struct nla_policy nft_counter_policy[NFTA_COUNTER_MAX + 1] = {
1967         [NFTA_COUNTER_PACKETS]  = { .type = NLA_U64 },
1968         [NFTA_COUNTER_BYTES]    = { .type = NLA_U64 },
1969 };
1970
1971 static struct nft_stats __percpu *nft_stats_alloc(const struct nlattr *attr)
1972 {
1973         struct nlattr *tb[NFTA_COUNTER_MAX+1];
1974         struct nft_stats __percpu *newstats;
1975         struct nft_stats *stats;
1976         int err;
1977
1978         err = nla_parse_nested_deprecated(tb, NFTA_COUNTER_MAX, attr,
1979                                           nft_counter_policy, NULL);
1980         if (err < 0)
1981                 return ERR_PTR(err);
1982
1983         if (!tb[NFTA_COUNTER_BYTES] || !tb[NFTA_COUNTER_PACKETS])
1984                 return ERR_PTR(-EINVAL);
1985
1986         newstats = netdev_alloc_pcpu_stats(struct nft_stats);
1987         if (newstats == NULL)
1988                 return ERR_PTR(-ENOMEM);
1989
1990         /* Restore old counters on this cpu, no problem. Per-cpu statistics
1991          * are not exposed to userspace.
1992          */
1993         preempt_disable();
1994         stats = this_cpu_ptr(newstats);
1995         stats->bytes = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_BYTES]));
1996         stats->pkts = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_PACKETS]));
1997         preempt_enable();
1998
1999         return newstats;
2000 }
2001
2002 static void nft_chain_stats_replace(struct nft_trans *trans)
2003 {
2004         struct nft_base_chain *chain = nft_base_chain(trans->ctx.chain);
2005
2006         if (!nft_trans_chain_stats(trans))
2007                 return;
2008
2009         nft_trans_chain_stats(trans) =
2010                 rcu_replace_pointer(chain->stats, nft_trans_chain_stats(trans),
2011                                     lockdep_commit_lock_is_held(trans->ctx.net));
2012
2013         if (!nft_trans_chain_stats(trans))
2014                 static_branch_inc(&nft_counters_enabled);
2015 }
2016
2017 static void nf_tables_chain_free_chain_rules(struct nft_chain *chain)
2018 {
2019         struct nft_rule_blob *g0 = rcu_dereference_raw(chain->blob_gen_0);
2020         struct nft_rule_blob *g1 = rcu_dereference_raw(chain->blob_gen_1);
2021
2022         if (g0 != g1)
2023                 kvfree(g1);
2024         kvfree(g0);
2025
2026         /* should be NULL either via abort or via successful commit */
2027         WARN_ON_ONCE(chain->blob_next);
2028         kvfree(chain->blob_next);
2029 }
2030
2031 void nf_tables_chain_destroy(struct nft_ctx *ctx)
2032 {
2033         struct nft_chain *chain = ctx->chain;
2034         struct nft_hook *hook, *next;
2035
2036         if (WARN_ON(chain->use > 0))
2037                 return;
2038
2039         /* no concurrent access possible anymore */
2040         nf_tables_chain_free_chain_rules(chain);
2041
2042         if (nft_is_base_chain(chain)) {
2043                 struct nft_base_chain *basechain = nft_base_chain(chain);
2044
2045                 if (nft_base_chain_netdev(ctx->family, basechain->ops.hooknum)) {
2046                         list_for_each_entry_safe(hook, next,
2047                                                  &basechain->hook_list, list) {
2048                                 list_del_rcu(&hook->list);
2049                                 kfree_rcu(hook, rcu);
2050                         }
2051                 }
2052                 module_put(basechain->type->owner);
2053                 if (rcu_access_pointer(basechain->stats)) {
2054                         static_branch_dec(&nft_counters_enabled);
2055                         free_percpu(rcu_dereference_raw(basechain->stats));
2056                 }
2057                 kfree(chain->name);
2058                 kfree(chain->udata);
2059                 kfree(basechain);
2060         } else {
2061                 kfree(chain->name);
2062                 kfree(chain->udata);
2063                 kfree(chain);
2064         }
2065 }
2066
2067 static struct nft_hook *nft_netdev_hook_alloc(struct net *net,
2068                                               const struct nlattr *attr)
2069 {
2070         struct net_device *dev;
2071         char ifname[IFNAMSIZ];
2072         struct nft_hook *hook;
2073         int err;
2074
2075         hook = kmalloc(sizeof(struct nft_hook), GFP_KERNEL_ACCOUNT);
2076         if (!hook) {
2077                 err = -ENOMEM;
2078                 goto err_hook_alloc;
2079         }
2080
2081         nla_strscpy(ifname, attr, IFNAMSIZ);
2082         /* nf_tables_netdev_event() is called under rtnl_mutex, this is
2083          * indirectly serializing all the other holders of the commit_mutex with
2084          * the rtnl_mutex.
2085          */
2086         dev = __dev_get_by_name(net, ifname);
2087         if (!dev) {
2088                 err = -ENOENT;
2089                 goto err_hook_dev;
2090         }
2091         hook->ops.dev = dev;
2092
2093         return hook;
2094
2095 err_hook_dev:
2096         kfree(hook);
2097 err_hook_alloc:
2098         return ERR_PTR(err);
2099 }
2100
2101 static struct nft_hook *nft_hook_list_find(struct list_head *hook_list,
2102                                            const struct nft_hook *this)
2103 {
2104         struct nft_hook *hook;
2105
2106         list_for_each_entry(hook, hook_list, list) {
2107                 if (this->ops.dev == hook->ops.dev)
2108                         return hook;
2109         }
2110
2111         return NULL;
2112 }
2113
2114 static int nf_tables_parse_netdev_hooks(struct net *net,
2115                                         const struct nlattr *attr,
2116                                         struct list_head *hook_list,
2117                                         struct netlink_ext_ack *extack)
2118 {
2119         struct nft_hook *hook, *next;
2120         const struct nlattr *tmp;
2121         int rem, n = 0, err;
2122
2123         nla_for_each_nested(tmp, attr, rem) {
2124                 if (nla_type(tmp) != NFTA_DEVICE_NAME) {
2125                         err = -EINVAL;
2126                         goto err_hook;
2127                 }
2128
2129                 hook = nft_netdev_hook_alloc(net, tmp);
2130                 if (IS_ERR(hook)) {
2131                         NL_SET_BAD_ATTR(extack, tmp);
2132                         err = PTR_ERR(hook);
2133                         goto err_hook;
2134                 }
2135                 if (nft_hook_list_find(hook_list, hook)) {
2136                         NL_SET_BAD_ATTR(extack, tmp);
2137                         kfree(hook);
2138                         err = -EEXIST;
2139                         goto err_hook;
2140                 }
2141                 list_add_tail(&hook->list, hook_list);
2142                 n++;
2143
2144                 if (n == NFT_NETDEVICE_MAX) {
2145                         err = -EFBIG;
2146                         goto err_hook;
2147                 }
2148         }
2149
2150         return 0;
2151
2152 err_hook:
2153         list_for_each_entry_safe(hook, next, hook_list, list) {
2154                 list_del(&hook->list);
2155                 kfree(hook);
2156         }
2157         return err;
2158 }
2159
2160 struct nft_chain_hook {
2161         u32                             num;
2162         s32                             priority;
2163         const struct nft_chain_type     *type;
2164         struct list_head                list;
2165 };
2166
2167 static int nft_chain_parse_netdev(struct net *net, struct nlattr *tb[],
2168                                   struct list_head *hook_list,
2169                                   struct netlink_ext_ack *extack, u32 flags)
2170 {
2171         struct nft_hook *hook;
2172         int err;
2173
2174         if (tb[NFTA_HOOK_DEV]) {
2175                 hook = nft_netdev_hook_alloc(net, tb[NFTA_HOOK_DEV]);
2176                 if (IS_ERR(hook)) {
2177                         NL_SET_BAD_ATTR(extack, tb[NFTA_HOOK_DEV]);
2178                         return PTR_ERR(hook);
2179                 }
2180
2181                 list_add_tail(&hook->list, hook_list);
2182         } else if (tb[NFTA_HOOK_DEVS]) {
2183                 err = nf_tables_parse_netdev_hooks(net, tb[NFTA_HOOK_DEVS],
2184                                                    hook_list, extack);
2185                 if (err < 0)
2186                         return err;
2187
2188         }
2189
2190         if (flags & NFT_CHAIN_HW_OFFLOAD &&
2191             list_empty(hook_list))
2192                 return -EINVAL;
2193
2194         return 0;
2195 }
2196
2197 static int nft_chain_parse_hook(struct net *net,
2198                                 struct nft_base_chain *basechain,
2199                                 const struct nlattr * const nla[],
2200                                 struct nft_chain_hook *hook, u8 family,
2201                                 u32 flags, struct netlink_ext_ack *extack)
2202 {
2203         struct nftables_pernet *nft_net = nft_pernet(net);
2204         struct nlattr *ha[NFTA_HOOK_MAX + 1];
2205         const struct nft_chain_type *type;
2206         int err;
2207
2208         lockdep_assert_held(&nft_net->commit_mutex);
2209         lockdep_nfnl_nft_mutex_not_held();
2210
2211         err = nla_parse_nested_deprecated(ha, NFTA_HOOK_MAX,
2212                                           nla[NFTA_CHAIN_HOOK],
2213                                           nft_hook_policy, NULL);
2214         if (err < 0)
2215                 return err;
2216
2217         if (!basechain) {
2218                 if (!ha[NFTA_HOOK_HOOKNUM] ||
2219                     !ha[NFTA_HOOK_PRIORITY]) {
2220                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2221                         return -ENOENT;
2222                 }
2223
2224                 hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
2225                 hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
2226
2227                 type = __nft_chain_type_get(family, NFT_CHAIN_T_DEFAULT);
2228                 if (!type)
2229                         return -EOPNOTSUPP;
2230
2231                 if (nla[NFTA_CHAIN_TYPE]) {
2232                         type = nf_tables_chain_type_lookup(net, nla[NFTA_CHAIN_TYPE],
2233                                                            family, true);
2234                         if (IS_ERR(type)) {
2235                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2236                                 return PTR_ERR(type);
2237                         }
2238                 }
2239                 if (hook->num >= NFT_MAX_HOOKS || !(type->hook_mask & (1 << hook->num)))
2240                         return -EOPNOTSUPP;
2241
2242                 if (type->type == NFT_CHAIN_T_NAT &&
2243                     hook->priority <= NF_IP_PRI_CONNTRACK)
2244                         return -EOPNOTSUPP;
2245         } else {
2246                 if (ha[NFTA_HOOK_HOOKNUM]) {
2247                         hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
2248                         if (hook->num != basechain->ops.hooknum)
2249                                 return -EOPNOTSUPP;
2250                 }
2251                 if (ha[NFTA_HOOK_PRIORITY]) {
2252                         hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
2253                         if (hook->priority != basechain->ops.priority)
2254                                 return -EOPNOTSUPP;
2255                 }
2256
2257                 type = basechain->type;
2258         }
2259
2260         if (!try_module_get(type->owner)) {
2261                 if (nla[NFTA_CHAIN_TYPE])
2262                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2263                 return -ENOENT;
2264         }
2265
2266         hook->type = type;
2267
2268         INIT_LIST_HEAD(&hook->list);
2269         if (nft_base_chain_netdev(family, hook->num)) {
2270                 err = nft_chain_parse_netdev(net, ha, &hook->list, extack, flags);
2271                 if (err < 0) {
2272                         module_put(type->owner);
2273                         return err;
2274                 }
2275         } else if (ha[NFTA_HOOK_DEV] || ha[NFTA_HOOK_DEVS]) {
2276                 module_put(type->owner);
2277                 return -EOPNOTSUPP;
2278         }
2279
2280         return 0;
2281 }
2282
2283 static void nft_chain_release_hook(struct nft_chain_hook *hook)
2284 {
2285         struct nft_hook *h, *next;
2286
2287         list_for_each_entry_safe(h, next, &hook->list, list) {
2288                 list_del(&h->list);
2289                 kfree(h);
2290         }
2291         module_put(hook->type->owner);
2292 }
2293
2294 static void nft_last_rule(const struct nft_chain *chain, const void *ptr)
2295 {
2296         struct nft_rule_dp_last *lrule;
2297
2298         BUILD_BUG_ON(offsetof(struct nft_rule_dp_last, end) != 0);
2299
2300         lrule = (struct nft_rule_dp_last *)ptr;
2301         lrule->end.is_last = 1;
2302         lrule->chain = chain;
2303         /* blob size does not include the trailer rule */
2304 }
2305
2306 static struct nft_rule_blob *nf_tables_chain_alloc_rules(const struct nft_chain *chain,
2307                                                          unsigned int size)
2308 {
2309         struct nft_rule_blob *blob;
2310
2311         if (size > INT_MAX)
2312                 return NULL;
2313
2314         size += sizeof(struct nft_rule_blob) + sizeof(struct nft_rule_dp_last);
2315
2316         blob = kvmalloc(size, GFP_KERNEL_ACCOUNT);
2317         if (!blob)
2318                 return NULL;
2319
2320         blob->size = 0;
2321         nft_last_rule(chain, blob->data);
2322
2323         return blob;
2324 }
2325
2326 static void nft_basechain_hook_init(struct nf_hook_ops *ops, u8 family,
2327                                     const struct nft_chain_hook *hook,
2328                                     struct nft_chain *chain)
2329 {
2330         ops->pf                 = family;
2331         ops->hooknum            = hook->num;
2332         ops->priority           = hook->priority;
2333         ops->priv               = chain;
2334         ops->hook               = hook->type->hooks[ops->hooknum];
2335         ops->hook_ops_type      = NF_HOOK_OP_NF_TABLES;
2336 }
2337
2338 static int nft_basechain_init(struct nft_base_chain *basechain, u8 family,
2339                               struct nft_chain_hook *hook, u32 flags)
2340 {
2341         struct nft_chain *chain;
2342         struct nft_hook *h;
2343
2344         basechain->type = hook->type;
2345         INIT_LIST_HEAD(&basechain->hook_list);
2346         chain = &basechain->chain;
2347
2348         if (nft_base_chain_netdev(family, hook->num)) {
2349                 list_splice_init(&hook->list, &basechain->hook_list);
2350                 list_for_each_entry(h, &basechain->hook_list, list)
2351                         nft_basechain_hook_init(&h->ops, family, hook, chain);
2352         }
2353         nft_basechain_hook_init(&basechain->ops, family, hook, chain);
2354
2355         chain->flags |= NFT_CHAIN_BASE | flags;
2356         basechain->policy = NF_ACCEPT;
2357         if (chain->flags & NFT_CHAIN_HW_OFFLOAD &&
2358             !nft_chain_offload_support(basechain)) {
2359                 list_splice_init(&basechain->hook_list, &hook->list);
2360                 return -EOPNOTSUPP;
2361         }
2362
2363         flow_block_init(&basechain->flow_block);
2364
2365         return 0;
2366 }
2367
2368 int nft_chain_add(struct nft_table *table, struct nft_chain *chain)
2369 {
2370         int err;
2371
2372         err = rhltable_insert_key(&table->chains_ht, chain->name,
2373                                   &chain->rhlhead, nft_chain_ht_params);
2374         if (err)
2375                 return err;
2376
2377         list_add_tail_rcu(&chain->list, &table->chains);
2378
2379         return 0;
2380 }
2381
2382 static u64 chain_id;
2383
2384 static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 genmask,
2385                               u8 policy, u32 flags,
2386                               struct netlink_ext_ack *extack)
2387 {
2388         const struct nlattr * const *nla = ctx->nla;
2389         struct nft_table *table = ctx->table;
2390         struct nft_base_chain *basechain;
2391         struct net *net = ctx->net;
2392         char name[NFT_NAME_MAXLEN];
2393         struct nft_rule_blob *blob;
2394         struct nft_trans *trans;
2395         struct nft_chain *chain;
2396         int err;
2397
2398         if (table->use == UINT_MAX)
2399                 return -EOVERFLOW;
2400
2401         if (nla[NFTA_CHAIN_HOOK]) {
2402                 struct nft_stats __percpu *stats = NULL;
2403                 struct nft_chain_hook hook = {};
2404
2405                 if (flags & NFT_CHAIN_BINDING)
2406                         return -EOPNOTSUPP;
2407
2408                 err = nft_chain_parse_hook(net, NULL, nla, &hook, family, flags,
2409                                            extack);
2410                 if (err < 0)
2411                         return err;
2412
2413                 basechain = kzalloc(sizeof(*basechain), GFP_KERNEL_ACCOUNT);
2414                 if (basechain == NULL) {
2415                         nft_chain_release_hook(&hook);
2416                         return -ENOMEM;
2417                 }
2418                 chain = &basechain->chain;
2419
2420                 if (nla[NFTA_CHAIN_COUNTERS]) {
2421                         stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2422                         if (IS_ERR(stats)) {
2423                                 nft_chain_release_hook(&hook);
2424                                 kfree(basechain);
2425                                 return PTR_ERR(stats);
2426                         }
2427                         rcu_assign_pointer(basechain->stats, stats);
2428                 }
2429
2430                 err = nft_basechain_init(basechain, family, &hook, flags);
2431                 if (err < 0) {
2432                         nft_chain_release_hook(&hook);
2433                         kfree(basechain);
2434                         free_percpu(stats);
2435                         return err;
2436                 }
2437                 if (stats)
2438                         static_branch_inc(&nft_counters_enabled);
2439         } else {
2440                 if (flags & NFT_CHAIN_BASE)
2441                         return -EINVAL;
2442                 if (flags & NFT_CHAIN_HW_OFFLOAD)
2443                         return -EOPNOTSUPP;
2444
2445                 chain = kzalloc(sizeof(*chain), GFP_KERNEL_ACCOUNT);
2446                 if (chain == NULL)
2447                         return -ENOMEM;
2448
2449                 chain->flags = flags;
2450         }
2451         ctx->chain = chain;
2452
2453         INIT_LIST_HEAD(&chain->rules);
2454         chain->handle = nf_tables_alloc_handle(table);
2455         chain->table = table;
2456
2457         if (nla[NFTA_CHAIN_NAME]) {
2458                 chain->name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
2459         } else {
2460                 if (!(flags & NFT_CHAIN_BINDING)) {
2461                         err = -EINVAL;
2462                         goto err_destroy_chain;
2463                 }
2464
2465                 snprintf(name, sizeof(name), "__chain%llu", ++chain_id);
2466                 chain->name = kstrdup(name, GFP_KERNEL_ACCOUNT);
2467         }
2468
2469         if (!chain->name) {
2470                 err = -ENOMEM;
2471                 goto err_destroy_chain;
2472         }
2473
2474         if (nla[NFTA_CHAIN_USERDATA]) {
2475                 chain->udata = nla_memdup(nla[NFTA_CHAIN_USERDATA], GFP_KERNEL_ACCOUNT);
2476                 if (chain->udata == NULL) {
2477                         err = -ENOMEM;
2478                         goto err_destroy_chain;
2479                 }
2480                 chain->udlen = nla_len(nla[NFTA_CHAIN_USERDATA]);
2481         }
2482
2483         blob = nf_tables_chain_alloc_rules(chain, 0);
2484         if (!blob) {
2485                 err = -ENOMEM;
2486                 goto err_destroy_chain;
2487         }
2488
2489         RCU_INIT_POINTER(chain->blob_gen_0, blob);
2490         RCU_INIT_POINTER(chain->blob_gen_1, blob);
2491
2492         err = nf_tables_register_hook(net, table, chain);
2493         if (err < 0)
2494                 goto err_destroy_chain;
2495
2496         trans = nft_trans_chain_add(ctx, NFT_MSG_NEWCHAIN);
2497         if (IS_ERR(trans)) {
2498                 err = PTR_ERR(trans);
2499                 goto err_unregister_hook;
2500         }
2501
2502         nft_trans_chain_policy(trans) = NFT_CHAIN_POLICY_UNSET;
2503         if (nft_is_base_chain(chain))
2504                 nft_trans_chain_policy(trans) = policy;
2505
2506         err = nft_chain_add(table, chain);
2507         if (err < 0) {
2508                 nft_trans_destroy(trans);
2509                 goto err_unregister_hook;
2510         }
2511
2512         table->use++;
2513
2514         return 0;
2515 err_unregister_hook:
2516         nf_tables_unregister_hook(net, table, chain);
2517 err_destroy_chain:
2518         nf_tables_chain_destroy(ctx);
2519
2520         return err;
2521 }
2522
2523 static int nf_tables_updchain(struct nft_ctx *ctx, u8 genmask, u8 policy,
2524                               u32 flags, const struct nlattr *attr,
2525                               struct netlink_ext_ack *extack)
2526 {
2527         const struct nlattr * const *nla = ctx->nla;
2528         struct nft_base_chain *basechain = NULL;
2529         struct nft_table *table = ctx->table;
2530         struct nft_chain *chain = ctx->chain;
2531         struct nft_chain_hook hook = {};
2532         struct nft_stats *stats = NULL;
2533         struct nft_hook *h, *next;
2534         struct nf_hook_ops *ops;
2535         struct nft_trans *trans;
2536         bool unregister = false;
2537         int err;
2538
2539         if (chain->flags ^ flags)
2540                 return -EOPNOTSUPP;
2541
2542         INIT_LIST_HEAD(&hook.list);
2543
2544         if (nla[NFTA_CHAIN_HOOK]) {
2545                 if (!nft_is_base_chain(chain)) {
2546                         NL_SET_BAD_ATTR(extack, attr);
2547                         return -EEXIST;
2548                 }
2549
2550                 basechain = nft_base_chain(chain);
2551                 err = nft_chain_parse_hook(ctx->net, basechain, nla, &hook,
2552                                            ctx->family, flags, extack);
2553                 if (err < 0)
2554                         return err;
2555
2556                 if (basechain->type != hook.type) {
2557                         nft_chain_release_hook(&hook);
2558                         NL_SET_BAD_ATTR(extack, attr);
2559                         return -EEXIST;
2560                 }
2561
2562                 if (nft_base_chain_netdev(ctx->family, basechain->ops.hooknum)) {
2563                         list_for_each_entry_safe(h, next, &hook.list, list) {
2564                                 h->ops.pf       = basechain->ops.pf;
2565                                 h->ops.hooknum  = basechain->ops.hooknum;
2566                                 h->ops.priority = basechain->ops.priority;
2567                                 h->ops.priv     = basechain->ops.priv;
2568                                 h->ops.hook     = basechain->ops.hook;
2569
2570                                 if (nft_hook_list_find(&basechain->hook_list, h)) {
2571                                         list_del(&h->list);
2572                                         kfree(h);
2573                                 }
2574                         }
2575                 } else {
2576                         ops = &basechain->ops;
2577                         if (ops->hooknum != hook.num ||
2578                             ops->priority != hook.priority) {
2579                                 nft_chain_release_hook(&hook);
2580                                 NL_SET_BAD_ATTR(extack, attr);
2581                                 return -EEXIST;
2582                         }
2583                 }
2584         }
2585
2586         if (nla[NFTA_CHAIN_HANDLE] &&
2587             nla[NFTA_CHAIN_NAME]) {
2588                 struct nft_chain *chain2;
2589
2590                 chain2 = nft_chain_lookup(ctx->net, table,
2591                                           nla[NFTA_CHAIN_NAME], genmask);
2592                 if (!IS_ERR(chain2)) {
2593                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2594                         err = -EEXIST;
2595                         goto err_hooks;
2596                 }
2597         }
2598
2599         if (nla[NFTA_CHAIN_COUNTERS]) {
2600                 if (!nft_is_base_chain(chain)) {
2601                         err = -EOPNOTSUPP;
2602                         goto err_hooks;
2603                 }
2604
2605                 stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2606                 if (IS_ERR(stats)) {
2607                         err = PTR_ERR(stats);
2608                         goto err_hooks;
2609                 }
2610         }
2611
2612         if (!(table->flags & NFT_TABLE_F_DORMANT) &&
2613             nft_is_base_chain(chain) &&
2614             !list_empty(&hook.list)) {
2615                 basechain = nft_base_chain(chain);
2616                 ops = &basechain->ops;
2617
2618                 if (nft_base_chain_netdev(table->family, basechain->ops.hooknum)) {
2619                         err = nft_netdev_register_hooks(ctx->net, &hook.list);
2620                         if (err < 0)
2621                                 goto err_hooks;
2622                 }
2623         }
2624
2625         unregister = true;
2626         err = -ENOMEM;
2627         trans = nft_trans_alloc(ctx, NFT_MSG_NEWCHAIN,
2628                                 sizeof(struct nft_trans_chain));
2629         if (trans == NULL)
2630                 goto err_trans;
2631
2632         nft_trans_chain_stats(trans) = stats;
2633         nft_trans_chain_update(trans) = true;
2634
2635         if (nla[NFTA_CHAIN_POLICY])
2636                 nft_trans_chain_policy(trans) = policy;
2637         else
2638                 nft_trans_chain_policy(trans) = -1;
2639
2640         if (nla[NFTA_CHAIN_HANDLE] &&
2641             nla[NFTA_CHAIN_NAME]) {
2642                 struct nftables_pernet *nft_net = nft_pernet(ctx->net);
2643                 struct nft_trans *tmp;
2644                 char *name;
2645
2646                 err = -ENOMEM;
2647                 name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
2648                 if (!name)
2649                         goto err_trans;
2650
2651                 err = -EEXIST;
2652                 list_for_each_entry(tmp, &nft_net->commit_list, list) {
2653                         if (tmp->msg_type == NFT_MSG_NEWCHAIN &&
2654                             tmp->ctx.table == table &&
2655                             nft_trans_chain_update(tmp) &&
2656                             nft_trans_chain_name(tmp) &&
2657                             strcmp(name, nft_trans_chain_name(tmp)) == 0) {
2658                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2659                                 kfree(name);
2660                                 goto err_trans;
2661                         }
2662                 }
2663
2664                 nft_trans_chain_name(trans) = name;
2665         }
2666
2667         nft_trans_basechain(trans) = basechain;
2668         INIT_LIST_HEAD(&nft_trans_chain_hooks(trans));
2669         list_splice(&hook.list, &nft_trans_chain_hooks(trans));
2670
2671         nft_trans_commit_list_add_tail(ctx->net, trans);
2672
2673         return 0;
2674
2675 err_trans:
2676         free_percpu(stats);
2677         kfree(trans);
2678 err_hooks:
2679         if (nla[NFTA_CHAIN_HOOK]) {
2680                 list_for_each_entry_safe(h, next, &hook.list, list) {
2681                         if (unregister)
2682                                 nf_unregister_net_hook(ctx->net, &h->ops);
2683                         list_del(&h->list);
2684                         kfree_rcu(h, rcu);
2685                 }
2686                 module_put(hook.type->owner);
2687         }
2688
2689         return err;
2690 }
2691
2692 static struct nft_chain *nft_chain_lookup_byid(const struct net *net,
2693                                                const struct nft_table *table,
2694                                                const struct nlattr *nla)
2695 {
2696         struct nftables_pernet *nft_net = nft_pernet(net);
2697         u32 id = ntohl(nla_get_be32(nla));
2698         struct nft_trans *trans;
2699
2700         list_for_each_entry(trans, &nft_net->commit_list, list) {
2701                 struct nft_chain *chain = trans->ctx.chain;
2702
2703                 if (trans->msg_type == NFT_MSG_NEWCHAIN &&
2704                     chain->table == table &&
2705                     id == nft_trans_chain_id(trans))
2706                         return chain;
2707         }
2708         return ERR_PTR(-ENOENT);
2709 }
2710
2711 static int nf_tables_newchain(struct sk_buff *skb, const struct nfnl_info *info,
2712                               const struct nlattr * const nla[])
2713 {
2714         struct nftables_pernet *nft_net = nft_pernet(info->net);
2715         struct netlink_ext_ack *extack = info->extack;
2716         u8 genmask = nft_genmask_next(info->net);
2717         u8 family = info->nfmsg->nfgen_family;
2718         struct nft_chain *chain = NULL;
2719         struct net *net = info->net;
2720         const struct nlattr *attr;
2721         struct nft_table *table;
2722         u8 policy = NF_ACCEPT;
2723         struct nft_ctx ctx;
2724         u64 handle = 0;
2725         u32 flags = 0;
2726
2727         lockdep_assert_held(&nft_net->commit_mutex);
2728
2729         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
2730                                  NETLINK_CB(skb).portid);
2731         if (IS_ERR(table)) {
2732                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2733                 return PTR_ERR(table);
2734         }
2735
2736         chain = NULL;
2737         attr = nla[NFTA_CHAIN_NAME];
2738
2739         if (nla[NFTA_CHAIN_HANDLE]) {
2740                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_CHAIN_HANDLE]));
2741                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2742                 if (IS_ERR(chain)) {
2743                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_HANDLE]);
2744                         return PTR_ERR(chain);
2745                 }
2746                 attr = nla[NFTA_CHAIN_HANDLE];
2747         } else if (nla[NFTA_CHAIN_NAME]) {
2748                 chain = nft_chain_lookup(net, table, attr, genmask);
2749                 if (IS_ERR(chain)) {
2750                         if (PTR_ERR(chain) != -ENOENT) {
2751                                 NL_SET_BAD_ATTR(extack, attr);
2752                                 return PTR_ERR(chain);
2753                         }
2754                         chain = NULL;
2755                 }
2756         } else if (!nla[NFTA_CHAIN_ID]) {
2757                 return -EINVAL;
2758         }
2759
2760         if (nla[NFTA_CHAIN_POLICY]) {
2761                 if (chain != NULL &&
2762                     !nft_is_base_chain(chain)) {
2763                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2764                         return -EOPNOTSUPP;
2765                 }
2766
2767                 if (chain == NULL &&
2768                     nla[NFTA_CHAIN_HOOK] == NULL) {
2769                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2770                         return -EOPNOTSUPP;
2771                 }
2772
2773                 policy = ntohl(nla_get_be32(nla[NFTA_CHAIN_POLICY]));
2774                 switch (policy) {
2775                 case NF_DROP:
2776                 case NF_ACCEPT:
2777                         break;
2778                 default:
2779                         return -EINVAL;
2780                 }
2781         }
2782
2783         if (nla[NFTA_CHAIN_FLAGS])
2784                 flags = ntohl(nla_get_be32(nla[NFTA_CHAIN_FLAGS]));
2785         else if (chain)
2786                 flags = chain->flags;
2787
2788         if (flags & ~NFT_CHAIN_FLAGS)
2789                 return -EOPNOTSUPP;
2790
2791         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
2792
2793         if (chain != NULL) {
2794                 if (chain->flags & NFT_CHAIN_BINDING)
2795                         return -EINVAL;
2796
2797                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
2798                         NL_SET_BAD_ATTR(extack, attr);
2799                         return -EEXIST;
2800                 }
2801                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
2802                         return -EOPNOTSUPP;
2803
2804                 flags |= chain->flags & NFT_CHAIN_BASE;
2805                 return nf_tables_updchain(&ctx, genmask, policy, flags, attr,
2806                                           extack);
2807         }
2808
2809         return nf_tables_addchain(&ctx, family, genmask, policy, flags, extack);
2810 }
2811
2812 static int nft_delchain_hook(struct nft_ctx *ctx, struct nft_chain *chain,
2813                              struct netlink_ext_ack *extack)
2814 {
2815         const struct nlattr * const *nla = ctx->nla;
2816         struct nft_chain_hook chain_hook = {};
2817         struct nft_base_chain *basechain;
2818         struct nft_hook *this, *hook;
2819         LIST_HEAD(chain_del_list);
2820         struct nft_trans *trans;
2821         int err;
2822
2823         if (!nft_is_base_chain(chain))
2824                 return -EOPNOTSUPP;
2825
2826         basechain = nft_base_chain(chain);
2827         err = nft_chain_parse_hook(ctx->net, basechain, nla, &chain_hook,
2828                                    ctx->family, chain->flags, extack);
2829         if (err < 0)
2830                 return err;
2831
2832         list_for_each_entry(this, &chain_hook.list, list) {
2833                 hook = nft_hook_list_find(&basechain->hook_list, this);
2834                 if (!hook) {
2835                         err = -ENOENT;
2836                         goto err_chain_del_hook;
2837                 }
2838                 list_move(&hook->list, &chain_del_list);
2839         }
2840
2841         trans = nft_trans_alloc(ctx, NFT_MSG_DELCHAIN,
2842                                 sizeof(struct nft_trans_chain));
2843         if (!trans) {
2844                 err = -ENOMEM;
2845                 goto err_chain_del_hook;
2846         }
2847
2848         nft_trans_basechain(trans) = basechain;
2849         nft_trans_chain_update(trans) = true;
2850         INIT_LIST_HEAD(&nft_trans_chain_hooks(trans));
2851         list_splice(&chain_del_list, &nft_trans_chain_hooks(trans));
2852         nft_chain_release_hook(&chain_hook);
2853
2854         nft_trans_commit_list_add_tail(ctx->net, trans);
2855
2856         return 0;
2857
2858 err_chain_del_hook:
2859         list_splice(&chain_del_list, &basechain->hook_list);
2860         nft_chain_release_hook(&chain_hook);
2861
2862         return err;
2863 }
2864
2865 static int nf_tables_delchain(struct sk_buff *skb, const struct nfnl_info *info,
2866                               const struct nlattr * const nla[])
2867 {
2868         struct netlink_ext_ack *extack = info->extack;
2869         u8 genmask = nft_genmask_next(info->net);
2870         u8 family = info->nfmsg->nfgen_family;
2871         struct net *net = info->net;
2872         const struct nlattr *attr;
2873         struct nft_table *table;
2874         struct nft_chain *chain;
2875         struct nft_rule *rule;
2876         struct nft_ctx ctx;
2877         u64 handle;
2878         u32 use;
2879         int err;
2880
2881         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
2882                                  NETLINK_CB(skb).portid);
2883         if (IS_ERR(table)) {
2884                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2885                 return PTR_ERR(table);
2886         }
2887
2888         if (nla[NFTA_CHAIN_HANDLE]) {
2889                 attr = nla[NFTA_CHAIN_HANDLE];
2890                 handle = be64_to_cpu(nla_get_be64(attr));
2891                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2892         } else {
2893                 attr = nla[NFTA_CHAIN_NAME];
2894                 chain = nft_chain_lookup(net, table, attr, genmask);
2895         }
2896         if (IS_ERR(chain)) {
2897                 if (PTR_ERR(chain) == -ENOENT &&
2898                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYCHAIN)
2899                         return 0;
2900
2901                 NL_SET_BAD_ATTR(extack, attr);
2902                 return PTR_ERR(chain);
2903         }
2904
2905         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
2906
2907         if (nla[NFTA_CHAIN_HOOK]) {
2908                 if (chain->flags & NFT_CHAIN_HW_OFFLOAD)
2909                         return -EOPNOTSUPP;
2910
2911                 return nft_delchain_hook(&ctx, chain, extack);
2912         }
2913
2914         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
2915             chain->use > 0)
2916                 return -EBUSY;
2917
2918         use = chain->use;
2919         list_for_each_entry(rule, &chain->rules, list) {
2920                 if (!nft_is_active_next(net, rule))
2921                         continue;
2922                 use--;
2923
2924                 err = nft_delrule(&ctx, rule);
2925                 if (err < 0)
2926                         return err;
2927         }
2928
2929         /* There are rules and elements that are still holding references to us,
2930          * we cannot do a recursive removal in this case.
2931          */
2932         if (use > 0) {
2933                 NL_SET_BAD_ATTR(extack, attr);
2934                 return -EBUSY;
2935         }
2936
2937         return nft_delchain(&ctx);
2938 }
2939
2940 /*
2941  * Expressions
2942  */
2943
2944 /**
2945  *      nft_register_expr - register nf_tables expr type
2946  *      @type: expr type
2947  *
2948  *      Registers the expr type for use with nf_tables. Returns zero on
2949  *      success or a negative errno code otherwise.
2950  */
2951 int nft_register_expr(struct nft_expr_type *type)
2952 {
2953         nfnl_lock(NFNL_SUBSYS_NFTABLES);
2954         if (type->family == NFPROTO_UNSPEC)
2955                 list_add_tail_rcu(&type->list, &nf_tables_expressions);
2956         else
2957                 list_add_rcu(&type->list, &nf_tables_expressions);
2958         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
2959         return 0;
2960 }
2961 EXPORT_SYMBOL_GPL(nft_register_expr);
2962
2963 /**
2964  *      nft_unregister_expr - unregister nf_tables expr type
2965  *      @type: expr type
2966  *
2967  *      Unregisters the expr typefor use with nf_tables.
2968  */
2969 void nft_unregister_expr(struct nft_expr_type *type)
2970 {
2971         nfnl_lock(NFNL_SUBSYS_NFTABLES);
2972         list_del_rcu(&type->list);
2973         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
2974 }
2975 EXPORT_SYMBOL_GPL(nft_unregister_expr);
2976
2977 static const struct nft_expr_type *__nft_expr_type_get(u8 family,
2978                                                        struct nlattr *nla)
2979 {
2980         const struct nft_expr_type *type, *candidate = NULL;
2981
2982         list_for_each_entry(type, &nf_tables_expressions, list) {
2983                 if (!nla_strcmp(nla, type->name)) {
2984                         if (!type->family && !candidate)
2985                                 candidate = type;
2986                         else if (type->family == family)
2987                                 candidate = type;
2988                 }
2989         }
2990         return candidate;
2991 }
2992
2993 #ifdef CONFIG_MODULES
2994 static int nft_expr_type_request_module(struct net *net, u8 family,
2995                                         struct nlattr *nla)
2996 {
2997         if (nft_request_module(net, "nft-expr-%u-%.*s", family,
2998                                nla_len(nla), (char *)nla_data(nla)) == -EAGAIN)
2999                 return -EAGAIN;
3000
3001         return 0;
3002 }
3003 #endif
3004
3005 static const struct nft_expr_type *nft_expr_type_get(struct net *net,
3006                                                      u8 family,
3007                                                      struct nlattr *nla)
3008 {
3009         const struct nft_expr_type *type;
3010
3011         if (nla == NULL)
3012                 return ERR_PTR(-EINVAL);
3013
3014         type = __nft_expr_type_get(family, nla);
3015         if (type != NULL && try_module_get(type->owner))
3016                 return type;
3017
3018         lockdep_nfnl_nft_mutex_not_held();
3019 #ifdef CONFIG_MODULES
3020         if (type == NULL) {
3021                 if (nft_expr_type_request_module(net, family, nla) == -EAGAIN)
3022                         return ERR_PTR(-EAGAIN);
3023
3024                 if (nft_request_module(net, "nft-expr-%.*s",
3025                                        nla_len(nla),
3026                                        (char *)nla_data(nla)) == -EAGAIN)
3027                         return ERR_PTR(-EAGAIN);
3028         }
3029 #endif
3030         return ERR_PTR(-ENOENT);
3031 }
3032
3033 static const struct nla_policy nft_expr_policy[NFTA_EXPR_MAX + 1] = {
3034         [NFTA_EXPR_NAME]        = { .type = NLA_STRING,
3035                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
3036         [NFTA_EXPR_DATA]        = { .type = NLA_NESTED },
3037 };
3038
3039 static int nf_tables_fill_expr_info(struct sk_buff *skb,
3040                                     const struct nft_expr *expr, bool reset)
3041 {
3042         if (nla_put_string(skb, NFTA_EXPR_NAME, expr->ops->type->name))
3043                 goto nla_put_failure;
3044
3045         if (expr->ops->dump) {
3046                 struct nlattr *data = nla_nest_start_noflag(skb,
3047                                                             NFTA_EXPR_DATA);
3048                 if (data == NULL)
3049                         goto nla_put_failure;
3050                 if (expr->ops->dump(skb, expr, reset) < 0)
3051                         goto nla_put_failure;
3052                 nla_nest_end(skb, data);
3053         }
3054
3055         return skb->len;
3056
3057 nla_put_failure:
3058         return -1;
3059 };
3060
3061 int nft_expr_dump(struct sk_buff *skb, unsigned int attr,
3062                   const struct nft_expr *expr, bool reset)
3063 {
3064         struct nlattr *nest;
3065
3066         nest = nla_nest_start_noflag(skb, attr);
3067         if (!nest)
3068                 goto nla_put_failure;
3069         if (nf_tables_fill_expr_info(skb, expr, reset) < 0)
3070                 goto nla_put_failure;
3071         nla_nest_end(skb, nest);
3072         return 0;
3073
3074 nla_put_failure:
3075         return -1;
3076 }
3077
3078 struct nft_expr_info {
3079         const struct nft_expr_ops       *ops;
3080         const struct nlattr             *attr;
3081         struct nlattr                   *tb[NFT_EXPR_MAXATTR + 1];
3082 };
3083
3084 static int nf_tables_expr_parse(const struct nft_ctx *ctx,
3085                                 const struct nlattr *nla,
3086                                 struct nft_expr_info *info)
3087 {
3088         const struct nft_expr_type *type;
3089         const struct nft_expr_ops *ops;
3090         struct nlattr *tb[NFTA_EXPR_MAX + 1];
3091         int err;
3092
3093         err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
3094                                           nft_expr_policy, NULL);
3095         if (err < 0)
3096                 return err;
3097
3098         type = nft_expr_type_get(ctx->net, ctx->family, tb[NFTA_EXPR_NAME]);
3099         if (IS_ERR(type))
3100                 return PTR_ERR(type);
3101
3102         if (tb[NFTA_EXPR_DATA]) {
3103                 err = nla_parse_nested_deprecated(info->tb, type->maxattr,
3104                                                   tb[NFTA_EXPR_DATA],
3105                                                   type->policy, NULL);
3106                 if (err < 0)
3107                         goto err1;
3108         } else
3109                 memset(info->tb, 0, sizeof(info->tb[0]) * (type->maxattr + 1));
3110
3111         if (type->select_ops != NULL) {
3112                 ops = type->select_ops(ctx,
3113                                        (const struct nlattr * const *)info->tb);
3114                 if (IS_ERR(ops)) {
3115                         err = PTR_ERR(ops);
3116 #ifdef CONFIG_MODULES
3117                         if (err == -EAGAIN)
3118                                 if (nft_expr_type_request_module(ctx->net,
3119                                                                  ctx->family,
3120                                                                  tb[NFTA_EXPR_NAME]) != -EAGAIN)
3121                                         err = -ENOENT;
3122 #endif
3123                         goto err1;
3124                 }
3125         } else
3126                 ops = type->ops;
3127
3128         info->attr = nla;
3129         info->ops = ops;
3130
3131         return 0;
3132
3133 err1:
3134         module_put(type->owner);
3135         return err;
3136 }
3137
3138 int nft_expr_inner_parse(const struct nft_ctx *ctx, const struct nlattr *nla,
3139                          struct nft_expr_info *info)
3140 {
3141         struct nlattr *tb[NFTA_EXPR_MAX + 1];
3142         const struct nft_expr_type *type;
3143         int err;
3144
3145         err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
3146                                           nft_expr_policy, NULL);
3147         if (err < 0)
3148                 return err;
3149
3150         if (!tb[NFTA_EXPR_DATA])
3151                 return -EINVAL;
3152
3153         type = __nft_expr_type_get(ctx->family, tb[NFTA_EXPR_NAME]);
3154         if (!type)
3155                 return -ENOENT;
3156
3157         if (!type->inner_ops)
3158                 return -EOPNOTSUPP;
3159
3160         err = nla_parse_nested_deprecated(info->tb, type->maxattr,
3161                                           tb[NFTA_EXPR_DATA],
3162                                           type->policy, NULL);
3163         if (err < 0)
3164                 goto err_nla_parse;
3165
3166         info->attr = nla;
3167         info->ops = type->inner_ops;
3168
3169         return 0;
3170
3171 err_nla_parse:
3172         return err;
3173 }
3174
3175 static int nf_tables_newexpr(const struct nft_ctx *ctx,
3176                              const struct nft_expr_info *expr_info,
3177                              struct nft_expr *expr)
3178 {
3179         const struct nft_expr_ops *ops = expr_info->ops;
3180         int err;
3181
3182         expr->ops = ops;
3183         if (ops->init) {
3184                 err = ops->init(ctx, expr, (const struct nlattr **)expr_info->tb);
3185                 if (err < 0)
3186                         goto err1;
3187         }
3188
3189         return 0;
3190 err1:
3191         expr->ops = NULL;
3192         return err;
3193 }
3194
3195 static void nf_tables_expr_destroy(const struct nft_ctx *ctx,
3196                                    struct nft_expr *expr)
3197 {
3198         const struct nft_expr_type *type = expr->ops->type;
3199
3200         if (expr->ops->destroy)
3201                 expr->ops->destroy(ctx, expr);
3202         module_put(type->owner);
3203 }
3204
3205 static struct nft_expr *nft_expr_init(const struct nft_ctx *ctx,
3206                                       const struct nlattr *nla)
3207 {
3208         struct nft_expr_info expr_info;
3209         struct nft_expr *expr;
3210         struct module *owner;
3211         int err;
3212
3213         err = nf_tables_expr_parse(ctx, nla, &expr_info);
3214         if (err < 0)
3215                 goto err_expr_parse;
3216
3217         err = -EOPNOTSUPP;
3218         if (!(expr_info.ops->type->flags & NFT_EXPR_STATEFUL))
3219                 goto err_expr_stateful;
3220
3221         err = -ENOMEM;
3222         expr = kzalloc(expr_info.ops->size, GFP_KERNEL_ACCOUNT);
3223         if (expr == NULL)
3224                 goto err_expr_stateful;
3225
3226         err = nf_tables_newexpr(ctx, &expr_info, expr);
3227         if (err < 0)
3228                 goto err_expr_new;
3229
3230         return expr;
3231 err_expr_new:
3232         kfree(expr);
3233 err_expr_stateful:
3234         owner = expr_info.ops->type->owner;
3235         if (expr_info.ops->type->release_ops)
3236                 expr_info.ops->type->release_ops(expr_info.ops);
3237
3238         module_put(owner);
3239 err_expr_parse:
3240         return ERR_PTR(err);
3241 }
3242
3243 int nft_expr_clone(struct nft_expr *dst, struct nft_expr *src)
3244 {
3245         int err;
3246
3247         if (src->ops->clone) {
3248                 dst->ops = src->ops;
3249                 err = src->ops->clone(dst, src);
3250                 if (err < 0)
3251                         return err;
3252         } else {
3253                 memcpy(dst, src, src->ops->size);
3254         }
3255
3256         __module_get(src->ops->type->owner);
3257
3258         return 0;
3259 }
3260
3261 void nft_expr_destroy(const struct nft_ctx *ctx, struct nft_expr *expr)
3262 {
3263         nf_tables_expr_destroy(ctx, expr);
3264         kfree(expr);
3265 }
3266
3267 /*
3268  * Rules
3269  */
3270
3271 static struct nft_rule *__nft_rule_lookup(const struct nft_chain *chain,
3272                                           u64 handle)
3273 {
3274         struct nft_rule *rule;
3275
3276         // FIXME: this sucks
3277         list_for_each_entry_rcu(rule, &chain->rules, list) {
3278                 if (handle == rule->handle)
3279                         return rule;
3280         }
3281
3282         return ERR_PTR(-ENOENT);
3283 }
3284
3285 static struct nft_rule *nft_rule_lookup(const struct nft_chain *chain,
3286                                         const struct nlattr *nla)
3287 {
3288         if (nla == NULL)
3289                 return ERR_PTR(-EINVAL);
3290
3291         return __nft_rule_lookup(chain, be64_to_cpu(nla_get_be64(nla)));
3292 }
3293
3294 static const struct nla_policy nft_rule_policy[NFTA_RULE_MAX + 1] = {
3295         [NFTA_RULE_TABLE]       = { .type = NLA_STRING,
3296                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
3297         [NFTA_RULE_CHAIN]       = { .type = NLA_STRING,
3298                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
3299         [NFTA_RULE_HANDLE]      = { .type = NLA_U64 },
3300         [NFTA_RULE_EXPRESSIONS] = { .type = NLA_NESTED },
3301         [NFTA_RULE_COMPAT]      = { .type = NLA_NESTED },
3302         [NFTA_RULE_POSITION]    = { .type = NLA_U64 },
3303         [NFTA_RULE_USERDATA]    = { .type = NLA_BINARY,
3304                                     .len = NFT_USERDATA_MAXLEN },
3305         [NFTA_RULE_ID]          = { .type = NLA_U32 },
3306         [NFTA_RULE_POSITION_ID] = { .type = NLA_U32 },
3307         [NFTA_RULE_CHAIN_ID]    = { .type = NLA_U32 },
3308 };
3309
3310 static int nf_tables_fill_rule_info(struct sk_buff *skb, struct net *net,
3311                                     u32 portid, u32 seq, int event,
3312                                     u32 flags, int family,
3313                                     const struct nft_table *table,
3314                                     const struct nft_chain *chain,
3315                                     const struct nft_rule *rule, u64 handle,
3316                                     bool reset)
3317 {
3318         struct nlmsghdr *nlh;
3319         const struct nft_expr *expr, *next;
3320         struct nlattr *list;
3321         u16 type = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
3322
3323         nlh = nfnl_msg_put(skb, portid, seq, type, flags, family, NFNETLINK_V0,
3324                            nft_base_seq(net));
3325         if (!nlh)
3326                 goto nla_put_failure;
3327
3328         if (nla_put_string(skb, NFTA_RULE_TABLE, table->name))
3329                 goto nla_put_failure;
3330         if (nla_put_string(skb, NFTA_RULE_CHAIN, chain->name))
3331                 goto nla_put_failure;
3332         if (nla_put_be64(skb, NFTA_RULE_HANDLE, cpu_to_be64(rule->handle),
3333                          NFTA_RULE_PAD))
3334                 goto nla_put_failure;
3335
3336         if (event != NFT_MSG_DELRULE && handle) {
3337                 if (nla_put_be64(skb, NFTA_RULE_POSITION, cpu_to_be64(handle),
3338                                  NFTA_RULE_PAD))
3339                         goto nla_put_failure;
3340         }
3341
3342         if (chain->flags & NFT_CHAIN_HW_OFFLOAD)
3343                 nft_flow_rule_stats(chain, rule);
3344
3345         list = nla_nest_start_noflag(skb, NFTA_RULE_EXPRESSIONS);
3346         if (list == NULL)
3347                 goto nla_put_failure;
3348         nft_rule_for_each_expr(expr, next, rule) {
3349                 if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr, reset) < 0)
3350                         goto nla_put_failure;
3351         }
3352         nla_nest_end(skb, list);
3353
3354         if (rule->udata) {
3355                 struct nft_userdata *udata = nft_userdata(rule);
3356                 if (nla_put(skb, NFTA_RULE_USERDATA, udata->len + 1,
3357                             udata->data) < 0)
3358                         goto nla_put_failure;
3359         }
3360
3361         nlmsg_end(skb, nlh);
3362         return 0;
3363
3364 nla_put_failure:
3365         nlmsg_trim(skb, nlh);
3366         return -1;
3367 }
3368
3369 static void nf_tables_rule_notify(const struct nft_ctx *ctx,
3370                                   const struct nft_rule *rule, int event)
3371 {
3372         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
3373         const struct nft_rule *prule;
3374         struct sk_buff *skb;
3375         u64 handle = 0;
3376         u16 flags = 0;
3377         int err;
3378
3379         if (!ctx->report &&
3380             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
3381                 return;
3382
3383         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
3384         if (skb == NULL)
3385                 goto err;
3386
3387         if (event == NFT_MSG_NEWRULE &&
3388             !list_is_first(&rule->list, &ctx->chain->rules) &&
3389             !list_is_last(&rule->list, &ctx->chain->rules)) {
3390                 prule = list_prev_entry(rule, list);
3391                 handle = prule->handle;
3392         }
3393         if (ctx->flags & (NLM_F_APPEND | NLM_F_REPLACE))
3394                 flags |= NLM_F_APPEND;
3395         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
3396                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
3397
3398         err = nf_tables_fill_rule_info(skb, ctx->net, ctx->portid, ctx->seq,
3399                                        event, flags, ctx->family, ctx->table,
3400                                        ctx->chain, rule, handle, false);
3401         if (err < 0) {
3402                 kfree_skb(skb);
3403                 goto err;
3404         }
3405
3406         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
3407         return;
3408 err:
3409         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
3410 }
3411
3412 struct nft_rule_dump_ctx {
3413         char *table;
3414         char *chain;
3415 };
3416
3417 static int __nf_tables_dump_rules(struct sk_buff *skb,
3418                                   unsigned int *idx,
3419                                   struct netlink_callback *cb,
3420                                   const struct nft_table *table,
3421                                   const struct nft_chain *chain,
3422                                   bool reset)
3423 {
3424         struct net *net = sock_net(skb->sk);
3425         const struct nft_rule *rule, *prule;
3426         unsigned int s_idx = cb->args[0];
3427         u64 handle;
3428
3429         prule = NULL;
3430         list_for_each_entry_rcu(rule, &chain->rules, list) {
3431                 if (!nft_is_active(net, rule))
3432                         goto cont_skip;
3433                 if (*idx < s_idx)
3434                         goto cont;
3435                 if (*idx > s_idx) {
3436                         memset(&cb->args[1], 0,
3437                                         sizeof(cb->args) - sizeof(cb->args[0]));
3438                 }
3439                 if (prule)
3440                         handle = prule->handle;
3441                 else
3442                         handle = 0;
3443
3444                 if (nf_tables_fill_rule_info(skb, net, NETLINK_CB(cb->skb).portid,
3445                                         cb->nlh->nlmsg_seq,
3446                                         NFT_MSG_NEWRULE,
3447                                         NLM_F_MULTI | NLM_F_APPEND,
3448                                         table->family,
3449                                         table, chain, rule, handle, reset) < 0)
3450                         return 1;
3451
3452                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
3453 cont:
3454                 prule = rule;
3455 cont_skip:
3456                 (*idx)++;
3457         }
3458         return 0;
3459 }
3460
3461 static int nf_tables_dump_rules(struct sk_buff *skb,
3462                                 struct netlink_callback *cb)
3463 {
3464         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
3465         const struct nft_rule_dump_ctx *ctx = cb->data;
3466         struct nft_table *table;
3467         const struct nft_chain *chain;
3468         unsigned int idx = 0;
3469         struct net *net = sock_net(skb->sk);
3470         int family = nfmsg->nfgen_family;
3471         struct nftables_pernet *nft_net;
3472         bool reset = false;
3473
3474         if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETRULE_RESET)
3475                 reset = true;
3476
3477         rcu_read_lock();
3478         nft_net = nft_pernet(net);
3479         cb->seq = READ_ONCE(nft_net->base_seq);
3480
3481         list_for_each_entry_rcu(table, &nft_net->tables, list) {
3482                 if (family != NFPROTO_UNSPEC && family != table->family)
3483                         continue;
3484
3485                 if (ctx && ctx->table && strcmp(ctx->table, table->name) != 0)
3486                         continue;
3487
3488                 if (ctx && ctx->table && ctx->chain) {
3489                         struct rhlist_head *list, *tmp;
3490
3491                         list = rhltable_lookup(&table->chains_ht, ctx->chain,
3492                                                nft_chain_ht_params);
3493                         if (!list)
3494                                 goto done;
3495
3496                         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
3497                                 if (!nft_is_active(net, chain))
3498                                         continue;
3499                                 __nf_tables_dump_rules(skb, &idx,
3500                                                        cb, table, chain, reset);
3501                                 break;
3502                         }
3503                         goto done;
3504                 }
3505
3506                 list_for_each_entry_rcu(chain, &table->chains, list) {
3507                         if (__nf_tables_dump_rules(skb, &idx,
3508                                                    cb, table, chain, reset))
3509                                 goto done;
3510                 }
3511
3512                 if (ctx && ctx->table)
3513                         break;
3514         }
3515 done:
3516         rcu_read_unlock();
3517
3518         cb->args[0] = idx;
3519         return skb->len;
3520 }
3521
3522 static int nf_tables_dump_rules_start(struct netlink_callback *cb)
3523 {
3524         const struct nlattr * const *nla = cb->data;
3525         struct nft_rule_dump_ctx *ctx = NULL;
3526
3527         if (nla[NFTA_RULE_TABLE] || nla[NFTA_RULE_CHAIN]) {
3528                 ctx = kzalloc(sizeof(*ctx), GFP_ATOMIC);
3529                 if (!ctx)
3530                         return -ENOMEM;
3531
3532                 if (nla[NFTA_RULE_TABLE]) {
3533                         ctx->table = nla_strdup(nla[NFTA_RULE_TABLE],
3534                                                         GFP_ATOMIC);
3535                         if (!ctx->table) {
3536                                 kfree(ctx);
3537                                 return -ENOMEM;
3538                         }
3539                 }
3540                 if (nla[NFTA_RULE_CHAIN]) {
3541                         ctx->chain = nla_strdup(nla[NFTA_RULE_CHAIN],
3542                                                 GFP_ATOMIC);
3543                         if (!ctx->chain) {
3544                                 kfree(ctx->table);
3545                                 kfree(ctx);
3546                                 return -ENOMEM;
3547                         }
3548                 }
3549         }
3550
3551         cb->data = ctx;
3552         return 0;
3553 }
3554
3555 static int nf_tables_dump_rules_done(struct netlink_callback *cb)
3556 {
3557         struct nft_rule_dump_ctx *ctx = cb->data;
3558
3559         if (ctx) {
3560                 kfree(ctx->table);
3561                 kfree(ctx->chain);
3562                 kfree(ctx);
3563         }
3564         return 0;
3565 }
3566
3567 /* called with rcu_read_lock held */
3568 static int nf_tables_getrule(struct sk_buff *skb, const struct nfnl_info *info,
3569                              const struct nlattr * const nla[])
3570 {
3571         struct netlink_ext_ack *extack = info->extack;
3572         u8 genmask = nft_genmask_cur(info->net);
3573         u8 family = info->nfmsg->nfgen_family;
3574         const struct nft_chain *chain;
3575         const struct nft_rule *rule;
3576         struct net *net = info->net;
3577         struct nft_table *table;
3578         struct sk_buff *skb2;
3579         bool reset = false;
3580         int err;
3581
3582         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
3583                 struct netlink_dump_control c = {
3584                         .start= nf_tables_dump_rules_start,
3585                         .dump = nf_tables_dump_rules,
3586                         .done = nf_tables_dump_rules_done,
3587                         .module = THIS_MODULE,
3588                         .data = (void *)nla,
3589                 };
3590
3591                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
3592         }
3593
3594         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask, 0);
3595         if (IS_ERR(table)) {
3596                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3597                 return PTR_ERR(table);
3598         }
3599
3600         chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN], genmask);
3601         if (IS_ERR(chain)) {
3602                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3603                 return PTR_ERR(chain);
3604         }
3605
3606         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
3607         if (IS_ERR(rule)) {
3608                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3609                 return PTR_ERR(rule);
3610         }
3611
3612         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
3613         if (!skb2)
3614                 return -ENOMEM;
3615
3616         if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETRULE_RESET)
3617                 reset = true;
3618
3619         err = nf_tables_fill_rule_info(skb2, net, NETLINK_CB(skb).portid,
3620                                        info->nlh->nlmsg_seq, NFT_MSG_NEWRULE, 0,
3621                                        family, table, chain, rule, 0, reset);
3622         if (err < 0)
3623                 goto err_fill_rule_info;
3624
3625         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
3626
3627 err_fill_rule_info:
3628         kfree_skb(skb2);
3629         return err;
3630 }
3631
3632 void nf_tables_rule_destroy(const struct nft_ctx *ctx, struct nft_rule *rule)
3633 {
3634         struct nft_expr *expr, *next;
3635
3636         /*
3637          * Careful: some expressions might not be initialized in case this
3638          * is called on error from nf_tables_newrule().
3639          */
3640         expr = nft_expr_first(rule);
3641         while (nft_expr_more(rule, expr)) {
3642                 next = nft_expr_next(expr);
3643                 nf_tables_expr_destroy(ctx, expr);
3644                 expr = next;
3645         }
3646         kfree(rule);
3647 }
3648
3649 static void nf_tables_rule_release(const struct nft_ctx *ctx, struct nft_rule *rule)
3650 {
3651         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_RELEASE);
3652         nf_tables_rule_destroy(ctx, rule);
3653 }
3654
3655 int nft_chain_validate(const struct nft_ctx *ctx, const struct nft_chain *chain)
3656 {
3657         struct nft_expr *expr, *last;
3658         const struct nft_data *data;
3659         struct nft_rule *rule;
3660         int err;
3661
3662         if (ctx->level == NFT_JUMP_STACK_SIZE)
3663                 return -EMLINK;
3664
3665         list_for_each_entry(rule, &chain->rules, list) {
3666                 if (!nft_is_active_next(ctx->net, rule))
3667                         continue;
3668
3669                 nft_rule_for_each_expr(expr, last, rule) {
3670                         if (!expr->ops->validate)
3671                                 continue;
3672
3673                         err = expr->ops->validate(ctx, expr, &data);
3674                         if (err < 0)
3675                                 return err;
3676                 }
3677
3678                 cond_resched();
3679         }
3680
3681         return 0;
3682 }
3683 EXPORT_SYMBOL_GPL(nft_chain_validate);
3684
3685 static int nft_table_validate(struct net *net, const struct nft_table *table)
3686 {
3687         struct nft_chain *chain;
3688         struct nft_ctx ctx = {
3689                 .net    = net,
3690                 .family = table->family,
3691         };
3692         int err;
3693
3694         list_for_each_entry(chain, &table->chains, list) {
3695                 if (!nft_is_base_chain(chain))
3696                         continue;
3697
3698                 ctx.chain = chain;
3699                 err = nft_chain_validate(&ctx, chain);
3700                 if (err < 0)
3701                         return err;
3702         }
3703
3704         return 0;
3705 }
3706
3707 int nft_setelem_validate(const struct nft_ctx *ctx, struct nft_set *set,
3708                          const struct nft_set_iter *iter,
3709                          struct nft_set_elem *elem)
3710 {
3711         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
3712         struct nft_ctx *pctx = (struct nft_ctx *)ctx;
3713         const struct nft_data *data;
3714         int err;
3715
3716         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
3717             *nft_set_ext_flags(ext) & NFT_SET_ELEM_INTERVAL_END)
3718                 return 0;
3719
3720         data = nft_set_ext_data(ext);
3721         switch (data->verdict.code) {
3722         case NFT_JUMP:
3723         case NFT_GOTO:
3724                 pctx->level++;
3725                 err = nft_chain_validate(ctx, data->verdict.chain);
3726                 if (err < 0)
3727                         return err;
3728                 pctx->level--;
3729                 break;
3730         default:
3731                 break;
3732         }
3733
3734         return 0;
3735 }
3736
3737 int nft_set_catchall_validate(const struct nft_ctx *ctx, struct nft_set *set)
3738 {
3739         u8 genmask = nft_genmask_next(ctx->net);
3740         struct nft_set_elem_catchall *catchall;
3741         struct nft_set_elem elem;
3742         struct nft_set_ext *ext;
3743         int ret = 0;
3744
3745         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
3746                 ext = nft_set_elem_ext(set, catchall->elem);
3747                 if (!nft_set_elem_active(ext, genmask))
3748                         continue;
3749
3750                 elem.priv = catchall->elem;
3751                 ret = nft_setelem_validate(ctx, set, NULL, &elem);
3752                 if (ret < 0)
3753                         return ret;
3754         }
3755
3756         return ret;
3757 }
3758
3759 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
3760                                              const struct nft_chain *chain,
3761                                              const struct nlattr *nla);
3762
3763 #define NFT_RULE_MAXEXPRS       128
3764
3765 static int nf_tables_newrule(struct sk_buff *skb, const struct nfnl_info *info,
3766                              const struct nlattr * const nla[])
3767 {
3768         struct nftables_pernet *nft_net = nft_pernet(info->net);
3769         struct netlink_ext_ack *extack = info->extack;
3770         unsigned int size, i, n, ulen = 0, usize = 0;
3771         u8 genmask = nft_genmask_next(info->net);
3772         struct nft_rule *rule, *old_rule = NULL;
3773         struct nft_expr_info *expr_info = NULL;
3774         u8 family = info->nfmsg->nfgen_family;
3775         struct nft_flow_rule *flow = NULL;
3776         struct net *net = info->net;
3777         struct nft_userdata *udata;
3778         struct nft_table *table;
3779         struct nft_chain *chain;
3780         struct nft_trans *trans;
3781         u64 handle, pos_handle;
3782         struct nft_expr *expr;
3783         struct nft_ctx ctx;
3784         struct nlattr *tmp;
3785         int err, rem;
3786
3787         lockdep_assert_held(&nft_net->commit_mutex);
3788
3789         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
3790                                  NETLINK_CB(skb).portid);
3791         if (IS_ERR(table)) {
3792                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3793                 return PTR_ERR(table);
3794         }
3795
3796         if (nla[NFTA_RULE_CHAIN]) {
3797                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
3798                                          genmask);
3799                 if (IS_ERR(chain)) {
3800                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3801                         return PTR_ERR(chain);
3802                 }
3803                 if (nft_chain_is_bound(chain))
3804                         return -EOPNOTSUPP;
3805
3806         } else if (nla[NFTA_RULE_CHAIN_ID]) {
3807                 chain = nft_chain_lookup_byid(net, table, nla[NFTA_RULE_CHAIN_ID]);
3808                 if (IS_ERR(chain)) {
3809                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN_ID]);
3810                         return PTR_ERR(chain);
3811                 }
3812         } else {
3813                 return -EINVAL;
3814         }
3815
3816         if (nla[NFTA_RULE_HANDLE]) {
3817                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_HANDLE]));
3818                 rule = __nft_rule_lookup(chain, handle);
3819                 if (IS_ERR(rule)) {
3820                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3821                         return PTR_ERR(rule);
3822                 }
3823
3824                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
3825                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3826                         return -EEXIST;
3827                 }
3828                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
3829                         old_rule = rule;
3830                 else
3831                         return -EOPNOTSUPP;
3832         } else {
3833                 if (!(info->nlh->nlmsg_flags & NLM_F_CREATE) ||
3834                     info->nlh->nlmsg_flags & NLM_F_REPLACE)
3835                         return -EINVAL;
3836                 handle = nf_tables_alloc_handle(table);
3837
3838                 if (chain->use == UINT_MAX)
3839                         return -EOVERFLOW;
3840
3841                 if (nla[NFTA_RULE_POSITION]) {
3842                         pos_handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_POSITION]));
3843                         old_rule = __nft_rule_lookup(chain, pos_handle);
3844                         if (IS_ERR(old_rule)) {
3845                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION]);
3846                                 return PTR_ERR(old_rule);
3847                         }
3848                 } else if (nla[NFTA_RULE_POSITION_ID]) {
3849                         old_rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_POSITION_ID]);
3850                         if (IS_ERR(old_rule)) {
3851                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION_ID]);
3852                                 return PTR_ERR(old_rule);
3853                         }
3854                 }
3855         }
3856
3857         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
3858
3859         n = 0;
3860         size = 0;
3861         if (nla[NFTA_RULE_EXPRESSIONS]) {
3862                 expr_info = kvmalloc_array(NFT_RULE_MAXEXPRS,
3863                                            sizeof(struct nft_expr_info),
3864                                            GFP_KERNEL);
3865                 if (!expr_info)
3866                         return -ENOMEM;
3867
3868                 nla_for_each_nested(tmp, nla[NFTA_RULE_EXPRESSIONS], rem) {
3869                         err = -EINVAL;
3870                         if (nla_type(tmp) != NFTA_LIST_ELEM)
3871                                 goto err_release_expr;
3872                         if (n == NFT_RULE_MAXEXPRS)
3873                                 goto err_release_expr;
3874                         err = nf_tables_expr_parse(&ctx, tmp, &expr_info[n]);
3875                         if (err < 0) {
3876                                 NL_SET_BAD_ATTR(extack, tmp);
3877                                 goto err_release_expr;
3878                         }
3879                         size += expr_info[n].ops->size;
3880                         n++;
3881                 }
3882         }
3883         /* Check for overflow of dlen field */
3884         err = -EFBIG;
3885         if (size >= 1 << 12)
3886                 goto err_release_expr;
3887
3888         if (nla[NFTA_RULE_USERDATA]) {
3889                 ulen = nla_len(nla[NFTA_RULE_USERDATA]);
3890                 if (ulen > 0)
3891                         usize = sizeof(struct nft_userdata) + ulen;
3892         }
3893
3894         err = -ENOMEM;
3895         rule = kzalloc(sizeof(*rule) + size + usize, GFP_KERNEL_ACCOUNT);
3896         if (rule == NULL)
3897                 goto err_release_expr;
3898
3899         nft_activate_next(net, rule);
3900
3901         rule->handle = handle;
3902         rule->dlen   = size;
3903         rule->udata  = ulen ? 1 : 0;
3904
3905         if (ulen) {
3906                 udata = nft_userdata(rule);
3907                 udata->len = ulen - 1;
3908                 nla_memcpy(udata->data, nla[NFTA_RULE_USERDATA], ulen);
3909         }
3910
3911         expr = nft_expr_first(rule);
3912         for (i = 0; i < n; i++) {
3913                 err = nf_tables_newexpr(&ctx, &expr_info[i], expr);
3914                 if (err < 0) {
3915                         NL_SET_BAD_ATTR(extack, expr_info[i].attr);
3916                         goto err_release_rule;
3917                 }
3918
3919                 if (expr_info[i].ops->validate)
3920                         nft_validate_state_update(table, NFT_VALIDATE_NEED);
3921
3922                 expr_info[i].ops = NULL;
3923                 expr = nft_expr_next(expr);
3924         }
3925
3926         if (chain->flags & NFT_CHAIN_HW_OFFLOAD) {
3927                 flow = nft_flow_rule_create(net, rule);
3928                 if (IS_ERR(flow)) {
3929                         err = PTR_ERR(flow);
3930                         goto err_release_rule;
3931                 }
3932         }
3933
3934         if (info->nlh->nlmsg_flags & NLM_F_REPLACE) {
3935                 err = nft_delrule(&ctx, old_rule);
3936                 if (err < 0)
3937                         goto err_destroy_flow_rule;
3938
3939                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
3940                 if (trans == NULL) {
3941                         err = -ENOMEM;
3942                         goto err_destroy_flow_rule;
3943                 }
3944                 list_add_tail_rcu(&rule->list, &old_rule->list);
3945         } else {
3946                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
3947                 if (!trans) {
3948                         err = -ENOMEM;
3949                         goto err_destroy_flow_rule;
3950                 }
3951
3952                 if (info->nlh->nlmsg_flags & NLM_F_APPEND) {
3953                         if (old_rule)
3954                                 list_add_rcu(&rule->list, &old_rule->list);
3955                         else
3956                                 list_add_tail_rcu(&rule->list, &chain->rules);
3957                  } else {
3958                         if (old_rule)
3959                                 list_add_tail_rcu(&rule->list, &old_rule->list);
3960                         else
3961                                 list_add_rcu(&rule->list, &chain->rules);
3962                 }
3963         }
3964         kvfree(expr_info);
3965         chain->use++;
3966
3967         if (flow)
3968                 nft_trans_flow_rule(trans) = flow;
3969
3970         if (table->validate_state == NFT_VALIDATE_DO)
3971                 return nft_table_validate(net, table);
3972
3973         return 0;
3974
3975 err_destroy_flow_rule:
3976         if (flow)
3977                 nft_flow_rule_destroy(flow);
3978 err_release_rule:
3979         nft_rule_expr_deactivate(&ctx, rule, NFT_TRANS_PREPARE_ERROR);
3980         nf_tables_rule_destroy(&ctx, rule);
3981 err_release_expr:
3982         for (i = 0; i < n; i++) {
3983                 if (expr_info[i].ops) {
3984                         module_put(expr_info[i].ops->type->owner);
3985                         if (expr_info[i].ops->type->release_ops)
3986                                 expr_info[i].ops->type->release_ops(expr_info[i].ops);
3987                 }
3988         }
3989         kvfree(expr_info);
3990
3991         return err;
3992 }
3993
3994 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
3995                                              const struct nft_chain *chain,
3996                                              const struct nlattr *nla)
3997 {
3998         struct nftables_pernet *nft_net = nft_pernet(net);
3999         u32 id = ntohl(nla_get_be32(nla));
4000         struct nft_trans *trans;
4001
4002         list_for_each_entry(trans, &nft_net->commit_list, list) {
4003                 if (trans->msg_type == NFT_MSG_NEWRULE &&
4004                     trans->ctx.chain == chain &&
4005                     id == nft_trans_rule_id(trans))
4006                         return nft_trans_rule(trans);
4007         }
4008         return ERR_PTR(-ENOENT);
4009 }
4010
4011 static int nf_tables_delrule(struct sk_buff *skb, const struct nfnl_info *info,
4012                              const struct nlattr * const nla[])
4013 {
4014         struct netlink_ext_ack *extack = info->extack;
4015         u8 genmask = nft_genmask_next(info->net);
4016         u8 family = info->nfmsg->nfgen_family;
4017         struct nft_chain *chain = NULL;
4018         struct net *net = info->net;
4019         struct nft_table *table;
4020         struct nft_rule *rule;
4021         struct nft_ctx ctx;
4022         int err = 0;
4023
4024         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
4025                                  NETLINK_CB(skb).portid);
4026         if (IS_ERR(table)) {
4027                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
4028                 return PTR_ERR(table);
4029         }
4030
4031         if (nla[NFTA_RULE_CHAIN]) {
4032                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
4033                                          genmask);
4034                 if (IS_ERR(chain)) {
4035                         if (PTR_ERR(chain) == -ENOENT &&
4036                             NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYRULE)
4037                                 return 0;
4038
4039                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
4040                         return PTR_ERR(chain);
4041                 }
4042                 if (nft_chain_is_bound(chain))
4043                         return -EOPNOTSUPP;
4044         }
4045
4046         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
4047
4048         if (chain) {
4049                 if (nla[NFTA_RULE_HANDLE]) {
4050                         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
4051                         if (IS_ERR(rule)) {
4052                                 if (PTR_ERR(rule) == -ENOENT &&
4053                                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYRULE)
4054                                         return 0;
4055
4056                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
4057                                 return PTR_ERR(rule);
4058                         }
4059
4060                         err = nft_delrule(&ctx, rule);
4061                 } else if (nla[NFTA_RULE_ID]) {
4062                         rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_ID]);
4063                         if (IS_ERR(rule)) {
4064                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_ID]);
4065                                 return PTR_ERR(rule);
4066                         }
4067
4068                         err = nft_delrule(&ctx, rule);
4069                 } else {
4070                         err = nft_delrule_by_chain(&ctx);
4071                 }
4072         } else {
4073                 list_for_each_entry(chain, &table->chains, list) {
4074                         if (!nft_is_active_next(net, chain))
4075                                 continue;
4076
4077                         ctx.chain = chain;
4078                         err = nft_delrule_by_chain(&ctx);
4079                         if (err < 0)
4080                                 break;
4081                 }
4082         }
4083
4084         return err;
4085 }
4086
4087 /*
4088  * Sets
4089  */
4090 static const struct nft_set_type *nft_set_types[] = {
4091         &nft_set_hash_fast_type,
4092         &nft_set_hash_type,
4093         &nft_set_rhash_type,
4094         &nft_set_bitmap_type,
4095         &nft_set_rbtree_type,
4096 #if defined(CONFIG_X86_64) && !defined(CONFIG_UML)
4097         &nft_set_pipapo_avx2_type,
4098 #endif
4099         &nft_set_pipapo_type,
4100 };
4101
4102 #define NFT_SET_FEATURES        (NFT_SET_INTERVAL | NFT_SET_MAP | \
4103                                  NFT_SET_TIMEOUT | NFT_SET_OBJECT | \
4104                                  NFT_SET_EVAL)
4105
4106 static bool nft_set_ops_candidate(const struct nft_set_type *type, u32 flags)
4107 {
4108         return (flags & type->features) == (flags & NFT_SET_FEATURES);
4109 }
4110
4111 /*
4112  * Select a set implementation based on the data characteristics and the
4113  * given policy. The total memory use might not be known if no size is
4114  * given, in that case the amount of memory per element is used.
4115  */
4116 static const struct nft_set_ops *
4117 nft_select_set_ops(const struct nft_ctx *ctx,
4118                    const struct nlattr * const nla[],
4119                    const struct nft_set_desc *desc)
4120 {
4121         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
4122         const struct nft_set_ops *ops, *bops;
4123         struct nft_set_estimate est, best;
4124         const struct nft_set_type *type;
4125         u32 flags = 0;
4126         int i;
4127
4128         lockdep_assert_held(&nft_net->commit_mutex);
4129         lockdep_nfnl_nft_mutex_not_held();
4130
4131         if (nla[NFTA_SET_FLAGS] != NULL)
4132                 flags = ntohl(nla_get_be32(nla[NFTA_SET_FLAGS]));
4133
4134         bops        = NULL;
4135         best.size   = ~0;
4136         best.lookup = ~0;
4137         best.space  = ~0;
4138
4139         for (i = 0; i < ARRAY_SIZE(nft_set_types); i++) {
4140                 type = nft_set_types[i];
4141                 ops = &type->ops;
4142
4143                 if (!nft_set_ops_candidate(type, flags))
4144                         continue;
4145                 if (!ops->estimate(desc, flags, &est))
4146                         continue;
4147
4148                 switch (desc->policy) {
4149                 case NFT_SET_POL_PERFORMANCE:
4150                         if (est.lookup < best.lookup)
4151                                 break;
4152                         if (est.lookup == best.lookup &&
4153                             est.space < best.space)
4154                                 break;
4155                         continue;
4156                 case NFT_SET_POL_MEMORY:
4157                         if (!desc->size) {
4158                                 if (est.space < best.space)
4159                                         break;
4160                                 if (est.space == best.space &&
4161                                     est.lookup < best.lookup)
4162                                         break;
4163                         } else if (est.size < best.size || !bops) {
4164                                 break;
4165                         }
4166                         continue;
4167                 default:
4168                         break;
4169                 }
4170
4171                 bops = ops;
4172                 best = est;
4173         }
4174
4175         if (bops != NULL)
4176                 return bops;
4177
4178         return ERR_PTR(-EOPNOTSUPP);
4179 }
4180
4181 static const struct nla_policy nft_set_policy[NFTA_SET_MAX + 1] = {
4182         [NFTA_SET_TABLE]                = { .type = NLA_STRING,
4183                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
4184         [NFTA_SET_NAME]                 = { .type = NLA_STRING,
4185                                             .len = NFT_SET_MAXNAMELEN - 1 },
4186         [NFTA_SET_FLAGS]                = { .type = NLA_U32 },
4187         [NFTA_SET_KEY_TYPE]             = { .type = NLA_U32 },
4188         [NFTA_SET_KEY_LEN]              = { .type = NLA_U32 },
4189         [NFTA_SET_DATA_TYPE]            = { .type = NLA_U32 },
4190         [NFTA_SET_DATA_LEN]             = { .type = NLA_U32 },
4191         [NFTA_SET_POLICY]               = { .type = NLA_U32 },
4192         [NFTA_SET_DESC]                 = { .type = NLA_NESTED },
4193         [NFTA_SET_ID]                   = { .type = NLA_U32 },
4194         [NFTA_SET_TIMEOUT]              = { .type = NLA_U64 },
4195         [NFTA_SET_GC_INTERVAL]          = { .type = NLA_U32 },
4196         [NFTA_SET_USERDATA]             = { .type = NLA_BINARY,
4197                                             .len  = NFT_USERDATA_MAXLEN },
4198         [NFTA_SET_OBJ_TYPE]             = { .type = NLA_U32 },
4199         [NFTA_SET_HANDLE]               = { .type = NLA_U64 },
4200         [NFTA_SET_EXPR]                 = { .type = NLA_NESTED },
4201         [NFTA_SET_EXPRESSIONS]          = { .type = NLA_NESTED },
4202 };
4203
4204 static const struct nla_policy nft_set_desc_policy[NFTA_SET_DESC_MAX + 1] = {
4205         [NFTA_SET_DESC_SIZE]            = { .type = NLA_U32 },
4206         [NFTA_SET_DESC_CONCAT]          = { .type = NLA_NESTED },
4207 };
4208
4209 static struct nft_set *nft_set_lookup(const struct nft_table *table,
4210                                       const struct nlattr *nla, u8 genmask)
4211 {
4212         struct nft_set *set;
4213
4214         if (nla == NULL)
4215                 return ERR_PTR(-EINVAL);
4216
4217         list_for_each_entry_rcu(set, &table->sets, list) {
4218                 if (!nla_strcmp(nla, set->name) &&
4219                     nft_active_genmask(set, genmask))
4220                         return set;
4221         }
4222         return ERR_PTR(-ENOENT);
4223 }
4224
4225 static struct nft_set *nft_set_lookup_byhandle(const struct nft_table *table,
4226                                                const struct nlattr *nla,
4227                                                u8 genmask)
4228 {
4229         struct nft_set *set;
4230
4231         list_for_each_entry(set, &table->sets, list) {
4232                 if (be64_to_cpu(nla_get_be64(nla)) == set->handle &&
4233                     nft_active_genmask(set, genmask))
4234                         return set;
4235         }
4236         return ERR_PTR(-ENOENT);
4237 }
4238
4239 static struct nft_set *nft_set_lookup_byid(const struct net *net,
4240                                            const struct nft_table *table,
4241                                            const struct nlattr *nla, u8 genmask)
4242 {
4243         struct nftables_pernet *nft_net = nft_pernet(net);
4244         u32 id = ntohl(nla_get_be32(nla));
4245         struct nft_trans *trans;
4246
4247         list_for_each_entry(trans, &nft_net->commit_list, list) {
4248                 if (trans->msg_type == NFT_MSG_NEWSET) {
4249                         struct nft_set *set = nft_trans_set(trans);
4250
4251                         if (id == nft_trans_set_id(trans) &&
4252                             set->table == table &&
4253                             nft_active_genmask(set, genmask))
4254                                 return set;
4255                 }
4256         }
4257         return ERR_PTR(-ENOENT);
4258 }
4259
4260 struct nft_set *nft_set_lookup_global(const struct net *net,
4261                                       const struct nft_table *table,
4262                                       const struct nlattr *nla_set_name,
4263                                       const struct nlattr *nla_set_id,
4264                                       u8 genmask)
4265 {
4266         struct nft_set *set;
4267
4268         set = nft_set_lookup(table, nla_set_name, genmask);
4269         if (IS_ERR(set)) {
4270                 if (!nla_set_id)
4271                         return set;
4272
4273                 set = nft_set_lookup_byid(net, table, nla_set_id, genmask);
4274         }
4275         return set;
4276 }
4277 EXPORT_SYMBOL_GPL(nft_set_lookup_global);
4278
4279 static int nf_tables_set_alloc_name(struct nft_ctx *ctx, struct nft_set *set,
4280                                     const char *name)
4281 {
4282         const struct nft_set *i;
4283         const char *p;
4284         unsigned long *inuse;
4285         unsigned int n = 0, min = 0;
4286
4287         p = strchr(name, '%');
4288         if (p != NULL) {
4289                 if (p[1] != 'd' || strchr(p + 2, '%'))
4290                         return -EINVAL;
4291
4292                 inuse = (unsigned long *)get_zeroed_page(GFP_KERNEL);
4293                 if (inuse == NULL)
4294                         return -ENOMEM;
4295 cont:
4296                 list_for_each_entry(i, &ctx->table->sets, list) {
4297                         int tmp;
4298
4299                         if (!nft_is_active_next(ctx->net, i))
4300                                 continue;
4301                         if (!sscanf(i->name, name, &tmp))
4302                                 continue;
4303                         if (tmp < min || tmp >= min + BITS_PER_BYTE * PAGE_SIZE)
4304                                 continue;
4305
4306                         set_bit(tmp - min, inuse);
4307                 }
4308
4309                 n = find_first_zero_bit(inuse, BITS_PER_BYTE * PAGE_SIZE);
4310                 if (n >= BITS_PER_BYTE * PAGE_SIZE) {
4311                         min += BITS_PER_BYTE * PAGE_SIZE;
4312                         memset(inuse, 0, PAGE_SIZE);
4313                         goto cont;
4314                 }
4315                 free_page((unsigned long)inuse);
4316         }
4317
4318         set->name = kasprintf(GFP_KERNEL_ACCOUNT, name, min + n);
4319         if (!set->name)
4320                 return -ENOMEM;
4321
4322         list_for_each_entry(i, &ctx->table->sets, list) {
4323                 if (!nft_is_active_next(ctx->net, i))
4324                         continue;
4325                 if (!strcmp(set->name, i->name)) {
4326                         kfree(set->name);
4327                         set->name = NULL;
4328                         return -ENFILE;
4329                 }
4330         }
4331         return 0;
4332 }
4333
4334 int nf_msecs_to_jiffies64(const struct nlattr *nla, u64 *result)
4335 {
4336         u64 ms = be64_to_cpu(nla_get_be64(nla));
4337         u64 max = (u64)(~((u64)0));
4338
4339         max = div_u64(max, NSEC_PER_MSEC);
4340         if (ms >= max)
4341                 return -ERANGE;
4342
4343         ms *= NSEC_PER_MSEC;
4344         *result = nsecs_to_jiffies64(ms);
4345         return 0;
4346 }
4347
4348 __be64 nf_jiffies64_to_msecs(u64 input)
4349 {
4350         return cpu_to_be64(jiffies64_to_msecs(input));
4351 }
4352
4353 static int nf_tables_fill_set_concat(struct sk_buff *skb,
4354                                      const struct nft_set *set)
4355 {
4356         struct nlattr *concat, *field;
4357         int i;
4358
4359         concat = nla_nest_start_noflag(skb, NFTA_SET_DESC_CONCAT);
4360         if (!concat)
4361                 return -ENOMEM;
4362
4363         for (i = 0; i < set->field_count; i++) {
4364                 field = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
4365                 if (!field)
4366                         return -ENOMEM;
4367
4368                 if (nla_put_be32(skb, NFTA_SET_FIELD_LEN,
4369                                  htonl(set->field_len[i])))
4370                         return -ENOMEM;
4371
4372                 nla_nest_end(skb, field);
4373         }
4374
4375         nla_nest_end(skb, concat);
4376
4377         return 0;
4378 }
4379
4380 static int nf_tables_fill_set(struct sk_buff *skb, const struct nft_ctx *ctx,
4381                               const struct nft_set *set, u16 event, u16 flags)
4382 {
4383         u64 timeout = READ_ONCE(set->timeout);
4384         u32 gc_int = READ_ONCE(set->gc_int);
4385         u32 portid = ctx->portid;
4386         struct nlmsghdr *nlh;
4387         struct nlattr *nest;
4388         u32 seq = ctx->seq;
4389         int i;
4390
4391         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
4392         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
4393                            NFNETLINK_V0, nft_base_seq(ctx->net));
4394         if (!nlh)
4395                 goto nla_put_failure;
4396
4397         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
4398                 goto nla_put_failure;
4399         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
4400                 goto nla_put_failure;
4401         if (nla_put_be64(skb, NFTA_SET_HANDLE, cpu_to_be64(set->handle),
4402                          NFTA_SET_PAD))
4403                 goto nla_put_failure;
4404
4405         if (event == NFT_MSG_DELSET) {
4406                 nlmsg_end(skb, nlh);
4407                 return 0;
4408         }
4409
4410         if (set->flags != 0)
4411                 if (nla_put_be32(skb, NFTA_SET_FLAGS, htonl(set->flags)))
4412                         goto nla_put_failure;
4413
4414         if (nla_put_be32(skb, NFTA_SET_KEY_TYPE, htonl(set->ktype)))
4415                 goto nla_put_failure;
4416         if (nla_put_be32(skb, NFTA_SET_KEY_LEN, htonl(set->klen)))
4417                 goto nla_put_failure;
4418         if (set->flags & NFT_SET_MAP) {
4419                 if (nla_put_be32(skb, NFTA_SET_DATA_TYPE, htonl(set->dtype)))
4420                         goto nla_put_failure;
4421                 if (nla_put_be32(skb, NFTA_SET_DATA_LEN, htonl(set->dlen)))
4422                         goto nla_put_failure;
4423         }
4424         if (set->flags & NFT_SET_OBJECT &&
4425             nla_put_be32(skb, NFTA_SET_OBJ_TYPE, htonl(set->objtype)))
4426                 goto nla_put_failure;
4427
4428         if (timeout &&
4429             nla_put_be64(skb, NFTA_SET_TIMEOUT,
4430                          nf_jiffies64_to_msecs(timeout),
4431                          NFTA_SET_PAD))
4432                 goto nla_put_failure;
4433         if (gc_int &&
4434             nla_put_be32(skb, NFTA_SET_GC_INTERVAL, htonl(gc_int)))
4435                 goto nla_put_failure;
4436
4437         if (set->policy != NFT_SET_POL_PERFORMANCE) {
4438                 if (nla_put_be32(skb, NFTA_SET_POLICY, htonl(set->policy)))
4439                         goto nla_put_failure;
4440         }
4441
4442         if (set->udata &&
4443             nla_put(skb, NFTA_SET_USERDATA, set->udlen, set->udata))
4444                 goto nla_put_failure;
4445
4446         nest = nla_nest_start_noflag(skb, NFTA_SET_DESC);
4447         if (!nest)
4448                 goto nla_put_failure;
4449         if (set->size &&
4450             nla_put_be32(skb, NFTA_SET_DESC_SIZE, htonl(set->size)))
4451                 goto nla_put_failure;
4452
4453         if (set->field_count > 1 &&
4454             nf_tables_fill_set_concat(skb, set))
4455                 goto nla_put_failure;
4456
4457         nla_nest_end(skb, nest);
4458
4459         if (set->num_exprs == 1) {
4460                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPR);
4461                 if (nf_tables_fill_expr_info(skb, set->exprs[0], false) < 0)
4462                         goto nla_put_failure;
4463
4464                 nla_nest_end(skb, nest);
4465         } else if (set->num_exprs > 1) {
4466                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPRESSIONS);
4467                 if (nest == NULL)
4468                         goto nla_put_failure;
4469
4470                 for (i = 0; i < set->num_exprs; i++) {
4471                         if (nft_expr_dump(skb, NFTA_LIST_ELEM,
4472                                           set->exprs[i], false) < 0)
4473                                 goto nla_put_failure;
4474                 }
4475                 nla_nest_end(skb, nest);
4476         }
4477
4478         nlmsg_end(skb, nlh);
4479         return 0;
4480
4481 nla_put_failure:
4482         nlmsg_trim(skb, nlh);
4483         return -1;
4484 }
4485
4486 static void nf_tables_set_notify(const struct nft_ctx *ctx,
4487                                  const struct nft_set *set, int event,
4488                                  gfp_t gfp_flags)
4489 {
4490         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
4491         u32 portid = ctx->portid;
4492         struct sk_buff *skb;
4493         u16 flags = 0;
4494         int err;
4495
4496         if (!ctx->report &&
4497             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
4498                 return;
4499
4500         skb = nlmsg_new(NLMSG_GOODSIZE, gfp_flags);
4501         if (skb == NULL)
4502                 goto err;
4503
4504         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
4505                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
4506
4507         err = nf_tables_fill_set(skb, ctx, set, event, flags);
4508         if (err < 0) {
4509                 kfree_skb(skb);
4510                 goto err;
4511         }
4512
4513         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
4514         return;
4515 err:
4516         nfnetlink_set_err(ctx->net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
4517 }
4518
4519 static int nf_tables_dump_sets(struct sk_buff *skb, struct netlink_callback *cb)
4520 {
4521         const struct nft_set *set;
4522         unsigned int idx, s_idx = cb->args[0];
4523         struct nft_table *table, *cur_table = (struct nft_table *)cb->args[2];
4524         struct net *net = sock_net(skb->sk);
4525         struct nft_ctx *ctx = cb->data, ctx_set;
4526         struct nftables_pernet *nft_net;
4527
4528         if (cb->args[1])
4529                 return skb->len;
4530
4531         rcu_read_lock();
4532         nft_net = nft_pernet(net);
4533         cb->seq = READ_ONCE(nft_net->base_seq);
4534
4535         list_for_each_entry_rcu(table, &nft_net->tables, list) {
4536                 if (ctx->family != NFPROTO_UNSPEC &&
4537                     ctx->family != table->family)
4538                         continue;
4539
4540                 if (ctx->table && ctx->table != table)
4541                         continue;
4542
4543                 if (cur_table) {
4544                         if (cur_table != table)
4545                                 continue;
4546
4547                         cur_table = NULL;
4548                 }
4549                 idx = 0;
4550                 list_for_each_entry_rcu(set, &table->sets, list) {
4551                         if (idx < s_idx)
4552                                 goto cont;
4553                         if (!nft_is_active(net, set))
4554                                 goto cont;
4555
4556                         ctx_set = *ctx;
4557                         ctx_set.table = table;
4558                         ctx_set.family = table->family;
4559
4560                         if (nf_tables_fill_set(skb, &ctx_set, set,
4561                                                NFT_MSG_NEWSET,
4562                                                NLM_F_MULTI) < 0) {
4563                                 cb->args[0] = idx;
4564                                 cb->args[2] = (unsigned long) table;
4565                                 goto done;
4566                         }
4567                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
4568 cont:
4569                         idx++;
4570                 }
4571                 if (s_idx)
4572                         s_idx = 0;
4573         }
4574         cb->args[1] = 1;
4575 done:
4576         rcu_read_unlock();
4577         return skb->len;
4578 }
4579
4580 static int nf_tables_dump_sets_start(struct netlink_callback *cb)
4581 {
4582         struct nft_ctx *ctx_dump = NULL;
4583
4584         ctx_dump = kmemdup(cb->data, sizeof(*ctx_dump), GFP_ATOMIC);
4585         if (ctx_dump == NULL)
4586                 return -ENOMEM;
4587
4588         cb->data = ctx_dump;
4589         return 0;
4590 }
4591
4592 static int nf_tables_dump_sets_done(struct netlink_callback *cb)
4593 {
4594         kfree(cb->data);
4595         return 0;
4596 }
4597
4598 /* called with rcu_read_lock held */
4599 static int nf_tables_getset(struct sk_buff *skb, const struct nfnl_info *info,
4600                             const struct nlattr * const nla[])
4601 {
4602         struct netlink_ext_ack *extack = info->extack;
4603         u8 genmask = nft_genmask_cur(info->net);
4604         u8 family = info->nfmsg->nfgen_family;
4605         struct nft_table *table = NULL;
4606         struct net *net = info->net;
4607         const struct nft_set *set;
4608         struct sk_buff *skb2;
4609         struct nft_ctx ctx;
4610         int err;
4611
4612         if (nla[NFTA_SET_TABLE]) {
4613                 table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
4614                                          genmask, 0);
4615                 if (IS_ERR(table)) {
4616                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
4617                         return PTR_ERR(table);
4618                 }
4619         }
4620
4621         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
4622
4623         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
4624                 struct netlink_dump_control c = {
4625                         .start = nf_tables_dump_sets_start,
4626                         .dump = nf_tables_dump_sets,
4627                         .done = nf_tables_dump_sets_done,
4628                         .data = &ctx,
4629                         .module = THIS_MODULE,
4630                 };
4631
4632                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
4633         }
4634
4635         /* Only accept unspec with dump */
4636         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
4637                 return -EAFNOSUPPORT;
4638         if (!nla[NFTA_SET_TABLE])
4639                 return -EINVAL;
4640
4641         set = nft_set_lookup(table, nla[NFTA_SET_NAME], genmask);
4642         if (IS_ERR(set))
4643                 return PTR_ERR(set);
4644
4645         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
4646         if (skb2 == NULL)
4647                 return -ENOMEM;
4648
4649         err = nf_tables_fill_set(skb2, &ctx, set, NFT_MSG_NEWSET, 0);
4650         if (err < 0)
4651                 goto err_fill_set_info;
4652
4653         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
4654
4655 err_fill_set_info:
4656         kfree_skb(skb2);
4657         return err;
4658 }
4659
4660 static const struct nla_policy nft_concat_policy[NFTA_SET_FIELD_MAX + 1] = {
4661         [NFTA_SET_FIELD_LEN]    = { .type = NLA_U32 },
4662 };
4663
4664 static int nft_set_desc_concat_parse(const struct nlattr *attr,
4665                                      struct nft_set_desc *desc)
4666 {
4667         struct nlattr *tb[NFTA_SET_FIELD_MAX + 1];
4668         u32 len;
4669         int err;
4670
4671         if (desc->field_count >= ARRAY_SIZE(desc->field_len))
4672                 return -E2BIG;
4673
4674         err = nla_parse_nested_deprecated(tb, NFTA_SET_FIELD_MAX, attr,
4675                                           nft_concat_policy, NULL);
4676         if (err < 0)
4677                 return err;
4678
4679         if (!tb[NFTA_SET_FIELD_LEN])
4680                 return -EINVAL;
4681
4682         len = ntohl(nla_get_be32(tb[NFTA_SET_FIELD_LEN]));
4683         if (!len || len > U8_MAX)
4684                 return -EINVAL;
4685
4686         desc->field_len[desc->field_count++] = len;
4687
4688         return 0;
4689 }
4690
4691 static int nft_set_desc_concat(struct nft_set_desc *desc,
4692                                const struct nlattr *nla)
4693 {
4694         struct nlattr *attr;
4695         u32 num_regs = 0;
4696         int rem, err, i;
4697
4698         nla_for_each_nested(attr, nla, rem) {
4699                 if (nla_type(attr) != NFTA_LIST_ELEM)
4700                         return -EINVAL;
4701
4702                 err = nft_set_desc_concat_parse(attr, desc);
4703                 if (err < 0)
4704                         return err;
4705         }
4706
4707         for (i = 0; i < desc->field_count; i++)
4708                 num_regs += DIV_ROUND_UP(desc->field_len[i], sizeof(u32));
4709
4710         if (num_regs > NFT_REG32_COUNT)
4711                 return -E2BIG;
4712
4713         return 0;
4714 }
4715
4716 static int nf_tables_set_desc_parse(struct nft_set_desc *desc,
4717                                     const struct nlattr *nla)
4718 {
4719         struct nlattr *da[NFTA_SET_DESC_MAX + 1];
4720         int err;
4721
4722         err = nla_parse_nested_deprecated(da, NFTA_SET_DESC_MAX, nla,
4723                                           nft_set_desc_policy, NULL);
4724         if (err < 0)
4725                 return err;
4726
4727         if (da[NFTA_SET_DESC_SIZE] != NULL)
4728                 desc->size = ntohl(nla_get_be32(da[NFTA_SET_DESC_SIZE]));
4729         if (da[NFTA_SET_DESC_CONCAT])
4730                 err = nft_set_desc_concat(desc, da[NFTA_SET_DESC_CONCAT]);
4731
4732         return err;
4733 }
4734
4735 static int nft_set_expr_alloc(struct nft_ctx *ctx, struct nft_set *set,
4736                               const struct nlattr * const *nla,
4737                               struct nft_expr **exprs, int *num_exprs,
4738                               u32 flags)
4739 {
4740         struct nft_expr *expr;
4741         int err, i;
4742
4743         if (nla[NFTA_SET_EXPR]) {
4744                 expr = nft_set_elem_expr_alloc(ctx, set, nla[NFTA_SET_EXPR]);
4745                 if (IS_ERR(expr)) {
4746                         err = PTR_ERR(expr);
4747                         goto err_set_expr_alloc;
4748                 }
4749                 exprs[0] = expr;
4750                 (*num_exprs)++;
4751         } else if (nla[NFTA_SET_EXPRESSIONS]) {
4752                 struct nlattr *tmp;
4753                 int left;
4754
4755                 if (!(flags & NFT_SET_EXPR)) {
4756                         err = -EINVAL;
4757                         goto err_set_expr_alloc;
4758                 }
4759                 i = 0;
4760                 nla_for_each_nested(tmp, nla[NFTA_SET_EXPRESSIONS], left) {
4761                         if (i == NFT_SET_EXPR_MAX) {
4762                                 err = -E2BIG;
4763                                 goto err_set_expr_alloc;
4764                         }
4765                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
4766                                 err = -EINVAL;
4767                                 goto err_set_expr_alloc;
4768                         }
4769                         expr = nft_set_elem_expr_alloc(ctx, set, tmp);
4770                         if (IS_ERR(expr)) {
4771                                 err = PTR_ERR(expr);
4772                                 goto err_set_expr_alloc;
4773                         }
4774                         exprs[i++] = expr;
4775                         (*num_exprs)++;
4776                 }
4777         }
4778
4779         return 0;
4780
4781 err_set_expr_alloc:
4782         for (i = 0; i < *num_exprs; i++)
4783                 nft_expr_destroy(ctx, exprs[i]);
4784
4785         return err;
4786 }
4787
4788 static bool nft_set_is_same(const struct nft_set *set,
4789                             const struct nft_set_desc *desc,
4790                             struct nft_expr *exprs[], u32 num_exprs, u32 flags)
4791 {
4792         int i;
4793
4794         if (set->ktype != desc->ktype ||
4795             set->dtype != desc->dtype ||
4796             set->flags != flags ||
4797             set->klen != desc->klen ||
4798             set->dlen != desc->dlen ||
4799             set->field_count != desc->field_count ||
4800             set->num_exprs != num_exprs)
4801                 return false;
4802
4803         for (i = 0; i < desc->field_count; i++) {
4804                 if (set->field_len[i] != desc->field_len[i])
4805                         return false;
4806         }
4807
4808         for (i = 0; i < num_exprs; i++) {
4809                 if (set->exprs[i]->ops != exprs[i]->ops)
4810                         return false;
4811         }
4812
4813         return true;
4814 }
4815
4816 static int nf_tables_newset(struct sk_buff *skb, const struct nfnl_info *info,
4817                             const struct nlattr * const nla[])
4818 {
4819         struct netlink_ext_ack *extack = info->extack;
4820         u8 genmask = nft_genmask_next(info->net);
4821         u8 family = info->nfmsg->nfgen_family;
4822         const struct nft_set_ops *ops;
4823         struct net *net = info->net;
4824         struct nft_set_desc desc;
4825         struct nft_table *table;
4826         unsigned char *udata;
4827         struct nft_set *set;
4828         struct nft_ctx ctx;
4829         size_t alloc_size;
4830         int num_exprs = 0;
4831         char *name;
4832         int err, i;
4833         u16 udlen;
4834         u32 flags;
4835         u64 size;
4836
4837         if (nla[NFTA_SET_TABLE] == NULL ||
4838             nla[NFTA_SET_NAME] == NULL ||
4839             nla[NFTA_SET_KEY_LEN] == NULL ||
4840             nla[NFTA_SET_ID] == NULL)
4841                 return -EINVAL;
4842
4843         memset(&desc, 0, sizeof(desc));
4844
4845         desc.ktype = NFT_DATA_VALUE;
4846         if (nla[NFTA_SET_KEY_TYPE] != NULL) {
4847                 desc.ktype = ntohl(nla_get_be32(nla[NFTA_SET_KEY_TYPE]));
4848                 if ((desc.ktype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK)
4849                         return -EINVAL;
4850         }
4851
4852         desc.klen = ntohl(nla_get_be32(nla[NFTA_SET_KEY_LEN]));
4853         if (desc.klen == 0 || desc.klen > NFT_DATA_VALUE_MAXLEN)
4854                 return -EINVAL;
4855
4856         flags = 0;
4857         if (nla[NFTA_SET_FLAGS] != NULL) {
4858                 flags = ntohl(nla_get_be32(nla[NFTA_SET_FLAGS]));
4859                 if (flags & ~(NFT_SET_ANONYMOUS | NFT_SET_CONSTANT |
4860                               NFT_SET_INTERVAL | NFT_SET_TIMEOUT |
4861                               NFT_SET_MAP | NFT_SET_EVAL |
4862                               NFT_SET_OBJECT | NFT_SET_CONCAT | NFT_SET_EXPR))
4863                         return -EOPNOTSUPP;
4864                 /* Only one of these operations is supported */
4865                 if ((flags & (NFT_SET_MAP | NFT_SET_OBJECT)) ==
4866                              (NFT_SET_MAP | NFT_SET_OBJECT))
4867                         return -EOPNOTSUPP;
4868                 if ((flags & (NFT_SET_EVAL | NFT_SET_OBJECT)) ==
4869                              (NFT_SET_EVAL | NFT_SET_OBJECT))
4870                         return -EOPNOTSUPP;
4871         }
4872
4873         desc.dtype = 0;
4874         if (nla[NFTA_SET_DATA_TYPE] != NULL) {
4875                 if (!(flags & NFT_SET_MAP))
4876                         return -EINVAL;
4877
4878                 desc.dtype = ntohl(nla_get_be32(nla[NFTA_SET_DATA_TYPE]));
4879                 if ((desc.dtype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK &&
4880                     desc.dtype != NFT_DATA_VERDICT)
4881                         return -EINVAL;
4882
4883                 if (desc.dtype != NFT_DATA_VERDICT) {
4884                         if (nla[NFTA_SET_DATA_LEN] == NULL)
4885                                 return -EINVAL;
4886                         desc.dlen = ntohl(nla_get_be32(nla[NFTA_SET_DATA_LEN]));
4887                         if (desc.dlen == 0 || desc.dlen > NFT_DATA_VALUE_MAXLEN)
4888                                 return -EINVAL;
4889                 } else
4890                         desc.dlen = sizeof(struct nft_verdict);
4891         } else if (flags & NFT_SET_MAP)
4892                 return -EINVAL;
4893
4894         if (nla[NFTA_SET_OBJ_TYPE] != NULL) {
4895                 if (!(flags & NFT_SET_OBJECT))
4896                         return -EINVAL;
4897
4898                 desc.objtype = ntohl(nla_get_be32(nla[NFTA_SET_OBJ_TYPE]));
4899                 if (desc.objtype == NFT_OBJECT_UNSPEC ||
4900                     desc.objtype > NFT_OBJECT_MAX)
4901                         return -EOPNOTSUPP;
4902         } else if (flags & NFT_SET_OBJECT)
4903                 return -EINVAL;
4904         else
4905                 desc.objtype = NFT_OBJECT_UNSPEC;
4906
4907         desc.timeout = 0;
4908         if (nla[NFTA_SET_TIMEOUT] != NULL) {
4909                 if (!(flags & NFT_SET_TIMEOUT))
4910                         return -EINVAL;
4911
4912                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_TIMEOUT], &desc.timeout);
4913                 if (err)
4914                         return err;
4915         }
4916         desc.gc_int = 0;
4917         if (nla[NFTA_SET_GC_INTERVAL] != NULL) {
4918                 if (!(flags & NFT_SET_TIMEOUT))
4919                         return -EINVAL;
4920                 desc.gc_int = ntohl(nla_get_be32(nla[NFTA_SET_GC_INTERVAL]));
4921         }
4922
4923         desc.policy = NFT_SET_POL_PERFORMANCE;
4924         if (nla[NFTA_SET_POLICY] != NULL)
4925                 desc.policy = ntohl(nla_get_be32(nla[NFTA_SET_POLICY]));
4926
4927         if (nla[NFTA_SET_DESC] != NULL) {
4928                 err = nf_tables_set_desc_parse(&desc, nla[NFTA_SET_DESC]);
4929                 if (err < 0)
4930                         return err;
4931
4932                 if (desc.field_count > 1 && !(flags & NFT_SET_CONCAT))
4933                         return -EINVAL;
4934         } else if (flags & NFT_SET_CONCAT) {
4935                 return -EINVAL;
4936         }
4937
4938         if (nla[NFTA_SET_EXPR] || nla[NFTA_SET_EXPRESSIONS])
4939                 desc.expr = true;
4940
4941         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family, genmask,
4942                                  NETLINK_CB(skb).portid);
4943         if (IS_ERR(table)) {
4944                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
4945                 return PTR_ERR(table);
4946         }
4947
4948         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
4949
4950         set = nft_set_lookup(table, nla[NFTA_SET_NAME], genmask);
4951         if (IS_ERR(set)) {
4952                 if (PTR_ERR(set) != -ENOENT) {
4953                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
4954                         return PTR_ERR(set);
4955                 }
4956         } else {
4957                 struct nft_expr *exprs[NFT_SET_EXPR_MAX] = {};
4958
4959                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
4960                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
4961                         return -EEXIST;
4962                 }
4963                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
4964                         return -EOPNOTSUPP;
4965
4966                 err = nft_set_expr_alloc(&ctx, set, nla, exprs, &num_exprs, flags);
4967                 if (err < 0)
4968                         return err;
4969
4970                 err = 0;
4971                 if (!nft_set_is_same(set, &desc, exprs, num_exprs, flags)) {
4972                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
4973                         err = -EEXIST;
4974                 }
4975
4976                 for (i = 0; i < num_exprs; i++)
4977                         nft_expr_destroy(&ctx, exprs[i]);
4978
4979                 if (err < 0)
4980                         return err;
4981
4982                 return __nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set, &desc);
4983         }
4984
4985         if (!(info->nlh->nlmsg_flags & NLM_F_CREATE))
4986                 return -ENOENT;
4987
4988         ops = nft_select_set_ops(&ctx, nla, &desc);
4989         if (IS_ERR(ops))
4990                 return PTR_ERR(ops);
4991
4992         udlen = 0;
4993         if (nla[NFTA_SET_USERDATA])
4994                 udlen = nla_len(nla[NFTA_SET_USERDATA]);
4995
4996         size = 0;
4997         if (ops->privsize != NULL)
4998                 size = ops->privsize(nla, &desc);
4999         alloc_size = sizeof(*set) + size + udlen;
5000         if (alloc_size < size || alloc_size > INT_MAX)
5001                 return -ENOMEM;
5002         set = kvzalloc(alloc_size, GFP_KERNEL_ACCOUNT);
5003         if (!set)
5004                 return -ENOMEM;
5005
5006         name = nla_strdup(nla[NFTA_SET_NAME], GFP_KERNEL_ACCOUNT);
5007         if (!name) {
5008                 err = -ENOMEM;
5009                 goto err_set_name;
5010         }
5011
5012         err = nf_tables_set_alloc_name(&ctx, set, name);
5013         kfree(name);
5014         if (err < 0)
5015                 goto err_set_name;
5016
5017         udata = NULL;
5018         if (udlen) {
5019                 udata = set->data + size;
5020                 nla_memcpy(udata, nla[NFTA_SET_USERDATA], udlen);
5021         }
5022
5023         INIT_LIST_HEAD(&set->bindings);
5024         INIT_LIST_HEAD(&set->catchall_list);
5025         set->table = table;
5026         write_pnet(&set->net, net);
5027         set->ops = ops;
5028         set->ktype = desc.ktype;
5029         set->klen = desc.klen;
5030         set->dtype = desc.dtype;
5031         set->objtype = desc.objtype;
5032         set->dlen = desc.dlen;
5033         set->flags = flags;
5034         set->size = desc.size;
5035         set->policy = desc.policy;
5036         set->udlen = udlen;
5037         set->udata = udata;
5038         set->timeout = desc.timeout;
5039         set->gc_int = desc.gc_int;
5040
5041         set->field_count = desc.field_count;
5042         for (i = 0; i < desc.field_count; i++)
5043                 set->field_len[i] = desc.field_len[i];
5044
5045         err = ops->init(set, &desc, nla);
5046         if (err < 0)
5047                 goto err_set_init;
5048
5049         err = nft_set_expr_alloc(&ctx, set, nla, set->exprs, &num_exprs, flags);
5050         if (err < 0)
5051                 goto err_set_destroy;
5052
5053         set->num_exprs = num_exprs;
5054         set->handle = nf_tables_alloc_handle(table);
5055         INIT_LIST_HEAD(&set->pending_update);
5056
5057         err = nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set);
5058         if (err < 0)
5059                 goto err_set_expr_alloc;
5060
5061         list_add_tail_rcu(&set->list, &table->sets);
5062         table->use++;
5063         return 0;
5064
5065 err_set_expr_alloc:
5066         for (i = 0; i < set->num_exprs; i++)
5067                 nft_expr_destroy(&ctx, set->exprs[i]);
5068 err_set_destroy:
5069         ops->destroy(&ctx, set);
5070 err_set_init:
5071         kfree(set->name);
5072 err_set_name:
5073         kvfree(set);
5074         return err;
5075 }
5076
5077 static void nft_set_catchall_destroy(const struct nft_ctx *ctx,
5078                                      struct nft_set *set)
5079 {
5080         struct nft_set_elem_catchall *next, *catchall;
5081
5082         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
5083                 list_del_rcu(&catchall->list);
5084                 nf_tables_set_elem_destroy(ctx, set, catchall->elem);
5085                 kfree_rcu(catchall, rcu);
5086         }
5087 }
5088
5089 static void nft_set_destroy(const struct nft_ctx *ctx, struct nft_set *set)
5090 {
5091         int i;
5092
5093         if (WARN_ON(set->use > 0))
5094                 return;
5095
5096         for (i = 0; i < set->num_exprs; i++)
5097                 nft_expr_destroy(ctx, set->exprs[i]);
5098
5099         set->ops->destroy(ctx, set);
5100         nft_set_catchall_destroy(ctx, set);
5101         kfree(set->name);
5102         kvfree(set);
5103 }
5104
5105 static int nf_tables_delset(struct sk_buff *skb, const struct nfnl_info *info,
5106                             const struct nlattr * const nla[])
5107 {
5108         struct netlink_ext_ack *extack = info->extack;
5109         u8 genmask = nft_genmask_next(info->net);
5110         u8 family = info->nfmsg->nfgen_family;
5111         struct net *net = info->net;
5112         const struct nlattr *attr;
5113         struct nft_table *table;
5114         struct nft_set *set;
5115         struct nft_ctx ctx;
5116
5117         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
5118                 return -EAFNOSUPPORT;
5119
5120         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
5121                                  genmask, NETLINK_CB(skb).portid);
5122         if (IS_ERR(table)) {
5123                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
5124                 return PTR_ERR(table);
5125         }
5126
5127         if (nla[NFTA_SET_HANDLE]) {
5128                 attr = nla[NFTA_SET_HANDLE];
5129                 set = nft_set_lookup_byhandle(table, attr, genmask);
5130         } else {
5131                 attr = nla[NFTA_SET_NAME];
5132                 set = nft_set_lookup(table, attr, genmask);
5133         }
5134
5135         if (IS_ERR(set)) {
5136                 if (PTR_ERR(set) == -ENOENT &&
5137                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYSET)
5138                         return 0;
5139
5140                 NL_SET_BAD_ATTR(extack, attr);
5141                 return PTR_ERR(set);
5142         }
5143         if (set->use ||
5144             (info->nlh->nlmsg_flags & NLM_F_NONREC &&
5145              atomic_read(&set->nelems) > 0)) {
5146                 NL_SET_BAD_ATTR(extack, attr);
5147                 return -EBUSY;
5148         }
5149
5150         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
5151
5152         return nft_delset(&ctx, set);
5153 }
5154
5155 static int nft_validate_register_store(const struct nft_ctx *ctx,
5156                                        enum nft_registers reg,
5157                                        const struct nft_data *data,
5158                                        enum nft_data_types type,
5159                                        unsigned int len);
5160
5161 static int nft_setelem_data_validate(const struct nft_ctx *ctx,
5162                                      struct nft_set *set,
5163                                      struct nft_set_elem *elem)
5164 {
5165         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
5166         enum nft_registers dreg;
5167
5168         dreg = nft_type_to_reg(set->dtype);
5169         return nft_validate_register_store(ctx, dreg, nft_set_ext_data(ext),
5170                                            set->dtype == NFT_DATA_VERDICT ?
5171                                            NFT_DATA_VERDICT : NFT_DATA_VALUE,
5172                                            set->dlen);
5173 }
5174
5175 static int nf_tables_bind_check_setelem(const struct nft_ctx *ctx,
5176                                         struct nft_set *set,
5177                                         const struct nft_set_iter *iter,
5178                                         struct nft_set_elem *elem)
5179 {
5180         return nft_setelem_data_validate(ctx, set, elem);
5181 }
5182
5183 static int nft_set_catchall_bind_check(const struct nft_ctx *ctx,
5184                                        struct nft_set *set)
5185 {
5186         u8 genmask = nft_genmask_next(ctx->net);
5187         struct nft_set_elem_catchall *catchall;
5188         struct nft_set_elem elem;
5189         struct nft_set_ext *ext;
5190         int ret = 0;
5191
5192         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5193                 ext = nft_set_elem_ext(set, catchall->elem);
5194                 if (!nft_set_elem_active(ext, genmask))
5195                         continue;
5196
5197                 elem.priv = catchall->elem;
5198                 ret = nft_setelem_data_validate(ctx, set, &elem);
5199                 if (ret < 0)
5200                         break;
5201         }
5202
5203         return ret;
5204 }
5205
5206 int nf_tables_bind_set(const struct nft_ctx *ctx, struct nft_set *set,
5207                        struct nft_set_binding *binding)
5208 {
5209         struct nft_set_binding *i;
5210         struct nft_set_iter iter;
5211
5212         if (set->use == UINT_MAX)
5213                 return -EOVERFLOW;
5214
5215         if (!list_empty(&set->bindings) && nft_set_is_anonymous(set))
5216                 return -EBUSY;
5217
5218         if (binding->flags & NFT_SET_MAP) {
5219                 /* If the set is already bound to the same chain all
5220                  * jumps are already validated for that chain.
5221                  */
5222                 list_for_each_entry(i, &set->bindings, list) {
5223                         if (i->flags & NFT_SET_MAP &&
5224                             i->chain == binding->chain)
5225                                 goto bind;
5226                 }
5227
5228                 iter.genmask    = nft_genmask_next(ctx->net);
5229                 iter.skip       = 0;
5230                 iter.count      = 0;
5231                 iter.err        = 0;
5232                 iter.fn         = nf_tables_bind_check_setelem;
5233
5234                 set->ops->walk(ctx, set, &iter);
5235                 if (!iter.err)
5236                         iter.err = nft_set_catchall_bind_check(ctx, set);
5237
5238                 if (iter.err < 0)
5239                         return iter.err;
5240         }
5241 bind:
5242         binding->chain = ctx->chain;
5243         list_add_tail_rcu(&binding->list, &set->bindings);
5244         nft_set_trans_bind(ctx, set);
5245         set->use++;
5246
5247         return 0;
5248 }
5249 EXPORT_SYMBOL_GPL(nf_tables_bind_set);
5250
5251 static void nf_tables_unbind_set(const struct nft_ctx *ctx, struct nft_set *set,
5252                                  struct nft_set_binding *binding, bool event)
5253 {
5254         list_del_rcu(&binding->list);
5255
5256         if (list_empty(&set->bindings) && nft_set_is_anonymous(set)) {
5257                 list_del_rcu(&set->list);
5258                 if (event)
5259                         nf_tables_set_notify(ctx, set, NFT_MSG_DELSET,
5260                                              GFP_KERNEL);
5261         }
5262 }
5263
5264 static void nft_setelem_data_activate(const struct net *net,
5265                                       const struct nft_set *set,
5266                                       struct nft_set_elem *elem);
5267
5268 static int nft_mapelem_activate(const struct nft_ctx *ctx,
5269                                 struct nft_set *set,
5270                                 const struct nft_set_iter *iter,
5271                                 struct nft_set_elem *elem)
5272 {
5273         nft_setelem_data_activate(ctx->net, set, elem);
5274
5275         return 0;
5276 }
5277
5278 static void nft_map_catchall_activate(const struct nft_ctx *ctx,
5279                                       struct nft_set *set)
5280 {
5281         u8 genmask = nft_genmask_next(ctx->net);
5282         struct nft_set_elem_catchall *catchall;
5283         struct nft_set_elem elem;
5284         struct nft_set_ext *ext;
5285
5286         list_for_each_entry(catchall, &set->catchall_list, list) {
5287                 ext = nft_set_elem_ext(set, catchall->elem);
5288                 if (!nft_set_elem_active(ext, genmask))
5289                         continue;
5290
5291                 elem.priv = catchall->elem;
5292                 nft_setelem_data_activate(ctx->net, set, &elem);
5293                 break;
5294         }
5295 }
5296
5297 static void nft_map_activate(const struct nft_ctx *ctx, struct nft_set *set)
5298 {
5299         struct nft_set_iter iter = {
5300                 .genmask        = nft_genmask_next(ctx->net),
5301                 .fn             = nft_mapelem_activate,
5302         };
5303
5304         set->ops->walk(ctx, set, &iter);
5305         WARN_ON_ONCE(iter.err);
5306
5307         nft_map_catchall_activate(ctx, set);
5308 }
5309
5310 void nf_tables_activate_set(const struct nft_ctx *ctx, struct nft_set *set)
5311 {
5312         if (nft_set_is_anonymous(set)) {
5313                 if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
5314                         nft_map_activate(ctx, set);
5315
5316                 nft_clear(ctx->net, set);
5317         }
5318
5319         set->use++;
5320 }
5321 EXPORT_SYMBOL_GPL(nf_tables_activate_set);
5322
5323 void nf_tables_deactivate_set(const struct nft_ctx *ctx, struct nft_set *set,
5324                               struct nft_set_binding *binding,
5325                               enum nft_trans_phase phase)
5326 {
5327         switch (phase) {
5328         case NFT_TRANS_PREPARE_ERROR:
5329                 nft_set_trans_unbind(ctx, set);
5330                 if (nft_set_is_anonymous(set))
5331                         nft_deactivate_next(ctx->net, set);
5332
5333                 set->use--;
5334                 break;
5335         case NFT_TRANS_PREPARE:
5336                 if (nft_set_is_anonymous(set)) {
5337                         if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
5338                                 nft_map_deactivate(ctx, set);
5339
5340                         nft_deactivate_next(ctx->net, set);
5341                 }
5342                 set->use--;
5343                 return;
5344         case NFT_TRANS_ABORT:
5345         case NFT_TRANS_RELEASE:
5346                 if (nft_set_is_anonymous(set) &&
5347                     set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
5348                         nft_map_deactivate(ctx, set);
5349
5350                 set->use--;
5351                 fallthrough;
5352         default:
5353                 nf_tables_unbind_set(ctx, set, binding,
5354                                      phase == NFT_TRANS_COMMIT);
5355         }
5356 }
5357 EXPORT_SYMBOL_GPL(nf_tables_deactivate_set);
5358
5359 void nf_tables_destroy_set(const struct nft_ctx *ctx, struct nft_set *set)
5360 {
5361         if (list_empty(&set->bindings) && nft_set_is_anonymous(set))
5362                 nft_set_destroy(ctx, set);
5363 }
5364 EXPORT_SYMBOL_GPL(nf_tables_destroy_set);
5365
5366 const struct nft_set_ext_type nft_set_ext_types[] = {
5367         [NFT_SET_EXT_KEY]               = {
5368                 .align  = __alignof__(u32),
5369         },
5370         [NFT_SET_EXT_DATA]              = {
5371                 .align  = __alignof__(u32),
5372         },
5373         [NFT_SET_EXT_EXPRESSIONS]       = {
5374                 .align  = __alignof__(struct nft_set_elem_expr),
5375         },
5376         [NFT_SET_EXT_OBJREF]            = {
5377                 .len    = sizeof(struct nft_object *),
5378                 .align  = __alignof__(struct nft_object *),
5379         },
5380         [NFT_SET_EXT_FLAGS]             = {
5381                 .len    = sizeof(u8),
5382                 .align  = __alignof__(u8),
5383         },
5384         [NFT_SET_EXT_TIMEOUT]           = {
5385                 .len    = sizeof(u64),
5386                 .align  = __alignof__(u64),
5387         },
5388         [NFT_SET_EXT_EXPIRATION]        = {
5389                 .len    = sizeof(u64),
5390                 .align  = __alignof__(u64),
5391         },
5392         [NFT_SET_EXT_USERDATA]          = {
5393                 .len    = sizeof(struct nft_userdata),
5394                 .align  = __alignof__(struct nft_userdata),
5395         },
5396         [NFT_SET_EXT_KEY_END]           = {
5397                 .align  = __alignof__(u32),
5398         },
5399 };
5400
5401 /*
5402  * Set elements
5403  */
5404
5405 static const struct nla_policy nft_set_elem_policy[NFTA_SET_ELEM_MAX + 1] = {
5406         [NFTA_SET_ELEM_KEY]             = { .type = NLA_NESTED },
5407         [NFTA_SET_ELEM_DATA]            = { .type = NLA_NESTED },
5408         [NFTA_SET_ELEM_FLAGS]           = { .type = NLA_U32 },
5409         [NFTA_SET_ELEM_TIMEOUT]         = { .type = NLA_U64 },
5410         [NFTA_SET_ELEM_EXPIRATION]      = { .type = NLA_U64 },
5411         [NFTA_SET_ELEM_USERDATA]        = { .type = NLA_BINARY,
5412                                             .len = NFT_USERDATA_MAXLEN },
5413         [NFTA_SET_ELEM_EXPR]            = { .type = NLA_NESTED },
5414         [NFTA_SET_ELEM_OBJREF]          = { .type = NLA_STRING,
5415                                             .len = NFT_OBJ_MAXNAMELEN - 1 },
5416         [NFTA_SET_ELEM_KEY_END]         = { .type = NLA_NESTED },
5417         [NFTA_SET_ELEM_EXPRESSIONS]     = { .type = NLA_NESTED },
5418 };
5419
5420 static const struct nla_policy nft_set_elem_list_policy[NFTA_SET_ELEM_LIST_MAX + 1] = {
5421         [NFTA_SET_ELEM_LIST_TABLE]      = { .type = NLA_STRING,
5422                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
5423         [NFTA_SET_ELEM_LIST_SET]        = { .type = NLA_STRING,
5424                                             .len = NFT_SET_MAXNAMELEN - 1 },
5425         [NFTA_SET_ELEM_LIST_ELEMENTS]   = { .type = NLA_NESTED },
5426         [NFTA_SET_ELEM_LIST_SET_ID]     = { .type = NLA_U32 },
5427 };
5428
5429 static int nft_set_elem_expr_dump(struct sk_buff *skb,
5430                                   const struct nft_set *set,
5431                                   const struct nft_set_ext *ext)
5432 {
5433         struct nft_set_elem_expr *elem_expr;
5434         u32 size, num_exprs = 0;
5435         struct nft_expr *expr;
5436         struct nlattr *nest;
5437
5438         elem_expr = nft_set_ext_expr(ext);
5439         nft_setelem_expr_foreach(expr, elem_expr, size)
5440                 num_exprs++;
5441
5442         if (num_exprs == 1) {
5443                 expr = nft_setelem_expr_at(elem_expr, 0);
5444                 if (nft_expr_dump(skb, NFTA_SET_ELEM_EXPR, expr, false) < 0)
5445                         return -1;
5446
5447                 return 0;
5448         } else if (num_exprs > 1) {
5449                 nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_EXPRESSIONS);
5450                 if (nest == NULL)
5451                         goto nla_put_failure;
5452
5453                 nft_setelem_expr_foreach(expr, elem_expr, size) {
5454                         expr = nft_setelem_expr_at(elem_expr, size);
5455                         if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr, false) < 0)
5456                                 goto nla_put_failure;
5457                 }
5458                 nla_nest_end(skb, nest);
5459         }
5460         return 0;
5461
5462 nla_put_failure:
5463         return -1;
5464 }
5465
5466 static int nf_tables_fill_setelem(struct sk_buff *skb,
5467                                   const struct nft_set *set,
5468                                   const struct nft_set_elem *elem)
5469 {
5470         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
5471         unsigned char *b = skb_tail_pointer(skb);
5472         struct nlattr *nest;
5473
5474         nest = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
5475         if (nest == NULL)
5476                 goto nla_put_failure;
5477
5478         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
5479             nft_data_dump(skb, NFTA_SET_ELEM_KEY, nft_set_ext_key(ext),
5480                           NFT_DATA_VALUE, set->klen) < 0)
5481                 goto nla_put_failure;
5482
5483         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
5484             nft_data_dump(skb, NFTA_SET_ELEM_KEY_END, nft_set_ext_key_end(ext),
5485                           NFT_DATA_VALUE, set->klen) < 0)
5486                 goto nla_put_failure;
5487
5488         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
5489             nft_data_dump(skb, NFTA_SET_ELEM_DATA, nft_set_ext_data(ext),
5490                           set->dtype == NFT_DATA_VERDICT ? NFT_DATA_VERDICT : NFT_DATA_VALUE,
5491                           set->dlen) < 0)
5492                 goto nla_put_failure;
5493
5494         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS) &&
5495             nft_set_elem_expr_dump(skb, set, ext))
5496                 goto nla_put_failure;
5497
5498         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
5499             nla_put_string(skb, NFTA_SET_ELEM_OBJREF,
5500                            (*nft_set_ext_obj(ext))->key.name) < 0)
5501                 goto nla_put_failure;
5502
5503         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
5504             nla_put_be32(skb, NFTA_SET_ELEM_FLAGS,
5505                          htonl(*nft_set_ext_flags(ext))))
5506                 goto nla_put_failure;
5507
5508         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT) &&
5509             nla_put_be64(skb, NFTA_SET_ELEM_TIMEOUT,
5510                          nf_jiffies64_to_msecs(*nft_set_ext_timeout(ext)),
5511                          NFTA_SET_ELEM_PAD))
5512                 goto nla_put_failure;
5513
5514         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
5515                 u64 expires, now = get_jiffies_64();
5516
5517                 expires = *nft_set_ext_expiration(ext);
5518                 if (time_before64(now, expires))
5519                         expires -= now;
5520                 else
5521                         expires = 0;
5522
5523                 if (nla_put_be64(skb, NFTA_SET_ELEM_EXPIRATION,
5524                                  nf_jiffies64_to_msecs(expires),
5525                                  NFTA_SET_ELEM_PAD))
5526                         goto nla_put_failure;
5527         }
5528
5529         if (nft_set_ext_exists(ext, NFT_SET_EXT_USERDATA)) {
5530                 struct nft_userdata *udata;
5531
5532                 udata = nft_set_ext_userdata(ext);
5533                 if (nla_put(skb, NFTA_SET_ELEM_USERDATA,
5534                             udata->len + 1, udata->data))
5535                         goto nla_put_failure;
5536         }
5537
5538         nla_nest_end(skb, nest);
5539         return 0;
5540
5541 nla_put_failure:
5542         nlmsg_trim(skb, b);
5543         return -EMSGSIZE;
5544 }
5545
5546 struct nft_set_dump_args {
5547         const struct netlink_callback   *cb;
5548         struct nft_set_iter             iter;
5549         struct sk_buff                  *skb;
5550 };
5551
5552 static int nf_tables_dump_setelem(const struct nft_ctx *ctx,
5553                                   struct nft_set *set,
5554                                   const struct nft_set_iter *iter,
5555                                   struct nft_set_elem *elem)
5556 {
5557         struct nft_set_dump_args *args;
5558
5559         args = container_of(iter, struct nft_set_dump_args, iter);
5560         return nf_tables_fill_setelem(args->skb, set, elem);
5561 }
5562
5563 struct nft_set_dump_ctx {
5564         const struct nft_set    *set;
5565         struct nft_ctx          ctx;
5566 };
5567
5568 static int nft_set_catchall_dump(struct net *net, struct sk_buff *skb,
5569                                  const struct nft_set *set)
5570 {
5571         struct nft_set_elem_catchall *catchall;
5572         u8 genmask = nft_genmask_cur(net);
5573         struct nft_set_elem elem;
5574         struct nft_set_ext *ext;
5575         int ret = 0;
5576
5577         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5578                 ext = nft_set_elem_ext(set, catchall->elem);
5579                 if (!nft_set_elem_active(ext, genmask) ||
5580                     nft_set_elem_expired(ext))
5581                         continue;
5582
5583                 elem.priv = catchall->elem;
5584                 ret = nf_tables_fill_setelem(skb, set, &elem);
5585                 break;
5586         }
5587
5588         return ret;
5589 }
5590
5591 static int nf_tables_dump_set(struct sk_buff *skb, struct netlink_callback *cb)
5592 {
5593         struct nft_set_dump_ctx *dump_ctx = cb->data;
5594         struct net *net = sock_net(skb->sk);
5595         struct nftables_pernet *nft_net;
5596         struct nft_table *table;
5597         struct nft_set *set;
5598         struct nft_set_dump_args args;
5599         bool set_found = false;
5600         struct nlmsghdr *nlh;
5601         struct nlattr *nest;
5602         u32 portid, seq;
5603         int event;
5604
5605         rcu_read_lock();
5606         nft_net = nft_pernet(net);
5607         cb->seq = READ_ONCE(nft_net->base_seq);
5608
5609         list_for_each_entry_rcu(table, &nft_net->tables, list) {
5610                 if (dump_ctx->ctx.family != NFPROTO_UNSPEC &&
5611                     dump_ctx->ctx.family != table->family)
5612                         continue;
5613
5614                 if (table != dump_ctx->ctx.table)
5615                         continue;
5616
5617                 list_for_each_entry_rcu(set, &table->sets, list) {
5618                         if (set == dump_ctx->set) {
5619                                 set_found = true;
5620                                 break;
5621                         }
5622                 }
5623                 break;
5624         }
5625
5626         if (!set_found) {
5627                 rcu_read_unlock();
5628                 return -ENOENT;
5629         }
5630
5631         event  = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWSETELEM);
5632         portid = NETLINK_CB(cb->skb).portid;
5633         seq    = cb->nlh->nlmsg_seq;
5634
5635         nlh = nfnl_msg_put(skb, portid, seq, event, NLM_F_MULTI,
5636                            table->family, NFNETLINK_V0, nft_base_seq(net));
5637         if (!nlh)
5638                 goto nla_put_failure;
5639
5640         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_TABLE, table->name))
5641                 goto nla_put_failure;
5642         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_SET, set->name))
5643                 goto nla_put_failure;
5644
5645         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
5646         if (nest == NULL)
5647                 goto nla_put_failure;
5648
5649         args.cb                 = cb;
5650         args.skb                = skb;
5651         args.iter.genmask       = nft_genmask_cur(net);
5652         args.iter.skip          = cb->args[0];
5653         args.iter.count         = 0;
5654         args.iter.err           = 0;
5655         args.iter.fn            = nf_tables_dump_setelem;
5656         set->ops->walk(&dump_ctx->ctx, set, &args.iter);
5657
5658         if (!args.iter.err && args.iter.count == cb->args[0])
5659                 args.iter.err = nft_set_catchall_dump(net, skb, set);
5660         rcu_read_unlock();
5661
5662         nla_nest_end(skb, nest);
5663         nlmsg_end(skb, nlh);
5664
5665         if (args.iter.err && args.iter.err != -EMSGSIZE)
5666                 return args.iter.err;
5667         if (args.iter.count == cb->args[0])
5668                 return 0;
5669
5670         cb->args[0] = args.iter.count;
5671         return skb->len;
5672
5673 nla_put_failure:
5674         rcu_read_unlock();
5675         return -ENOSPC;
5676 }
5677
5678 static int nf_tables_dump_set_start(struct netlink_callback *cb)
5679 {
5680         struct nft_set_dump_ctx *dump_ctx = cb->data;
5681
5682         cb->data = kmemdup(dump_ctx, sizeof(*dump_ctx), GFP_ATOMIC);
5683
5684         return cb->data ? 0 : -ENOMEM;
5685 }
5686
5687 static int nf_tables_dump_set_done(struct netlink_callback *cb)
5688 {
5689         kfree(cb->data);
5690         return 0;
5691 }
5692
5693 static int nf_tables_fill_setelem_info(struct sk_buff *skb,
5694                                        const struct nft_ctx *ctx, u32 seq,
5695                                        u32 portid, int event, u16 flags,
5696                                        const struct nft_set *set,
5697                                        const struct nft_set_elem *elem)
5698 {
5699         struct nlmsghdr *nlh;
5700         struct nlattr *nest;
5701         int err;
5702
5703         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
5704         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
5705                            NFNETLINK_V0, nft_base_seq(ctx->net));
5706         if (!nlh)
5707                 goto nla_put_failure;
5708
5709         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
5710                 goto nla_put_failure;
5711         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
5712                 goto nla_put_failure;
5713
5714         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
5715         if (nest == NULL)
5716                 goto nla_put_failure;
5717
5718         err = nf_tables_fill_setelem(skb, set, elem);
5719         if (err < 0)
5720                 goto nla_put_failure;
5721
5722         nla_nest_end(skb, nest);
5723
5724         nlmsg_end(skb, nlh);
5725         return 0;
5726
5727 nla_put_failure:
5728         nlmsg_trim(skb, nlh);
5729         return -1;
5730 }
5731
5732 static int nft_setelem_parse_flags(const struct nft_set *set,
5733                                    const struct nlattr *attr, u32 *flags)
5734 {
5735         if (attr == NULL)
5736                 return 0;
5737
5738         *flags = ntohl(nla_get_be32(attr));
5739         if (*flags & ~(NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
5740                 return -EOPNOTSUPP;
5741         if (!(set->flags & NFT_SET_INTERVAL) &&
5742             *flags & NFT_SET_ELEM_INTERVAL_END)
5743                 return -EINVAL;
5744         if ((*flags & (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL)) ==
5745             (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
5746                 return -EINVAL;
5747
5748         return 0;
5749 }
5750
5751 static int nft_setelem_parse_key(struct nft_ctx *ctx, struct nft_set *set,
5752                                  struct nft_data *key, struct nlattr *attr)
5753 {
5754         struct nft_data_desc desc = {
5755                 .type   = NFT_DATA_VALUE,
5756                 .size   = NFT_DATA_VALUE_MAXLEN,
5757                 .len    = set->klen,
5758         };
5759
5760         return nft_data_init(ctx, key, &desc, attr);
5761 }
5762
5763 static int nft_setelem_parse_data(struct nft_ctx *ctx, struct nft_set *set,
5764                                   struct nft_data_desc *desc,
5765                                   struct nft_data *data,
5766                                   struct nlattr *attr)
5767 {
5768         u32 dtype;
5769
5770         if (set->dtype == NFT_DATA_VERDICT)
5771                 dtype = NFT_DATA_VERDICT;
5772         else
5773                 dtype = NFT_DATA_VALUE;
5774
5775         desc->type = dtype;
5776         desc->size = NFT_DATA_VALUE_MAXLEN;
5777         desc->len = set->dlen;
5778         desc->flags = NFT_DATA_DESC_SETELEM;
5779
5780         return nft_data_init(ctx, data, desc, attr);
5781 }
5782
5783 static void *nft_setelem_catchall_get(const struct net *net,
5784                                       const struct nft_set *set)
5785 {
5786         struct nft_set_elem_catchall *catchall;
5787         u8 genmask = nft_genmask_cur(net);
5788         struct nft_set_ext *ext;
5789         void *priv = NULL;
5790
5791         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5792                 ext = nft_set_elem_ext(set, catchall->elem);
5793                 if (!nft_set_elem_active(ext, genmask) ||
5794                     nft_set_elem_expired(ext))
5795                         continue;
5796
5797                 priv = catchall->elem;
5798                 break;
5799         }
5800
5801         return priv;
5802 }
5803
5804 static int nft_setelem_get(struct nft_ctx *ctx, struct nft_set *set,
5805                            struct nft_set_elem *elem, u32 flags)
5806 {
5807         void *priv;
5808
5809         if (!(flags & NFT_SET_ELEM_CATCHALL)) {
5810                 priv = set->ops->get(ctx->net, set, elem, flags);
5811                 if (IS_ERR(priv))
5812                         return PTR_ERR(priv);
5813         } else {
5814                 priv = nft_setelem_catchall_get(ctx->net, set);
5815                 if (!priv)
5816                         return -ENOENT;
5817         }
5818         elem->priv = priv;
5819
5820         return 0;
5821 }
5822
5823 static int nft_get_set_elem(struct nft_ctx *ctx, struct nft_set *set,
5824                             const struct nlattr *attr)
5825 {
5826         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
5827         struct nft_set_elem elem;
5828         struct sk_buff *skb;
5829         uint32_t flags = 0;
5830         int err;
5831
5832         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
5833                                           nft_set_elem_policy, NULL);
5834         if (err < 0)
5835                 return err;
5836
5837         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
5838         if (err < 0)
5839                 return err;
5840
5841         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
5842                 return -EINVAL;
5843
5844         if (nla[NFTA_SET_ELEM_KEY]) {
5845                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
5846                                             nla[NFTA_SET_ELEM_KEY]);
5847                 if (err < 0)
5848                         return err;
5849         }
5850
5851         if (nla[NFTA_SET_ELEM_KEY_END]) {
5852                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
5853                                             nla[NFTA_SET_ELEM_KEY_END]);
5854                 if (err < 0)
5855                         return err;
5856         }
5857
5858         err = nft_setelem_get(ctx, set, &elem, flags);
5859         if (err < 0)
5860                 return err;
5861
5862         err = -ENOMEM;
5863         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_ATOMIC);
5864         if (skb == NULL)
5865                 return err;
5866
5867         err = nf_tables_fill_setelem_info(skb, ctx, ctx->seq, ctx->portid,
5868                                           NFT_MSG_NEWSETELEM, 0, set, &elem);
5869         if (err < 0)
5870                 goto err_fill_setelem;
5871
5872         return nfnetlink_unicast(skb, ctx->net, ctx->portid);
5873
5874 err_fill_setelem:
5875         kfree_skb(skb);
5876         return err;
5877 }
5878
5879 /* called with rcu_read_lock held */
5880 static int nf_tables_getsetelem(struct sk_buff *skb,
5881                                 const struct nfnl_info *info,
5882                                 const struct nlattr * const nla[])
5883 {
5884         struct netlink_ext_ack *extack = info->extack;
5885         u8 genmask = nft_genmask_cur(info->net);
5886         u8 family = info->nfmsg->nfgen_family;
5887         struct net *net = info->net;
5888         struct nft_table *table;
5889         struct nft_set *set;
5890         struct nlattr *attr;
5891         struct nft_ctx ctx;
5892         int rem, err = 0;
5893
5894         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
5895                                  genmask, 0);
5896         if (IS_ERR(table)) {
5897                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
5898                 return PTR_ERR(table);
5899         }
5900
5901         set = nft_set_lookup(table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
5902         if (IS_ERR(set))
5903                 return PTR_ERR(set);
5904
5905         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
5906
5907         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
5908                 struct netlink_dump_control c = {
5909                         .start = nf_tables_dump_set_start,
5910                         .dump = nf_tables_dump_set,
5911                         .done = nf_tables_dump_set_done,
5912                         .module = THIS_MODULE,
5913                 };
5914                 struct nft_set_dump_ctx dump_ctx = {
5915                         .set = set,
5916                         .ctx = ctx,
5917                 };
5918
5919                 c.data = &dump_ctx;
5920                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
5921         }
5922
5923         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
5924                 return -EINVAL;
5925
5926         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
5927                 err = nft_get_set_elem(&ctx, set, attr);
5928                 if (err < 0) {
5929                         NL_SET_BAD_ATTR(extack, attr);
5930                         break;
5931                 }
5932         }
5933
5934         return err;
5935 }
5936
5937 static void nf_tables_setelem_notify(const struct nft_ctx *ctx,
5938                                      const struct nft_set *set,
5939                                      const struct nft_set_elem *elem,
5940                                      int event)
5941 {
5942         struct nftables_pernet *nft_net;
5943         struct net *net = ctx->net;
5944         u32 portid = ctx->portid;
5945         struct sk_buff *skb;
5946         u16 flags = 0;
5947         int err;
5948
5949         if (!ctx->report && !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
5950                 return;
5951
5952         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
5953         if (skb == NULL)
5954                 goto err;
5955
5956         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
5957                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
5958
5959         err = nf_tables_fill_setelem_info(skb, ctx, 0, portid, event, flags,
5960                                           set, elem);
5961         if (err < 0) {
5962                 kfree_skb(skb);
5963                 goto err;
5964         }
5965
5966         nft_net = nft_pernet(net);
5967         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
5968         return;
5969 err:
5970         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
5971 }
5972
5973 static struct nft_trans *nft_trans_elem_alloc(struct nft_ctx *ctx,
5974                                               int msg_type,
5975                                               struct nft_set *set)
5976 {
5977         struct nft_trans *trans;
5978
5979         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_elem));
5980         if (trans == NULL)
5981                 return NULL;
5982
5983         nft_trans_elem_set(trans) = set;
5984         return trans;
5985 }
5986
5987 struct nft_expr *nft_set_elem_expr_alloc(const struct nft_ctx *ctx,
5988                                          const struct nft_set *set,
5989                                          const struct nlattr *attr)
5990 {
5991         struct nft_expr *expr;
5992         int err;
5993
5994         expr = nft_expr_init(ctx, attr);
5995         if (IS_ERR(expr))
5996                 return expr;
5997
5998         err = -EOPNOTSUPP;
5999         if (expr->ops->type->flags & NFT_EXPR_GC) {
6000                 if (set->flags & NFT_SET_TIMEOUT)
6001                         goto err_set_elem_expr;
6002                 if (!set->ops->gc_init)
6003                         goto err_set_elem_expr;
6004                 set->ops->gc_init(set);
6005         }
6006
6007         return expr;
6008
6009 err_set_elem_expr:
6010         nft_expr_destroy(ctx, expr);
6011         return ERR_PTR(err);
6012 }
6013
6014 static int nft_set_ext_check(const struct nft_set_ext_tmpl *tmpl, u8 id, u32 len)
6015 {
6016         len += nft_set_ext_types[id].len;
6017         if (len > tmpl->ext_len[id] ||
6018             len > U8_MAX)
6019                 return -1;
6020
6021         return 0;
6022 }
6023
6024 static int nft_set_ext_memcpy(const struct nft_set_ext_tmpl *tmpl, u8 id,
6025                               void *to, const void *from, u32 len)
6026 {
6027         if (nft_set_ext_check(tmpl, id, len) < 0)
6028                 return -1;
6029
6030         memcpy(to, from, len);
6031
6032         return 0;
6033 }
6034
6035 void *nft_set_elem_init(const struct nft_set *set,
6036                         const struct nft_set_ext_tmpl *tmpl,
6037                         const u32 *key, const u32 *key_end,
6038                         const u32 *data, u64 timeout, u64 expiration, gfp_t gfp)
6039 {
6040         struct nft_set_ext *ext;
6041         void *elem;
6042
6043         elem = kzalloc(set->ops->elemsize + tmpl->len, gfp);
6044         if (elem == NULL)
6045                 return ERR_PTR(-ENOMEM);
6046
6047         ext = nft_set_elem_ext(set, elem);
6048         nft_set_ext_init(ext, tmpl);
6049
6050         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
6051             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY,
6052                                nft_set_ext_key(ext), key, set->klen) < 0)
6053                 goto err_ext_check;
6054
6055         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
6056             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY_END,
6057                                nft_set_ext_key_end(ext), key_end, set->klen) < 0)
6058                 goto err_ext_check;
6059
6060         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
6061             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_DATA,
6062                                nft_set_ext_data(ext), data, set->dlen) < 0)
6063                 goto err_ext_check;
6064
6065         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
6066                 *nft_set_ext_expiration(ext) = get_jiffies_64() + expiration;
6067                 if (expiration == 0)
6068                         *nft_set_ext_expiration(ext) += timeout;
6069         }
6070         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT))
6071                 *nft_set_ext_timeout(ext) = timeout;
6072
6073         return elem;
6074
6075 err_ext_check:
6076         kfree(elem);
6077
6078         return ERR_PTR(-EINVAL);
6079 }
6080
6081 static void __nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
6082                                         struct nft_expr *expr)
6083 {
6084         if (expr->ops->destroy_clone) {
6085                 expr->ops->destroy_clone(ctx, expr);
6086                 module_put(expr->ops->type->owner);
6087         } else {
6088                 nf_tables_expr_destroy(ctx, expr);
6089         }
6090 }
6091
6092 static void nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
6093                                       struct nft_set_elem_expr *elem_expr)
6094 {
6095         struct nft_expr *expr;
6096         u32 size;
6097
6098         nft_setelem_expr_foreach(expr, elem_expr, size)
6099                 __nft_set_elem_expr_destroy(ctx, expr);
6100 }
6101
6102 /* Drop references and destroy. Called from gc, dynset and abort path. */
6103 void nft_set_elem_destroy(const struct nft_set *set, void *elem,
6104                           bool destroy_expr)
6105 {
6106         struct nft_set_ext *ext = nft_set_elem_ext(set, elem);
6107         struct nft_ctx ctx = {
6108                 .net    = read_pnet(&set->net),
6109                 .family = set->table->family,
6110         };
6111
6112         nft_data_release(nft_set_ext_key(ext), NFT_DATA_VALUE);
6113         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
6114                 nft_data_release(nft_set_ext_data(ext), set->dtype);
6115         if (destroy_expr && nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
6116                 nft_set_elem_expr_destroy(&ctx, nft_set_ext_expr(ext));
6117
6118         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
6119                 (*nft_set_ext_obj(ext))->use--;
6120         kfree(elem);
6121 }
6122 EXPORT_SYMBOL_GPL(nft_set_elem_destroy);
6123
6124 /* Destroy element. References have been already dropped in the preparation
6125  * path via nft_setelem_data_deactivate().
6126  */
6127 void nf_tables_set_elem_destroy(const struct nft_ctx *ctx,
6128                                 const struct nft_set *set, void *elem)
6129 {
6130         struct nft_set_ext *ext = nft_set_elem_ext(set, elem);
6131
6132         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
6133                 nft_set_elem_expr_destroy(ctx, nft_set_ext_expr(ext));
6134
6135         kfree(elem);
6136 }
6137
6138 int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set,
6139                             struct nft_expr *expr_array[])
6140 {
6141         struct nft_expr *expr;
6142         int err, i, k;
6143
6144         for (i = 0; i < set->num_exprs; i++) {
6145                 expr = kzalloc(set->exprs[i]->ops->size, GFP_KERNEL_ACCOUNT);
6146                 if (!expr)
6147                         goto err_expr;
6148
6149                 err = nft_expr_clone(expr, set->exprs[i]);
6150                 if (err < 0) {
6151                         kfree(expr);
6152                         goto err_expr;
6153                 }
6154                 expr_array[i] = expr;
6155         }
6156
6157         return 0;
6158
6159 err_expr:
6160         for (k = i - 1; k >= 0; k--)
6161                 nft_expr_destroy(ctx, expr_array[k]);
6162
6163         return -ENOMEM;
6164 }
6165
6166 static int nft_set_elem_expr_setup(struct nft_ctx *ctx,
6167                                    const struct nft_set_ext_tmpl *tmpl,
6168                                    const struct nft_set_ext *ext,
6169                                    struct nft_expr *expr_array[],
6170                                    u32 num_exprs)
6171 {
6172         struct nft_set_elem_expr *elem_expr = nft_set_ext_expr(ext);
6173         u32 len = sizeof(struct nft_set_elem_expr);
6174         struct nft_expr *expr;
6175         int i, err;
6176
6177         if (num_exprs == 0)
6178                 return 0;
6179
6180         for (i = 0; i < num_exprs; i++)
6181                 len += expr_array[i]->ops->size;
6182
6183         if (nft_set_ext_check(tmpl, NFT_SET_EXT_EXPRESSIONS, len) < 0)
6184                 return -EINVAL;
6185
6186         for (i = 0; i < num_exprs; i++) {
6187                 expr = nft_setelem_expr_at(elem_expr, elem_expr->size);
6188                 err = nft_expr_clone(expr, expr_array[i]);
6189                 if (err < 0)
6190                         goto err_elem_expr_setup;
6191
6192                 elem_expr->size += expr_array[i]->ops->size;
6193                 nft_expr_destroy(ctx, expr_array[i]);
6194                 expr_array[i] = NULL;
6195         }
6196
6197         return 0;
6198
6199 err_elem_expr_setup:
6200         for (; i < num_exprs; i++) {
6201                 nft_expr_destroy(ctx, expr_array[i]);
6202                 expr_array[i] = NULL;
6203         }
6204
6205         return -ENOMEM;
6206 }
6207
6208 struct nft_set_ext *nft_set_catchall_lookup(const struct net *net,
6209                                             const struct nft_set *set)
6210 {
6211         struct nft_set_elem_catchall *catchall;
6212         u8 genmask = nft_genmask_cur(net);
6213         struct nft_set_ext *ext;
6214
6215         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
6216                 ext = nft_set_elem_ext(set, catchall->elem);
6217                 if (nft_set_elem_active(ext, genmask) &&
6218                     !nft_set_elem_expired(ext))
6219                         return ext;
6220         }
6221
6222         return NULL;
6223 }
6224 EXPORT_SYMBOL_GPL(nft_set_catchall_lookup);
6225
6226 void *nft_set_catchall_gc(const struct nft_set *set)
6227 {
6228         struct nft_set_elem_catchall *catchall, *next;
6229         struct nft_set_ext *ext;
6230         void *elem = NULL;
6231
6232         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
6233                 ext = nft_set_elem_ext(set, catchall->elem);
6234
6235                 if (!nft_set_elem_expired(ext) ||
6236                     nft_set_elem_mark_busy(ext))
6237                         continue;
6238
6239                 elem = catchall->elem;
6240                 list_del_rcu(&catchall->list);
6241                 kfree_rcu(catchall, rcu);
6242                 break;
6243         }
6244
6245         return elem;
6246 }
6247 EXPORT_SYMBOL_GPL(nft_set_catchall_gc);
6248
6249 static int nft_setelem_catchall_insert(const struct net *net,
6250                                        struct nft_set *set,
6251                                        const struct nft_set_elem *elem,
6252                                        struct nft_set_ext **pext)
6253 {
6254         struct nft_set_elem_catchall *catchall;
6255         u8 genmask = nft_genmask_next(net);
6256         struct nft_set_ext *ext;
6257
6258         list_for_each_entry(catchall, &set->catchall_list, list) {
6259                 ext = nft_set_elem_ext(set, catchall->elem);
6260                 if (nft_set_elem_active(ext, genmask)) {
6261                         *pext = ext;
6262                         return -EEXIST;
6263                 }
6264         }
6265
6266         catchall = kmalloc(sizeof(*catchall), GFP_KERNEL);
6267         if (!catchall)
6268                 return -ENOMEM;
6269
6270         catchall->elem = elem->priv;
6271         list_add_tail_rcu(&catchall->list, &set->catchall_list);
6272
6273         return 0;
6274 }
6275
6276 static int nft_setelem_insert(const struct net *net,
6277                               struct nft_set *set,
6278                               const struct nft_set_elem *elem,
6279                               struct nft_set_ext **ext, unsigned int flags)
6280 {
6281         int ret;
6282
6283         if (flags & NFT_SET_ELEM_CATCHALL)
6284                 ret = nft_setelem_catchall_insert(net, set, elem, ext);
6285         else
6286                 ret = set->ops->insert(net, set, elem, ext);
6287
6288         return ret;
6289 }
6290
6291 static bool nft_setelem_is_catchall(const struct nft_set *set,
6292                                     const struct nft_set_elem *elem)
6293 {
6294         struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6295
6296         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
6297             *nft_set_ext_flags(ext) & NFT_SET_ELEM_CATCHALL)
6298                 return true;
6299
6300         return false;
6301 }
6302
6303 static void nft_setelem_activate(struct net *net, struct nft_set *set,
6304                                  struct nft_set_elem *elem)
6305 {
6306         struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6307
6308         if (nft_setelem_is_catchall(set, elem)) {
6309                 nft_set_elem_change_active(net, set, ext);
6310                 nft_set_elem_clear_busy(ext);
6311         } else {
6312                 set->ops->activate(net, set, elem);
6313         }
6314 }
6315
6316 static int nft_setelem_catchall_deactivate(const struct net *net,
6317                                            struct nft_set *set,
6318                                            struct nft_set_elem *elem)
6319 {
6320         struct nft_set_elem_catchall *catchall;
6321         struct nft_set_ext *ext;
6322
6323         list_for_each_entry(catchall, &set->catchall_list, list) {
6324                 ext = nft_set_elem_ext(set, catchall->elem);
6325                 if (!nft_is_active(net, ext) ||
6326                     nft_set_elem_mark_busy(ext))
6327                         continue;
6328
6329                 kfree(elem->priv);
6330                 elem->priv = catchall->elem;
6331                 nft_set_elem_change_active(net, set, ext);
6332                 return 0;
6333         }
6334
6335         return -ENOENT;
6336 }
6337
6338 static int __nft_setelem_deactivate(const struct net *net,
6339                                     struct nft_set *set,
6340                                     struct nft_set_elem *elem)
6341 {
6342         void *priv;
6343
6344         priv = set->ops->deactivate(net, set, elem);
6345         if (!priv)
6346                 return -ENOENT;
6347
6348         kfree(elem->priv);
6349         elem->priv = priv;
6350         set->ndeact++;
6351
6352         return 0;
6353 }
6354
6355 static int nft_setelem_deactivate(const struct net *net,
6356                                   struct nft_set *set,
6357                                   struct nft_set_elem *elem, u32 flags)
6358 {
6359         int ret;
6360
6361         if (flags & NFT_SET_ELEM_CATCHALL)
6362                 ret = nft_setelem_catchall_deactivate(net, set, elem);
6363         else
6364                 ret = __nft_setelem_deactivate(net, set, elem);
6365
6366         return ret;
6367 }
6368
6369 static void nft_setelem_catchall_remove(const struct net *net,
6370                                         const struct nft_set *set,
6371                                         const struct nft_set_elem *elem)
6372 {
6373         struct nft_set_elem_catchall *catchall, *next;
6374
6375         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
6376                 if (catchall->elem == elem->priv) {
6377                         list_del_rcu(&catchall->list);
6378                         kfree_rcu(catchall, rcu);
6379                         break;
6380                 }
6381         }
6382 }
6383
6384 static void nft_setelem_remove(const struct net *net,
6385                                const struct nft_set *set,
6386                                const struct nft_set_elem *elem)
6387 {
6388         if (nft_setelem_is_catchall(set, elem))
6389                 nft_setelem_catchall_remove(net, set, elem);
6390         else
6391                 set->ops->remove(net, set, elem);
6392 }
6393
6394 static bool nft_setelem_valid_key_end(const struct nft_set *set,
6395                                       struct nlattr **nla, u32 flags)
6396 {
6397         if ((set->flags & (NFT_SET_CONCAT | NFT_SET_INTERVAL)) ==
6398                           (NFT_SET_CONCAT | NFT_SET_INTERVAL)) {
6399                 if (flags & NFT_SET_ELEM_INTERVAL_END)
6400                         return false;
6401
6402                 if (nla[NFTA_SET_ELEM_KEY_END] &&
6403                     flags & NFT_SET_ELEM_CATCHALL)
6404                         return false;
6405         } else {
6406                 if (nla[NFTA_SET_ELEM_KEY_END])
6407                         return false;
6408         }
6409
6410         return true;
6411 }
6412
6413 static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set,
6414                             const struct nlattr *attr, u32 nlmsg_flags)
6415 {
6416         struct nft_expr *expr_array[NFT_SET_EXPR_MAX] = {};
6417         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
6418         u8 genmask = nft_genmask_next(ctx->net);
6419         u32 flags = 0, size = 0, num_exprs = 0;
6420         struct nft_set_ext_tmpl tmpl;
6421         struct nft_set_ext *ext, *ext2;
6422         struct nft_set_elem elem;
6423         struct nft_set_binding *binding;
6424         struct nft_object *obj = NULL;
6425         struct nft_userdata *udata;
6426         struct nft_data_desc desc;
6427         enum nft_registers dreg;
6428         struct nft_trans *trans;
6429         u64 timeout;
6430         u64 expiration;
6431         int err, i;
6432         u8 ulen;
6433
6434         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
6435                                           nft_set_elem_policy, NULL);
6436         if (err < 0)
6437                 return err;
6438
6439         nft_set_ext_prepare(&tmpl);
6440
6441         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
6442         if (err < 0)
6443                 return err;
6444
6445         if (((flags & NFT_SET_ELEM_CATCHALL) && nla[NFTA_SET_ELEM_KEY]) ||
6446             (!(flags & NFT_SET_ELEM_CATCHALL) && !nla[NFTA_SET_ELEM_KEY]))
6447                 return -EINVAL;
6448
6449         if (flags != 0) {
6450                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
6451                 if (err < 0)
6452                         return err;
6453         }
6454
6455         if (set->flags & NFT_SET_MAP) {
6456                 if (nla[NFTA_SET_ELEM_DATA] == NULL &&
6457                     !(flags & NFT_SET_ELEM_INTERVAL_END))
6458                         return -EINVAL;
6459         } else {
6460                 if (nla[NFTA_SET_ELEM_DATA] != NULL)
6461                         return -EINVAL;
6462         }
6463
6464         if (set->flags & NFT_SET_OBJECT) {
6465                 if (!nla[NFTA_SET_ELEM_OBJREF] &&
6466                     !(flags & NFT_SET_ELEM_INTERVAL_END))
6467                         return -EINVAL;
6468         } else {
6469                 if (nla[NFTA_SET_ELEM_OBJREF])
6470                         return -EINVAL;
6471         }
6472
6473         if (!nft_setelem_valid_key_end(set, nla, flags))
6474                 return -EINVAL;
6475
6476         if ((flags & NFT_SET_ELEM_INTERVAL_END) &&
6477              (nla[NFTA_SET_ELEM_DATA] ||
6478               nla[NFTA_SET_ELEM_OBJREF] ||
6479               nla[NFTA_SET_ELEM_TIMEOUT] ||
6480               nla[NFTA_SET_ELEM_EXPIRATION] ||
6481               nla[NFTA_SET_ELEM_USERDATA] ||
6482               nla[NFTA_SET_ELEM_EXPR] ||
6483               nla[NFTA_SET_ELEM_KEY_END] ||
6484               nla[NFTA_SET_ELEM_EXPRESSIONS]))
6485                 return -EINVAL;
6486
6487         timeout = 0;
6488         if (nla[NFTA_SET_ELEM_TIMEOUT] != NULL) {
6489                 if (!(set->flags & NFT_SET_TIMEOUT))
6490                         return -EINVAL;
6491                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_TIMEOUT],
6492                                             &timeout);
6493                 if (err)
6494                         return err;
6495         } else if (set->flags & NFT_SET_TIMEOUT &&
6496                    !(flags & NFT_SET_ELEM_INTERVAL_END)) {
6497                 timeout = READ_ONCE(set->timeout);
6498         }
6499
6500         expiration = 0;
6501         if (nla[NFTA_SET_ELEM_EXPIRATION] != NULL) {
6502                 if (!(set->flags & NFT_SET_TIMEOUT))
6503                         return -EINVAL;
6504                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_EXPIRATION],
6505                                             &expiration);
6506                 if (err)
6507                         return err;
6508         }
6509
6510         if (nla[NFTA_SET_ELEM_EXPR]) {
6511                 struct nft_expr *expr;
6512
6513                 if (set->num_exprs && set->num_exprs != 1)
6514                         return -EOPNOTSUPP;
6515
6516                 expr = nft_set_elem_expr_alloc(ctx, set,
6517                                                nla[NFTA_SET_ELEM_EXPR]);
6518                 if (IS_ERR(expr))
6519                         return PTR_ERR(expr);
6520
6521                 expr_array[0] = expr;
6522                 num_exprs = 1;
6523
6524                 if (set->num_exprs && set->exprs[0]->ops != expr->ops) {
6525                         err = -EOPNOTSUPP;
6526                         goto err_set_elem_expr;
6527                 }
6528         } else if (nla[NFTA_SET_ELEM_EXPRESSIONS]) {
6529                 struct nft_expr *expr;
6530                 struct nlattr *tmp;
6531                 int left;
6532
6533                 i = 0;
6534                 nla_for_each_nested(tmp, nla[NFTA_SET_ELEM_EXPRESSIONS], left) {
6535                         if (i == NFT_SET_EXPR_MAX ||
6536                             (set->num_exprs && set->num_exprs == i)) {
6537                                 err = -E2BIG;
6538                                 goto err_set_elem_expr;
6539                         }
6540                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
6541                                 err = -EINVAL;
6542                                 goto err_set_elem_expr;
6543                         }
6544                         expr = nft_set_elem_expr_alloc(ctx, set, tmp);
6545                         if (IS_ERR(expr)) {
6546                                 err = PTR_ERR(expr);
6547                                 goto err_set_elem_expr;
6548                         }
6549                         expr_array[i] = expr;
6550                         num_exprs++;
6551
6552                         if (set->num_exprs && expr->ops != set->exprs[i]->ops) {
6553                                 err = -EOPNOTSUPP;
6554                                 goto err_set_elem_expr;
6555                         }
6556                         i++;
6557                 }
6558                 if (set->num_exprs && set->num_exprs != i) {
6559                         err = -EOPNOTSUPP;
6560                         goto err_set_elem_expr;
6561                 }
6562         } else if (set->num_exprs > 0 &&
6563                    !(flags & NFT_SET_ELEM_INTERVAL_END)) {
6564                 err = nft_set_elem_expr_clone(ctx, set, expr_array);
6565                 if (err < 0)
6566                         goto err_set_elem_expr_clone;
6567
6568                 num_exprs = set->num_exprs;
6569         }
6570
6571         if (nla[NFTA_SET_ELEM_KEY]) {
6572                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
6573                                             nla[NFTA_SET_ELEM_KEY]);
6574                 if (err < 0)
6575                         goto err_set_elem_expr;
6576
6577                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
6578                 if (err < 0)
6579                         goto err_parse_key;
6580         }
6581
6582         if (nla[NFTA_SET_ELEM_KEY_END]) {
6583                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
6584                                             nla[NFTA_SET_ELEM_KEY_END]);
6585                 if (err < 0)
6586                         goto err_parse_key;
6587
6588                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
6589                 if (err < 0)
6590                         goto err_parse_key_end;
6591         }
6592
6593         if (timeout > 0) {
6594                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_EXPIRATION);
6595                 if (err < 0)
6596                         goto err_parse_key_end;
6597
6598                 if (timeout != READ_ONCE(set->timeout)) {
6599                         err = nft_set_ext_add(&tmpl, NFT_SET_EXT_TIMEOUT);
6600                         if (err < 0)
6601                                 goto err_parse_key_end;
6602                 }
6603         }
6604
6605         if (num_exprs) {
6606                 for (i = 0; i < num_exprs; i++)
6607                         size += expr_array[i]->ops->size;
6608
6609                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_EXPRESSIONS,
6610                                              sizeof(struct nft_set_elem_expr) + size);
6611                 if (err < 0)
6612                         goto err_parse_key_end;
6613         }
6614
6615         if (nla[NFTA_SET_ELEM_OBJREF] != NULL) {
6616                 obj = nft_obj_lookup(ctx->net, ctx->table,
6617                                      nla[NFTA_SET_ELEM_OBJREF],
6618                                      set->objtype, genmask);
6619                 if (IS_ERR(obj)) {
6620                         err = PTR_ERR(obj);
6621                         goto err_parse_key_end;
6622                 }
6623                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_OBJREF);
6624                 if (err < 0)
6625                         goto err_parse_key_end;
6626         }
6627
6628         if (nla[NFTA_SET_ELEM_DATA] != NULL) {
6629                 err = nft_setelem_parse_data(ctx, set, &desc, &elem.data.val,
6630                                              nla[NFTA_SET_ELEM_DATA]);
6631                 if (err < 0)
6632                         goto err_parse_key_end;
6633
6634                 dreg = nft_type_to_reg(set->dtype);
6635                 list_for_each_entry(binding, &set->bindings, list) {
6636                         struct nft_ctx bind_ctx = {
6637                                 .net    = ctx->net,
6638                                 .family = ctx->family,
6639                                 .table  = ctx->table,
6640                                 .chain  = (struct nft_chain *)binding->chain,
6641                         };
6642
6643                         if (!(binding->flags & NFT_SET_MAP))
6644                                 continue;
6645
6646                         err = nft_validate_register_store(&bind_ctx, dreg,
6647                                                           &elem.data.val,
6648                                                           desc.type, desc.len);
6649                         if (err < 0)
6650                                 goto err_parse_data;
6651
6652                         if (desc.type == NFT_DATA_VERDICT &&
6653                             (elem.data.val.verdict.code == NFT_GOTO ||
6654                              elem.data.val.verdict.code == NFT_JUMP))
6655                                 nft_validate_state_update(ctx->table,
6656                                                           NFT_VALIDATE_NEED);
6657                 }
6658
6659                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_DATA, desc.len);
6660                 if (err < 0)
6661                         goto err_parse_data;
6662         }
6663
6664         /* The full maximum length of userdata can exceed the maximum
6665          * offset value (U8_MAX) for following extensions, therefor it
6666          * must be the last extension added.
6667          */
6668         ulen = 0;
6669         if (nla[NFTA_SET_ELEM_USERDATA] != NULL) {
6670                 ulen = nla_len(nla[NFTA_SET_ELEM_USERDATA]);
6671                 if (ulen > 0) {
6672                         err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_USERDATA,
6673                                                      ulen);
6674                         if (err < 0)
6675                                 goto err_parse_data;
6676                 }
6677         }
6678
6679         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
6680                                       elem.key_end.val.data, elem.data.val.data,
6681                                       timeout, expiration, GFP_KERNEL_ACCOUNT);
6682         if (IS_ERR(elem.priv)) {
6683                 err = PTR_ERR(elem.priv);
6684                 goto err_parse_data;
6685         }
6686
6687         ext = nft_set_elem_ext(set, elem.priv);
6688         if (flags)
6689                 *nft_set_ext_flags(ext) = flags;
6690
6691         if (obj) {
6692                 *nft_set_ext_obj(ext) = obj;
6693                 obj->use++;
6694         }
6695         if (ulen > 0) {
6696                 if (nft_set_ext_check(&tmpl, NFT_SET_EXT_USERDATA, ulen) < 0) {
6697                         err = -EINVAL;
6698                         goto err_elem_free;
6699                 }
6700                 udata = nft_set_ext_userdata(ext);
6701                 udata->len = ulen - 1;
6702                 nla_memcpy(&udata->data, nla[NFTA_SET_ELEM_USERDATA], ulen);
6703         }
6704         err = nft_set_elem_expr_setup(ctx, &tmpl, ext, expr_array, num_exprs);
6705         if (err < 0)
6706                 goto err_elem_free;
6707
6708         trans = nft_trans_elem_alloc(ctx, NFT_MSG_NEWSETELEM, set);
6709         if (trans == NULL) {
6710                 err = -ENOMEM;
6711                 goto err_elem_free;
6712         }
6713
6714         ext->genmask = nft_genmask_cur(ctx->net) | NFT_SET_ELEM_BUSY_MASK;
6715
6716         err = nft_setelem_insert(ctx->net, set, &elem, &ext2, flags);
6717         if (err) {
6718                 if (err == -EEXIST) {
6719                         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) ^
6720                             nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) ||
6721                             nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) ^
6722                             nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF))
6723                                 goto err_element_clash;
6724                         if ((nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
6725                              nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) &&
6726                              memcmp(nft_set_ext_data(ext),
6727                                     nft_set_ext_data(ext2), set->dlen) != 0) ||
6728                             (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
6729                              nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF) &&
6730                              *nft_set_ext_obj(ext) != *nft_set_ext_obj(ext2)))
6731                                 goto err_element_clash;
6732                         else if (!(nlmsg_flags & NLM_F_EXCL))
6733                                 err = 0;
6734                 } else if (err == -ENOTEMPTY) {
6735                         /* ENOTEMPTY reports overlapping between this element
6736                          * and an existing one.
6737                          */
6738                         err = -EEXIST;
6739                 }
6740                 goto err_element_clash;
6741         }
6742
6743         if (!(flags & NFT_SET_ELEM_CATCHALL) && set->size &&
6744             !atomic_add_unless(&set->nelems, 1, set->size + set->ndeact)) {
6745                 err = -ENFILE;
6746                 goto err_set_full;
6747         }
6748
6749         nft_trans_elem(trans) = elem;
6750         nft_trans_commit_list_add_tail(ctx->net, trans);
6751         return 0;
6752
6753 err_set_full:
6754         nft_setelem_remove(ctx->net, set, &elem);
6755 err_element_clash:
6756         kfree(trans);
6757 err_elem_free:
6758         nft_set_elem_destroy(set, elem.priv, true);
6759 err_parse_data:
6760         if (nla[NFTA_SET_ELEM_DATA] != NULL)
6761                 nft_data_release(&elem.data.val, desc.type);
6762 err_parse_key_end:
6763         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
6764 err_parse_key:
6765         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
6766 err_set_elem_expr:
6767         for (i = 0; i < num_exprs && expr_array[i]; i++)
6768                 nft_expr_destroy(ctx, expr_array[i]);
6769 err_set_elem_expr_clone:
6770         return err;
6771 }
6772
6773 static int nf_tables_newsetelem(struct sk_buff *skb,
6774                                 const struct nfnl_info *info,
6775                                 const struct nlattr * const nla[])
6776 {
6777         struct netlink_ext_ack *extack = info->extack;
6778         u8 genmask = nft_genmask_next(info->net);
6779         u8 family = info->nfmsg->nfgen_family;
6780         struct net *net = info->net;
6781         const struct nlattr *attr;
6782         struct nft_table *table;
6783         struct nft_set *set;
6784         struct nft_ctx ctx;
6785         int rem, err;
6786
6787         if (nla[NFTA_SET_ELEM_LIST_ELEMENTS] == NULL)
6788                 return -EINVAL;
6789
6790         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
6791                                  genmask, NETLINK_CB(skb).portid);
6792         if (IS_ERR(table)) {
6793                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
6794                 return PTR_ERR(table);
6795         }
6796
6797         set = nft_set_lookup_global(net, table, nla[NFTA_SET_ELEM_LIST_SET],
6798                                     nla[NFTA_SET_ELEM_LIST_SET_ID], genmask);
6799         if (IS_ERR(set))
6800                 return PTR_ERR(set);
6801
6802         if (!list_empty(&set->bindings) &&
6803             (set->flags & (NFT_SET_CONSTANT | NFT_SET_ANONYMOUS)))
6804                 return -EBUSY;
6805
6806         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
6807
6808         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
6809                 err = nft_add_set_elem(&ctx, set, attr, info->nlh->nlmsg_flags);
6810                 if (err < 0) {
6811                         NL_SET_BAD_ATTR(extack, attr);
6812                         return err;
6813                 }
6814         }
6815
6816         if (table->validate_state == NFT_VALIDATE_DO)
6817                 return nft_table_validate(net, table);
6818
6819         return 0;
6820 }
6821
6822 /**
6823  *      nft_data_hold - hold a nft_data item
6824  *
6825  *      @data: struct nft_data to release
6826  *      @type: type of data
6827  *
6828  *      Hold a nft_data item. NFT_DATA_VALUE types can be silently discarded,
6829  *      NFT_DATA_VERDICT bumps the reference to chains in case of NFT_JUMP and
6830  *      NFT_GOTO verdicts. This function must be called on active data objects
6831  *      from the second phase of the commit protocol.
6832  */
6833 void nft_data_hold(const struct nft_data *data, enum nft_data_types type)
6834 {
6835         struct nft_chain *chain;
6836
6837         if (type == NFT_DATA_VERDICT) {
6838                 switch (data->verdict.code) {
6839                 case NFT_JUMP:
6840                 case NFT_GOTO:
6841                         chain = data->verdict.chain;
6842                         chain->use++;
6843                         break;
6844                 }
6845         }
6846 }
6847
6848 static void nft_setelem_data_activate(const struct net *net,
6849                                       const struct nft_set *set,
6850                                       struct nft_set_elem *elem)
6851 {
6852         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6853
6854         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
6855                 nft_data_hold(nft_set_ext_data(ext), set->dtype);
6856         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
6857                 (*nft_set_ext_obj(ext))->use++;
6858 }
6859
6860 static void nft_setelem_data_deactivate(const struct net *net,
6861                                         const struct nft_set *set,
6862                                         struct nft_set_elem *elem)
6863 {
6864         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6865
6866         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
6867                 nft_data_release(nft_set_ext_data(ext), set->dtype);
6868         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
6869                 (*nft_set_ext_obj(ext))->use--;
6870 }
6871
6872 static int nft_del_setelem(struct nft_ctx *ctx, struct nft_set *set,
6873                            const struct nlattr *attr)
6874 {
6875         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
6876         struct nft_set_ext_tmpl tmpl;
6877         struct nft_set_elem elem;
6878         struct nft_set_ext *ext;
6879         struct nft_trans *trans;
6880         u32 flags = 0;
6881         int err;
6882
6883         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
6884                                           nft_set_elem_policy, NULL);
6885         if (err < 0)
6886                 return err;
6887
6888         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
6889         if (err < 0)
6890                 return err;
6891
6892         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
6893                 return -EINVAL;
6894
6895         if (!nft_setelem_valid_key_end(set, nla, flags))
6896                 return -EINVAL;
6897
6898         nft_set_ext_prepare(&tmpl);
6899
6900         if (flags != 0) {
6901                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
6902                 if (err < 0)
6903                         return err;
6904         }
6905
6906         if (nla[NFTA_SET_ELEM_KEY]) {
6907                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
6908                                             nla[NFTA_SET_ELEM_KEY]);
6909                 if (err < 0)
6910                         return err;
6911
6912                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
6913                 if (err < 0)
6914                         goto fail_elem;
6915         }
6916
6917         if (nla[NFTA_SET_ELEM_KEY_END]) {
6918                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
6919                                             nla[NFTA_SET_ELEM_KEY_END]);
6920                 if (err < 0)
6921                         goto fail_elem;
6922
6923                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
6924                 if (err < 0)
6925                         goto fail_elem_key_end;
6926         }
6927
6928         err = -ENOMEM;
6929         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
6930                                       elem.key_end.val.data, NULL, 0, 0,
6931                                       GFP_KERNEL_ACCOUNT);
6932         if (IS_ERR(elem.priv)) {
6933                 err = PTR_ERR(elem.priv);
6934                 goto fail_elem_key_end;
6935         }
6936
6937         ext = nft_set_elem_ext(set, elem.priv);
6938         if (flags)
6939                 *nft_set_ext_flags(ext) = flags;
6940
6941         trans = nft_trans_elem_alloc(ctx, NFT_MSG_DELSETELEM, set);
6942         if (trans == NULL)
6943                 goto fail_trans;
6944
6945         err = nft_setelem_deactivate(ctx->net, set, &elem, flags);
6946         if (err < 0)
6947                 goto fail_ops;
6948
6949         nft_setelem_data_deactivate(ctx->net, set, &elem);
6950
6951         nft_trans_elem(trans) = elem;
6952         nft_trans_commit_list_add_tail(ctx->net, trans);
6953         return 0;
6954
6955 fail_ops:
6956         kfree(trans);
6957 fail_trans:
6958         kfree(elem.priv);
6959 fail_elem_key_end:
6960         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
6961 fail_elem:
6962         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
6963         return err;
6964 }
6965
6966 static int nft_setelem_flush(const struct nft_ctx *ctx,
6967                              struct nft_set *set,
6968                              const struct nft_set_iter *iter,
6969                              struct nft_set_elem *elem)
6970 {
6971         struct nft_trans *trans;
6972         int err;
6973
6974         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
6975                                     sizeof(struct nft_trans_elem), GFP_ATOMIC);
6976         if (!trans)
6977                 return -ENOMEM;
6978
6979         if (!set->ops->flush(ctx->net, set, elem->priv)) {
6980                 err = -ENOENT;
6981                 goto err1;
6982         }
6983         set->ndeact++;
6984
6985         nft_setelem_data_deactivate(ctx->net, set, elem);
6986         nft_trans_elem_set(trans) = set;
6987         nft_trans_elem(trans) = *elem;
6988         nft_trans_commit_list_add_tail(ctx->net, trans);
6989
6990         return 0;
6991 err1:
6992         kfree(trans);
6993         return err;
6994 }
6995
6996 static int __nft_set_catchall_flush(const struct nft_ctx *ctx,
6997                                     struct nft_set *set,
6998                                     struct nft_set_elem *elem)
6999 {
7000         struct nft_trans *trans;
7001
7002         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
7003                                     sizeof(struct nft_trans_elem), GFP_KERNEL);
7004         if (!trans)
7005                 return -ENOMEM;
7006
7007         nft_setelem_data_deactivate(ctx->net, set, elem);
7008         nft_trans_elem_set(trans) = set;
7009         nft_trans_elem(trans) = *elem;
7010         nft_trans_commit_list_add_tail(ctx->net, trans);
7011
7012         return 0;
7013 }
7014
7015 static int nft_set_catchall_flush(const struct nft_ctx *ctx,
7016                                   struct nft_set *set)
7017 {
7018         u8 genmask = nft_genmask_next(ctx->net);
7019         struct nft_set_elem_catchall *catchall;
7020         struct nft_set_elem elem;
7021         struct nft_set_ext *ext;
7022         int ret = 0;
7023
7024         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
7025                 ext = nft_set_elem_ext(set, catchall->elem);
7026                 if (!nft_set_elem_active(ext, genmask) ||
7027                     nft_set_elem_mark_busy(ext))
7028                         continue;
7029
7030                 elem.priv = catchall->elem;
7031                 ret = __nft_set_catchall_flush(ctx, set, &elem);
7032                 if (ret < 0)
7033                         break;
7034         }
7035
7036         return ret;
7037 }
7038
7039 static int nft_set_flush(struct nft_ctx *ctx, struct nft_set *set, u8 genmask)
7040 {
7041         struct nft_set_iter iter = {
7042                 .genmask        = genmask,
7043                 .fn             = nft_setelem_flush,
7044         };
7045
7046         set->ops->walk(ctx, set, &iter);
7047         if (!iter.err)
7048                 iter.err = nft_set_catchall_flush(ctx, set);
7049
7050         return iter.err;
7051 }
7052
7053 static int nf_tables_delsetelem(struct sk_buff *skb,
7054                                 const struct nfnl_info *info,
7055                                 const struct nlattr * const nla[])
7056 {
7057         struct netlink_ext_ack *extack = info->extack;
7058         u8 genmask = nft_genmask_next(info->net);
7059         u8 family = info->nfmsg->nfgen_family;
7060         struct net *net = info->net;
7061         const struct nlattr *attr;
7062         struct nft_table *table;
7063         struct nft_set *set;
7064         struct nft_ctx ctx;
7065         int rem, err = 0;
7066
7067         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
7068                                  genmask, NETLINK_CB(skb).portid);
7069         if (IS_ERR(table)) {
7070                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
7071                 return PTR_ERR(table);
7072         }
7073
7074         set = nft_set_lookup(table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
7075         if (IS_ERR(set))
7076                 return PTR_ERR(set);
7077
7078         if (!list_empty(&set->bindings) &&
7079             (set->flags & (NFT_SET_CONSTANT | NFT_SET_ANONYMOUS)))
7080                 return -EBUSY;
7081
7082         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7083
7084         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
7085                 return nft_set_flush(&ctx, set, genmask);
7086
7087         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
7088                 err = nft_del_setelem(&ctx, set, attr);
7089                 if (err == -ENOENT &&
7090                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYSETELEM)
7091                         continue;
7092
7093                 if (err < 0) {
7094                         NL_SET_BAD_ATTR(extack, attr);
7095                         break;
7096                 }
7097         }
7098         return err;
7099 }
7100
7101 void nft_set_gc_batch_release(struct rcu_head *rcu)
7102 {
7103         struct nft_set_gc_batch *gcb;
7104         unsigned int i;
7105
7106         gcb = container_of(rcu, struct nft_set_gc_batch, head.rcu);
7107         for (i = 0; i < gcb->head.cnt; i++)
7108                 nft_set_elem_destroy(gcb->head.set, gcb->elems[i], true);
7109         kfree(gcb);
7110 }
7111
7112 struct nft_set_gc_batch *nft_set_gc_batch_alloc(const struct nft_set *set,
7113                                                 gfp_t gfp)
7114 {
7115         struct nft_set_gc_batch *gcb;
7116
7117         gcb = kzalloc(sizeof(*gcb), gfp);
7118         if (gcb == NULL)
7119                 return gcb;
7120         gcb->head.set = set;
7121         return gcb;
7122 }
7123
7124 /*
7125  * Stateful objects
7126  */
7127
7128 /**
7129  *      nft_register_obj- register nf_tables stateful object type
7130  *      @obj_type: object type
7131  *
7132  *      Registers the object type for use with nf_tables. Returns zero on
7133  *      success or a negative errno code otherwise.
7134  */
7135 int nft_register_obj(struct nft_object_type *obj_type)
7136 {
7137         if (obj_type->type == NFT_OBJECT_UNSPEC)
7138                 return -EINVAL;
7139
7140         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7141         list_add_rcu(&obj_type->list, &nf_tables_objects);
7142         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7143         return 0;
7144 }
7145 EXPORT_SYMBOL_GPL(nft_register_obj);
7146
7147 /**
7148  *      nft_unregister_obj - unregister nf_tables object type
7149  *      @obj_type: object type
7150  *
7151  *      Unregisters the object type for use with nf_tables.
7152  */
7153 void nft_unregister_obj(struct nft_object_type *obj_type)
7154 {
7155         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7156         list_del_rcu(&obj_type->list);
7157         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7158 }
7159 EXPORT_SYMBOL_GPL(nft_unregister_obj);
7160
7161 struct nft_object *nft_obj_lookup(const struct net *net,
7162                                   const struct nft_table *table,
7163                                   const struct nlattr *nla, u32 objtype,
7164                                   u8 genmask)
7165 {
7166         struct nft_object_hash_key k = { .table = table };
7167         char search[NFT_OBJ_MAXNAMELEN];
7168         struct rhlist_head *tmp, *list;
7169         struct nft_object *obj;
7170
7171         nla_strscpy(search, nla, sizeof(search));
7172         k.name = search;
7173
7174         WARN_ON_ONCE(!rcu_read_lock_held() &&
7175                      !lockdep_commit_lock_is_held(net));
7176
7177         rcu_read_lock();
7178         list = rhltable_lookup(&nft_objname_ht, &k, nft_objname_ht_params);
7179         if (!list)
7180                 goto out;
7181
7182         rhl_for_each_entry_rcu(obj, tmp, list, rhlhead) {
7183                 if (objtype == obj->ops->type->type &&
7184                     nft_active_genmask(obj, genmask)) {
7185                         rcu_read_unlock();
7186                         return obj;
7187                 }
7188         }
7189 out:
7190         rcu_read_unlock();
7191         return ERR_PTR(-ENOENT);
7192 }
7193 EXPORT_SYMBOL_GPL(nft_obj_lookup);
7194
7195 static struct nft_object *nft_obj_lookup_byhandle(const struct nft_table *table,
7196                                                   const struct nlattr *nla,
7197                                                   u32 objtype, u8 genmask)
7198 {
7199         struct nft_object *obj;
7200
7201         list_for_each_entry(obj, &table->objects, list) {
7202                 if (be64_to_cpu(nla_get_be64(nla)) == obj->handle &&
7203                     objtype == obj->ops->type->type &&
7204                     nft_active_genmask(obj, genmask))
7205                         return obj;
7206         }
7207         return ERR_PTR(-ENOENT);
7208 }
7209
7210 static const struct nla_policy nft_obj_policy[NFTA_OBJ_MAX + 1] = {
7211         [NFTA_OBJ_TABLE]        = { .type = NLA_STRING,
7212                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
7213         [NFTA_OBJ_NAME]         = { .type = NLA_STRING,
7214                                     .len = NFT_OBJ_MAXNAMELEN - 1 },
7215         [NFTA_OBJ_TYPE]         = { .type = NLA_U32 },
7216         [NFTA_OBJ_DATA]         = { .type = NLA_NESTED },
7217         [NFTA_OBJ_HANDLE]       = { .type = NLA_U64},
7218         [NFTA_OBJ_USERDATA]     = { .type = NLA_BINARY,
7219                                     .len = NFT_USERDATA_MAXLEN },
7220 };
7221
7222 static struct nft_object *nft_obj_init(const struct nft_ctx *ctx,
7223                                        const struct nft_object_type *type,
7224                                        const struct nlattr *attr)
7225 {
7226         struct nlattr **tb;
7227         const struct nft_object_ops *ops;
7228         struct nft_object *obj;
7229         int err = -ENOMEM;
7230
7231         tb = kmalloc_array(type->maxattr + 1, sizeof(*tb), GFP_KERNEL);
7232         if (!tb)
7233                 goto err1;
7234
7235         if (attr) {
7236                 err = nla_parse_nested_deprecated(tb, type->maxattr, attr,
7237                                                   type->policy, NULL);
7238                 if (err < 0)
7239                         goto err2;
7240         } else {
7241                 memset(tb, 0, sizeof(tb[0]) * (type->maxattr + 1));
7242         }
7243
7244         if (type->select_ops) {
7245                 ops = type->select_ops(ctx, (const struct nlattr * const *)tb);
7246                 if (IS_ERR(ops)) {
7247                         err = PTR_ERR(ops);
7248                         goto err2;
7249                 }
7250         } else {
7251                 ops = type->ops;
7252         }
7253
7254         err = -ENOMEM;
7255         obj = kzalloc(sizeof(*obj) + ops->size, GFP_KERNEL_ACCOUNT);
7256         if (!obj)
7257                 goto err2;
7258
7259         err = ops->init(ctx, (const struct nlattr * const *)tb, obj);
7260         if (err < 0)
7261                 goto err3;
7262
7263         obj->ops = ops;
7264
7265         kfree(tb);
7266         return obj;
7267 err3:
7268         kfree(obj);
7269 err2:
7270         kfree(tb);
7271 err1:
7272         return ERR_PTR(err);
7273 }
7274
7275 static int nft_object_dump(struct sk_buff *skb, unsigned int attr,
7276                            struct nft_object *obj, bool reset)
7277 {
7278         struct nlattr *nest;
7279
7280         nest = nla_nest_start_noflag(skb, attr);
7281         if (!nest)
7282                 goto nla_put_failure;
7283         if (obj->ops->dump(skb, obj, reset) < 0)
7284                 goto nla_put_failure;
7285         nla_nest_end(skb, nest);
7286         return 0;
7287
7288 nla_put_failure:
7289         return -1;
7290 }
7291
7292 static const struct nft_object_type *__nft_obj_type_get(u32 objtype)
7293 {
7294         const struct nft_object_type *type;
7295
7296         list_for_each_entry(type, &nf_tables_objects, list) {
7297                 if (objtype == type->type)
7298                         return type;
7299         }
7300         return NULL;
7301 }
7302
7303 static const struct nft_object_type *
7304 nft_obj_type_get(struct net *net, u32 objtype)
7305 {
7306         const struct nft_object_type *type;
7307
7308         type = __nft_obj_type_get(objtype);
7309         if (type != NULL && try_module_get(type->owner))
7310                 return type;
7311
7312         lockdep_nfnl_nft_mutex_not_held();
7313 #ifdef CONFIG_MODULES
7314         if (type == NULL) {
7315                 if (nft_request_module(net, "nft-obj-%u", objtype) == -EAGAIN)
7316                         return ERR_PTR(-EAGAIN);
7317         }
7318 #endif
7319         return ERR_PTR(-ENOENT);
7320 }
7321
7322 static int nf_tables_updobj(const struct nft_ctx *ctx,
7323                             const struct nft_object_type *type,
7324                             const struct nlattr *attr,
7325                             struct nft_object *obj)
7326 {
7327         struct nft_object *newobj;
7328         struct nft_trans *trans;
7329         int err = -ENOMEM;
7330
7331         if (!try_module_get(type->owner))
7332                 return -ENOENT;
7333
7334         trans = nft_trans_alloc(ctx, NFT_MSG_NEWOBJ,
7335                                 sizeof(struct nft_trans_obj));
7336         if (!trans)
7337                 goto err_trans;
7338
7339         newobj = nft_obj_init(ctx, type, attr);
7340         if (IS_ERR(newobj)) {
7341                 err = PTR_ERR(newobj);
7342                 goto err_free_trans;
7343         }
7344
7345         nft_trans_obj(trans) = obj;
7346         nft_trans_obj_update(trans) = true;
7347         nft_trans_obj_newobj(trans) = newobj;
7348         nft_trans_commit_list_add_tail(ctx->net, trans);
7349
7350         return 0;
7351
7352 err_free_trans:
7353         kfree(trans);
7354 err_trans:
7355         module_put(type->owner);
7356         return err;
7357 }
7358
7359 static int nf_tables_newobj(struct sk_buff *skb, const struct nfnl_info *info,
7360                             const struct nlattr * const nla[])
7361 {
7362         struct netlink_ext_ack *extack = info->extack;
7363         u8 genmask = nft_genmask_next(info->net);
7364         u8 family = info->nfmsg->nfgen_family;
7365         const struct nft_object_type *type;
7366         struct net *net = info->net;
7367         struct nft_table *table;
7368         struct nft_object *obj;
7369         struct nft_ctx ctx;
7370         u32 objtype;
7371         int err;
7372
7373         if (!nla[NFTA_OBJ_TYPE] ||
7374             !nla[NFTA_OBJ_NAME] ||
7375             !nla[NFTA_OBJ_DATA])
7376                 return -EINVAL;
7377
7378         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
7379                                  NETLINK_CB(skb).portid);
7380         if (IS_ERR(table)) {
7381                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7382                 return PTR_ERR(table);
7383         }
7384
7385         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7386         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
7387         if (IS_ERR(obj)) {
7388                 err = PTR_ERR(obj);
7389                 if (err != -ENOENT) {
7390                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7391                         return err;
7392                 }
7393         } else {
7394                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
7395                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7396                         return -EEXIST;
7397                 }
7398                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
7399                         return -EOPNOTSUPP;
7400
7401                 type = __nft_obj_type_get(objtype);
7402                 if (WARN_ON_ONCE(!type))
7403                         return -ENOENT;
7404
7405                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7406
7407                 return nf_tables_updobj(&ctx, type, nla[NFTA_OBJ_DATA], obj);
7408         }
7409
7410         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7411
7412         type = nft_obj_type_get(net, objtype);
7413         if (IS_ERR(type))
7414                 return PTR_ERR(type);
7415
7416         obj = nft_obj_init(&ctx, type, nla[NFTA_OBJ_DATA]);
7417         if (IS_ERR(obj)) {
7418                 err = PTR_ERR(obj);
7419                 goto err_init;
7420         }
7421         obj->key.table = table;
7422         obj->handle = nf_tables_alloc_handle(table);
7423
7424         obj->key.name = nla_strdup(nla[NFTA_OBJ_NAME], GFP_KERNEL_ACCOUNT);
7425         if (!obj->key.name) {
7426                 err = -ENOMEM;
7427                 goto err_strdup;
7428         }
7429
7430         if (nla[NFTA_OBJ_USERDATA]) {
7431                 obj->udata = nla_memdup(nla[NFTA_OBJ_USERDATA], GFP_KERNEL_ACCOUNT);
7432                 if (obj->udata == NULL)
7433                         goto err_userdata;
7434
7435                 obj->udlen = nla_len(nla[NFTA_OBJ_USERDATA]);
7436         }
7437
7438         err = nft_trans_obj_add(&ctx, NFT_MSG_NEWOBJ, obj);
7439         if (err < 0)
7440                 goto err_trans;
7441
7442         err = rhltable_insert(&nft_objname_ht, &obj->rhlhead,
7443                               nft_objname_ht_params);
7444         if (err < 0)
7445                 goto err_obj_ht;
7446
7447         list_add_tail_rcu(&obj->list, &table->objects);
7448         table->use++;
7449         return 0;
7450 err_obj_ht:
7451         /* queued in transaction log */
7452         INIT_LIST_HEAD(&obj->list);
7453         return err;
7454 err_trans:
7455         kfree(obj->udata);
7456 err_userdata:
7457         kfree(obj->key.name);
7458 err_strdup:
7459         if (obj->ops->destroy)
7460                 obj->ops->destroy(&ctx, obj);
7461         kfree(obj);
7462 err_init:
7463         module_put(type->owner);
7464         return err;
7465 }
7466
7467 static int nf_tables_fill_obj_info(struct sk_buff *skb, struct net *net,
7468                                    u32 portid, u32 seq, int event, u32 flags,
7469                                    int family, const struct nft_table *table,
7470                                    struct nft_object *obj, bool reset)
7471 {
7472         struct nlmsghdr *nlh;
7473
7474         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
7475         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
7476                            NFNETLINK_V0, nft_base_seq(net));
7477         if (!nlh)
7478                 goto nla_put_failure;
7479
7480         if (nla_put_string(skb, NFTA_OBJ_TABLE, table->name) ||
7481             nla_put_string(skb, NFTA_OBJ_NAME, obj->key.name) ||
7482             nla_put_be64(skb, NFTA_OBJ_HANDLE, cpu_to_be64(obj->handle),
7483                          NFTA_OBJ_PAD))
7484                 goto nla_put_failure;
7485
7486         if (event == NFT_MSG_DELOBJ) {
7487                 nlmsg_end(skb, nlh);
7488                 return 0;
7489         }
7490
7491         if (nla_put_be32(skb, NFTA_OBJ_TYPE, htonl(obj->ops->type->type)) ||
7492             nla_put_be32(skb, NFTA_OBJ_USE, htonl(obj->use)) ||
7493             nft_object_dump(skb, NFTA_OBJ_DATA, obj, reset))
7494                 goto nla_put_failure;
7495
7496         if (obj->udata &&
7497             nla_put(skb, NFTA_OBJ_USERDATA, obj->udlen, obj->udata))
7498                 goto nla_put_failure;
7499
7500         nlmsg_end(skb, nlh);
7501         return 0;
7502
7503 nla_put_failure:
7504         nlmsg_trim(skb, nlh);
7505         return -1;
7506 }
7507
7508 struct nft_obj_filter {
7509         char            *table;
7510         u32             type;
7511 };
7512
7513 static int nf_tables_dump_obj(struct sk_buff *skb, struct netlink_callback *cb)
7514 {
7515         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
7516         const struct nft_table *table;
7517         unsigned int idx = 0, s_idx = cb->args[0];
7518         struct nft_obj_filter *filter = cb->data;
7519         struct net *net = sock_net(skb->sk);
7520         int family = nfmsg->nfgen_family;
7521         struct nftables_pernet *nft_net;
7522         struct nft_object *obj;
7523         bool reset = false;
7524
7525         if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
7526                 reset = true;
7527
7528         rcu_read_lock();
7529         nft_net = nft_pernet(net);
7530         cb->seq = READ_ONCE(nft_net->base_seq);
7531
7532         list_for_each_entry_rcu(table, &nft_net->tables, list) {
7533                 if (family != NFPROTO_UNSPEC && family != table->family)
7534                         continue;
7535
7536                 list_for_each_entry_rcu(obj, &table->objects, list) {
7537                         if (!nft_is_active(net, obj))
7538                                 goto cont;
7539                         if (idx < s_idx)
7540                                 goto cont;
7541                         if (idx > s_idx)
7542                                 memset(&cb->args[1], 0,
7543                                        sizeof(cb->args) - sizeof(cb->args[0]));
7544                         if (filter && filter->table &&
7545                             strcmp(filter->table, table->name))
7546                                 goto cont;
7547                         if (filter &&
7548                             filter->type != NFT_OBJECT_UNSPEC &&
7549                             obj->ops->type->type != filter->type)
7550                                 goto cont;
7551                         if (reset) {
7552                                 char *buf = kasprintf(GFP_ATOMIC,
7553                                                       "%s:%u",
7554                                                       table->name,
7555                                                       nft_net->base_seq);
7556
7557                                 audit_log_nfcfg(buf,
7558                                                 family,
7559                                                 obj->handle,
7560                                                 AUDIT_NFT_OP_OBJ_RESET,
7561                                                 GFP_ATOMIC);
7562                                 kfree(buf);
7563                         }
7564
7565                         if (nf_tables_fill_obj_info(skb, net, NETLINK_CB(cb->skb).portid,
7566                                                     cb->nlh->nlmsg_seq,
7567                                                     NFT_MSG_NEWOBJ,
7568                                                     NLM_F_MULTI | NLM_F_APPEND,
7569                                                     table->family, table,
7570                                                     obj, reset) < 0)
7571                                 goto done;
7572
7573                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
7574 cont:
7575                         idx++;
7576                 }
7577         }
7578 done:
7579         rcu_read_unlock();
7580
7581         cb->args[0] = idx;
7582         return skb->len;
7583 }
7584
7585 static int nf_tables_dump_obj_start(struct netlink_callback *cb)
7586 {
7587         const struct nlattr * const *nla = cb->data;
7588         struct nft_obj_filter *filter = NULL;
7589
7590         if (nla[NFTA_OBJ_TABLE] || nla[NFTA_OBJ_TYPE]) {
7591                 filter = kzalloc(sizeof(*filter), GFP_ATOMIC);
7592                 if (!filter)
7593                         return -ENOMEM;
7594
7595                 if (nla[NFTA_OBJ_TABLE]) {
7596                         filter->table = nla_strdup(nla[NFTA_OBJ_TABLE], GFP_ATOMIC);
7597                         if (!filter->table) {
7598                                 kfree(filter);
7599                                 return -ENOMEM;
7600                         }
7601                 }
7602
7603                 if (nla[NFTA_OBJ_TYPE])
7604                         filter->type = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7605         }
7606
7607         cb->data = filter;
7608         return 0;
7609 }
7610
7611 static int nf_tables_dump_obj_done(struct netlink_callback *cb)
7612 {
7613         struct nft_obj_filter *filter = cb->data;
7614
7615         if (filter) {
7616                 kfree(filter->table);
7617                 kfree(filter);
7618         }
7619
7620         return 0;
7621 }
7622
7623 /* called with rcu_read_lock held */
7624 static int nf_tables_getobj(struct sk_buff *skb, const struct nfnl_info *info,
7625                             const struct nlattr * const nla[])
7626 {
7627         struct netlink_ext_ack *extack = info->extack;
7628         u8 genmask = nft_genmask_cur(info->net);
7629         u8 family = info->nfmsg->nfgen_family;
7630         const struct nft_table *table;
7631         struct net *net = info->net;
7632         struct nft_object *obj;
7633         struct sk_buff *skb2;
7634         bool reset = false;
7635         u32 objtype;
7636         int err;
7637
7638         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
7639                 struct netlink_dump_control c = {
7640                         .start = nf_tables_dump_obj_start,
7641                         .dump = nf_tables_dump_obj,
7642                         .done = nf_tables_dump_obj_done,
7643                         .module = THIS_MODULE,
7644                         .data = (void *)nla,
7645                 };
7646
7647                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
7648         }
7649
7650         if (!nla[NFTA_OBJ_NAME] ||
7651             !nla[NFTA_OBJ_TYPE])
7652                 return -EINVAL;
7653
7654         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask, 0);
7655         if (IS_ERR(table)) {
7656                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7657                 return PTR_ERR(table);
7658         }
7659
7660         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7661         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
7662         if (IS_ERR(obj)) {
7663                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7664                 return PTR_ERR(obj);
7665         }
7666
7667         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
7668         if (!skb2)
7669                 return -ENOMEM;
7670
7671         if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
7672                 reset = true;
7673
7674         if (reset) {
7675                 const struct nftables_pernet *nft_net;
7676                 char *buf;
7677
7678                 nft_net = nft_pernet(net);
7679                 buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, nft_net->base_seq);
7680
7681                 audit_log_nfcfg(buf,
7682                                 family,
7683                                 obj->handle,
7684                                 AUDIT_NFT_OP_OBJ_RESET,
7685                                 GFP_ATOMIC);
7686                 kfree(buf);
7687         }
7688
7689         err = nf_tables_fill_obj_info(skb2, net, NETLINK_CB(skb).portid,
7690                                       info->nlh->nlmsg_seq, NFT_MSG_NEWOBJ, 0,
7691                                       family, table, obj, reset);
7692         if (err < 0)
7693                 goto err_fill_obj_info;
7694
7695         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
7696
7697 err_fill_obj_info:
7698         kfree_skb(skb2);
7699         return err;
7700 }
7701
7702 static void nft_obj_destroy(const struct nft_ctx *ctx, struct nft_object *obj)
7703 {
7704         if (obj->ops->destroy)
7705                 obj->ops->destroy(ctx, obj);
7706
7707         module_put(obj->ops->type->owner);
7708         kfree(obj->key.name);
7709         kfree(obj->udata);
7710         kfree(obj);
7711 }
7712
7713 static int nf_tables_delobj(struct sk_buff *skb, const struct nfnl_info *info,
7714                             const struct nlattr * const nla[])
7715 {
7716         struct netlink_ext_ack *extack = info->extack;
7717         u8 genmask = nft_genmask_next(info->net);
7718         u8 family = info->nfmsg->nfgen_family;
7719         struct net *net = info->net;
7720         const struct nlattr *attr;
7721         struct nft_table *table;
7722         struct nft_object *obj;
7723         struct nft_ctx ctx;
7724         u32 objtype;
7725
7726         if (!nla[NFTA_OBJ_TYPE] ||
7727             (!nla[NFTA_OBJ_NAME] && !nla[NFTA_OBJ_HANDLE]))
7728                 return -EINVAL;
7729
7730         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
7731                                  NETLINK_CB(skb).portid);
7732         if (IS_ERR(table)) {
7733                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7734                 return PTR_ERR(table);
7735         }
7736
7737         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7738         if (nla[NFTA_OBJ_HANDLE]) {
7739                 attr = nla[NFTA_OBJ_HANDLE];
7740                 obj = nft_obj_lookup_byhandle(table, attr, objtype, genmask);
7741         } else {
7742                 attr = nla[NFTA_OBJ_NAME];
7743                 obj = nft_obj_lookup(net, table, attr, objtype, genmask);
7744         }
7745
7746         if (IS_ERR(obj)) {
7747                 if (PTR_ERR(obj) == -ENOENT &&
7748                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYOBJ)
7749                         return 0;
7750
7751                 NL_SET_BAD_ATTR(extack, attr);
7752                 return PTR_ERR(obj);
7753         }
7754         if (obj->use > 0) {
7755                 NL_SET_BAD_ATTR(extack, attr);
7756                 return -EBUSY;
7757         }
7758
7759         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7760
7761         return nft_delobj(&ctx, obj);
7762 }
7763
7764 void nft_obj_notify(struct net *net, const struct nft_table *table,
7765                     struct nft_object *obj, u32 portid, u32 seq, int event,
7766                     u16 flags, int family, int report, gfp_t gfp)
7767 {
7768         struct nftables_pernet *nft_net = nft_pernet(net);
7769         struct sk_buff *skb;
7770         int err;
7771         char *buf = kasprintf(gfp, "%s:%u",
7772                               table->name, nft_net->base_seq);
7773
7774         audit_log_nfcfg(buf,
7775                         family,
7776                         obj->handle,
7777                         event == NFT_MSG_NEWOBJ ?
7778                                  AUDIT_NFT_OP_OBJ_REGISTER :
7779                                  AUDIT_NFT_OP_OBJ_UNREGISTER,
7780                         gfp);
7781         kfree(buf);
7782
7783         if (!report &&
7784             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
7785                 return;
7786
7787         skb = nlmsg_new(NLMSG_GOODSIZE, gfp);
7788         if (skb == NULL)
7789                 goto err;
7790
7791         err = nf_tables_fill_obj_info(skb, net, portid, seq, event,
7792                                       flags & (NLM_F_CREATE | NLM_F_EXCL),
7793                                       family, table, obj, false);
7794         if (err < 0) {
7795                 kfree_skb(skb);
7796                 goto err;
7797         }
7798
7799         nft_notify_enqueue(skb, report, &nft_net->notify_list);
7800         return;
7801 err:
7802         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
7803 }
7804 EXPORT_SYMBOL_GPL(nft_obj_notify);
7805
7806 static void nf_tables_obj_notify(const struct nft_ctx *ctx,
7807                                  struct nft_object *obj, int event)
7808 {
7809         nft_obj_notify(ctx->net, ctx->table, obj, ctx->portid, ctx->seq, event,
7810                        ctx->flags, ctx->family, ctx->report, GFP_KERNEL);
7811 }
7812
7813 /*
7814  * Flow tables
7815  */
7816 void nft_register_flowtable_type(struct nf_flowtable_type *type)
7817 {
7818         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7819         list_add_tail_rcu(&type->list, &nf_tables_flowtables);
7820         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7821 }
7822 EXPORT_SYMBOL_GPL(nft_register_flowtable_type);
7823
7824 void nft_unregister_flowtable_type(struct nf_flowtable_type *type)
7825 {
7826         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7827         list_del_rcu(&type->list);
7828         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7829 }
7830 EXPORT_SYMBOL_GPL(nft_unregister_flowtable_type);
7831
7832 static const struct nla_policy nft_flowtable_policy[NFTA_FLOWTABLE_MAX + 1] = {
7833         [NFTA_FLOWTABLE_TABLE]          = { .type = NLA_STRING,
7834                                             .len = NFT_NAME_MAXLEN - 1 },
7835         [NFTA_FLOWTABLE_NAME]           = { .type = NLA_STRING,
7836                                             .len = NFT_NAME_MAXLEN - 1 },
7837         [NFTA_FLOWTABLE_HOOK]           = { .type = NLA_NESTED },
7838         [NFTA_FLOWTABLE_HANDLE]         = { .type = NLA_U64 },
7839         [NFTA_FLOWTABLE_FLAGS]          = { .type = NLA_U32 },
7840 };
7841
7842 struct nft_flowtable *nft_flowtable_lookup(const struct nft_table *table,
7843                                            const struct nlattr *nla, u8 genmask)
7844 {
7845         struct nft_flowtable *flowtable;
7846
7847         list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
7848                 if (!nla_strcmp(nla, flowtable->name) &&
7849                     nft_active_genmask(flowtable, genmask))
7850                         return flowtable;
7851         }
7852         return ERR_PTR(-ENOENT);
7853 }
7854 EXPORT_SYMBOL_GPL(nft_flowtable_lookup);
7855
7856 void nf_tables_deactivate_flowtable(const struct nft_ctx *ctx,
7857                                     struct nft_flowtable *flowtable,
7858                                     enum nft_trans_phase phase)
7859 {
7860         switch (phase) {
7861         case NFT_TRANS_PREPARE_ERROR:
7862         case NFT_TRANS_PREPARE:
7863         case NFT_TRANS_ABORT:
7864         case NFT_TRANS_RELEASE:
7865                 flowtable->use--;
7866                 fallthrough;
7867         default:
7868                 return;
7869         }
7870 }
7871 EXPORT_SYMBOL_GPL(nf_tables_deactivate_flowtable);
7872
7873 static struct nft_flowtable *
7874 nft_flowtable_lookup_byhandle(const struct nft_table *table,
7875                               const struct nlattr *nla, u8 genmask)
7876 {
7877        struct nft_flowtable *flowtable;
7878
7879        list_for_each_entry(flowtable, &table->flowtables, list) {
7880                if (be64_to_cpu(nla_get_be64(nla)) == flowtable->handle &&
7881                    nft_active_genmask(flowtable, genmask))
7882                        return flowtable;
7883        }
7884        return ERR_PTR(-ENOENT);
7885 }
7886
7887 struct nft_flowtable_hook {
7888         u32                     num;
7889         int                     priority;
7890         struct list_head        list;
7891 };
7892
7893 static const struct nla_policy nft_flowtable_hook_policy[NFTA_FLOWTABLE_HOOK_MAX + 1] = {
7894         [NFTA_FLOWTABLE_HOOK_NUM]       = { .type = NLA_U32 },
7895         [NFTA_FLOWTABLE_HOOK_PRIORITY]  = { .type = NLA_U32 },
7896         [NFTA_FLOWTABLE_HOOK_DEVS]      = { .type = NLA_NESTED },
7897 };
7898
7899 static int nft_flowtable_parse_hook(const struct nft_ctx *ctx,
7900                                     const struct nlattr * const nla[],
7901                                     struct nft_flowtable_hook *flowtable_hook,
7902                                     struct nft_flowtable *flowtable,
7903                                     struct netlink_ext_ack *extack, bool add)
7904 {
7905         struct nlattr *tb[NFTA_FLOWTABLE_HOOK_MAX + 1];
7906         struct nft_hook *hook;
7907         int hooknum, priority;
7908         int err;
7909
7910         INIT_LIST_HEAD(&flowtable_hook->list);
7911
7912         err = nla_parse_nested_deprecated(tb, NFTA_FLOWTABLE_HOOK_MAX,
7913                                           nla[NFTA_FLOWTABLE_HOOK],
7914                                           nft_flowtable_hook_policy, NULL);
7915         if (err < 0)
7916                 return err;
7917
7918         if (add) {
7919                 if (!tb[NFTA_FLOWTABLE_HOOK_NUM] ||
7920                     !tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
7921                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
7922                         return -ENOENT;
7923                 }
7924
7925                 hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
7926                 if (hooknum != NF_NETDEV_INGRESS)
7927                         return -EOPNOTSUPP;
7928
7929                 priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
7930
7931                 flowtable_hook->priority        = priority;
7932                 flowtable_hook->num             = hooknum;
7933         } else {
7934                 if (tb[NFTA_FLOWTABLE_HOOK_NUM]) {
7935                         hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
7936                         if (hooknum != flowtable->hooknum)
7937                                 return -EOPNOTSUPP;
7938                 }
7939
7940                 if (tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
7941                         priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
7942                         if (priority != flowtable->data.priority)
7943                                 return -EOPNOTSUPP;
7944                 }
7945
7946                 flowtable_hook->priority        = flowtable->data.priority;
7947                 flowtable_hook->num             = flowtable->hooknum;
7948         }
7949
7950         if (tb[NFTA_FLOWTABLE_HOOK_DEVS]) {
7951                 err = nf_tables_parse_netdev_hooks(ctx->net,
7952                                                    tb[NFTA_FLOWTABLE_HOOK_DEVS],
7953                                                    &flowtable_hook->list,
7954                                                    extack);
7955                 if (err < 0)
7956                         return err;
7957         }
7958
7959         list_for_each_entry(hook, &flowtable_hook->list, list) {
7960                 hook->ops.pf            = NFPROTO_NETDEV;
7961                 hook->ops.hooknum       = flowtable_hook->num;
7962                 hook->ops.priority      = flowtable_hook->priority;
7963                 hook->ops.priv          = &flowtable->data;
7964                 hook->ops.hook          = flowtable->data.type->hook;
7965         }
7966
7967         return err;
7968 }
7969
7970 static const struct nf_flowtable_type *__nft_flowtable_type_get(u8 family)
7971 {
7972         const struct nf_flowtable_type *type;
7973
7974         list_for_each_entry(type, &nf_tables_flowtables, list) {
7975                 if (family == type->family)
7976                         return type;
7977         }
7978         return NULL;
7979 }
7980
7981 static const struct nf_flowtable_type *
7982 nft_flowtable_type_get(struct net *net, u8 family)
7983 {
7984         const struct nf_flowtable_type *type;
7985
7986         type = __nft_flowtable_type_get(family);
7987         if (type != NULL && try_module_get(type->owner))
7988                 return type;
7989
7990         lockdep_nfnl_nft_mutex_not_held();
7991 #ifdef CONFIG_MODULES
7992         if (type == NULL) {
7993                 if (nft_request_module(net, "nf-flowtable-%u", family) == -EAGAIN)
7994                         return ERR_PTR(-EAGAIN);
7995         }
7996 #endif
7997         return ERR_PTR(-ENOENT);
7998 }
7999
8000 /* Only called from error and netdev event paths. */
8001 static void nft_unregister_flowtable_hook(struct net *net,
8002                                           struct nft_flowtable *flowtable,
8003                                           struct nft_hook *hook)
8004 {
8005         nf_unregister_net_hook(net, &hook->ops);
8006         flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
8007                                     FLOW_BLOCK_UNBIND);
8008 }
8009
8010 static void __nft_unregister_flowtable_net_hooks(struct net *net,
8011                                                  struct list_head *hook_list,
8012                                                  bool release_netdev)
8013 {
8014         struct nft_hook *hook, *next;
8015
8016         list_for_each_entry_safe(hook, next, hook_list, list) {
8017                 nf_unregister_net_hook(net, &hook->ops);
8018                 if (release_netdev) {
8019                         list_del(&hook->list);
8020                         kfree_rcu(hook, rcu);
8021                 }
8022         }
8023 }
8024
8025 static void nft_unregister_flowtable_net_hooks(struct net *net,
8026                                                struct list_head *hook_list)
8027 {
8028         __nft_unregister_flowtable_net_hooks(net, hook_list, false);
8029 }
8030
8031 static int nft_register_flowtable_net_hooks(struct net *net,
8032                                             struct nft_table *table,
8033                                             struct list_head *hook_list,
8034                                             struct nft_flowtable *flowtable)
8035 {
8036         struct nft_hook *hook, *hook2, *next;
8037         struct nft_flowtable *ft;
8038         int err, i = 0;
8039
8040         list_for_each_entry(hook, hook_list, list) {
8041                 list_for_each_entry(ft, &table->flowtables, list) {
8042                         if (!nft_is_active_next(net, ft))
8043                                 continue;
8044
8045                         list_for_each_entry(hook2, &ft->hook_list, list) {
8046                                 if (hook->ops.dev == hook2->ops.dev &&
8047                                     hook->ops.pf == hook2->ops.pf) {
8048                                         err = -EEXIST;
8049                                         goto err_unregister_net_hooks;
8050                                 }
8051                         }
8052                 }
8053
8054                 err = flowtable->data.type->setup(&flowtable->data,
8055                                                   hook->ops.dev,
8056                                                   FLOW_BLOCK_BIND);
8057                 if (err < 0)
8058                         goto err_unregister_net_hooks;
8059
8060                 err = nf_register_net_hook(net, &hook->ops);
8061                 if (err < 0) {
8062                         flowtable->data.type->setup(&flowtable->data,
8063                                                     hook->ops.dev,
8064                                                     FLOW_BLOCK_UNBIND);
8065                         goto err_unregister_net_hooks;
8066                 }
8067
8068                 i++;
8069         }
8070
8071         return 0;
8072
8073 err_unregister_net_hooks:
8074         list_for_each_entry_safe(hook, next, hook_list, list) {
8075                 if (i-- <= 0)
8076                         break;
8077
8078                 nft_unregister_flowtable_hook(net, flowtable, hook);
8079                 list_del_rcu(&hook->list);
8080                 kfree_rcu(hook, rcu);
8081         }
8082
8083         return err;
8084 }
8085
8086 static void nft_hooks_destroy(struct list_head *hook_list)
8087 {
8088         struct nft_hook *hook, *next;
8089
8090         list_for_each_entry_safe(hook, next, hook_list, list) {
8091                 list_del_rcu(&hook->list);
8092                 kfree_rcu(hook, rcu);
8093         }
8094 }
8095
8096 static int nft_flowtable_update(struct nft_ctx *ctx, const struct nlmsghdr *nlh,
8097                                 struct nft_flowtable *flowtable,
8098                                 struct netlink_ext_ack *extack)
8099 {
8100         const struct nlattr * const *nla = ctx->nla;
8101         struct nft_flowtable_hook flowtable_hook;
8102         struct nft_hook *hook, *next;
8103         struct nft_trans *trans;
8104         bool unregister = false;
8105         u32 flags;
8106         int err;
8107
8108         err = nft_flowtable_parse_hook(ctx, nla, &flowtable_hook, flowtable,
8109                                        extack, false);
8110         if (err < 0)
8111                 return err;
8112
8113         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
8114                 if (nft_hook_list_find(&flowtable->hook_list, hook)) {
8115                         list_del(&hook->list);
8116                         kfree(hook);
8117                 }
8118         }
8119
8120         if (nla[NFTA_FLOWTABLE_FLAGS]) {
8121                 flags = ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
8122                 if (flags & ~NFT_FLOWTABLE_MASK) {
8123                         err = -EOPNOTSUPP;
8124                         goto err_flowtable_update_hook;
8125                 }
8126                 if ((flowtable->data.flags & NFT_FLOWTABLE_HW_OFFLOAD) ^
8127                     (flags & NFT_FLOWTABLE_HW_OFFLOAD)) {
8128                         err = -EOPNOTSUPP;
8129                         goto err_flowtable_update_hook;
8130                 }
8131         } else {
8132                 flags = flowtable->data.flags;
8133         }
8134
8135         err = nft_register_flowtable_net_hooks(ctx->net, ctx->table,
8136                                                &flowtable_hook.list, flowtable);
8137         if (err < 0)
8138                 goto err_flowtable_update_hook;
8139
8140         trans = nft_trans_alloc(ctx, NFT_MSG_NEWFLOWTABLE,
8141                                 sizeof(struct nft_trans_flowtable));
8142         if (!trans) {
8143                 unregister = true;
8144                 err = -ENOMEM;
8145                 goto err_flowtable_update_hook;
8146         }
8147
8148         nft_trans_flowtable_flags(trans) = flags;
8149         nft_trans_flowtable(trans) = flowtable;
8150         nft_trans_flowtable_update(trans) = true;
8151         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
8152         list_splice(&flowtable_hook.list, &nft_trans_flowtable_hooks(trans));
8153
8154         nft_trans_commit_list_add_tail(ctx->net, trans);
8155
8156         return 0;
8157
8158 err_flowtable_update_hook:
8159         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
8160                 if (unregister)
8161                         nft_unregister_flowtable_hook(ctx->net, flowtable, hook);
8162                 list_del_rcu(&hook->list);
8163                 kfree_rcu(hook, rcu);
8164         }
8165
8166         return err;
8167
8168 }
8169
8170 static int nf_tables_newflowtable(struct sk_buff *skb,
8171                                   const struct nfnl_info *info,
8172                                   const struct nlattr * const nla[])
8173 {
8174         struct netlink_ext_ack *extack = info->extack;
8175         struct nft_flowtable_hook flowtable_hook;
8176         u8 genmask = nft_genmask_next(info->net);
8177         u8 family = info->nfmsg->nfgen_family;
8178         const struct nf_flowtable_type *type;
8179         struct nft_flowtable *flowtable;
8180         struct nft_hook *hook, *next;
8181         struct net *net = info->net;
8182         struct nft_table *table;
8183         struct nft_ctx ctx;
8184         int err;
8185
8186         if (!nla[NFTA_FLOWTABLE_TABLE] ||
8187             !nla[NFTA_FLOWTABLE_NAME] ||
8188             !nla[NFTA_FLOWTABLE_HOOK])
8189                 return -EINVAL;
8190
8191         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8192                                  genmask, NETLINK_CB(skb).portid);
8193         if (IS_ERR(table)) {
8194                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
8195                 return PTR_ERR(table);
8196         }
8197
8198         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
8199                                          genmask);
8200         if (IS_ERR(flowtable)) {
8201                 err = PTR_ERR(flowtable);
8202                 if (err != -ENOENT) {
8203                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8204                         return err;
8205                 }
8206         } else {
8207                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
8208                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8209                         return -EEXIST;
8210                 }
8211
8212                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8213
8214                 return nft_flowtable_update(&ctx, info->nlh, flowtable, extack);
8215         }
8216
8217         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8218
8219         flowtable = kzalloc(sizeof(*flowtable), GFP_KERNEL_ACCOUNT);
8220         if (!flowtable)
8221                 return -ENOMEM;
8222
8223         flowtable->table = table;
8224         flowtable->handle = nf_tables_alloc_handle(table);
8225         INIT_LIST_HEAD(&flowtable->hook_list);
8226
8227         flowtable->name = nla_strdup(nla[NFTA_FLOWTABLE_NAME], GFP_KERNEL_ACCOUNT);
8228         if (!flowtable->name) {
8229                 err = -ENOMEM;
8230                 goto err1;
8231         }
8232
8233         type = nft_flowtable_type_get(net, family);
8234         if (IS_ERR(type)) {
8235                 err = PTR_ERR(type);
8236                 goto err2;
8237         }
8238
8239         if (nla[NFTA_FLOWTABLE_FLAGS]) {
8240                 flowtable->data.flags =
8241                         ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
8242                 if (flowtable->data.flags & ~NFT_FLOWTABLE_MASK) {
8243                         err = -EOPNOTSUPP;
8244                         goto err3;
8245                 }
8246         }
8247
8248         write_pnet(&flowtable->data.net, net);
8249         flowtable->data.type = type;
8250         err = type->init(&flowtable->data);
8251         if (err < 0)
8252                 goto err3;
8253
8254         err = nft_flowtable_parse_hook(&ctx, nla, &flowtable_hook, flowtable,
8255                                        extack, true);
8256         if (err < 0)
8257                 goto err4;
8258
8259         list_splice(&flowtable_hook.list, &flowtable->hook_list);
8260         flowtable->data.priority = flowtable_hook.priority;
8261         flowtable->hooknum = flowtable_hook.num;
8262
8263         err = nft_register_flowtable_net_hooks(ctx.net, table,
8264                                                &flowtable->hook_list,
8265                                                flowtable);
8266         if (err < 0) {
8267                 nft_hooks_destroy(&flowtable->hook_list);
8268                 goto err4;
8269         }
8270
8271         err = nft_trans_flowtable_add(&ctx, NFT_MSG_NEWFLOWTABLE, flowtable);
8272         if (err < 0)
8273                 goto err5;
8274
8275         list_add_tail_rcu(&flowtable->list, &table->flowtables);
8276         table->use++;
8277
8278         return 0;
8279 err5:
8280         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
8281                 nft_unregister_flowtable_hook(net, flowtable, hook);
8282                 list_del_rcu(&hook->list);
8283                 kfree_rcu(hook, rcu);
8284         }
8285 err4:
8286         flowtable->data.type->free(&flowtable->data);
8287 err3:
8288         module_put(type->owner);
8289 err2:
8290         kfree(flowtable->name);
8291 err1:
8292         kfree(flowtable);
8293         return err;
8294 }
8295
8296 static void nft_flowtable_hook_release(struct nft_flowtable_hook *flowtable_hook)
8297 {
8298         struct nft_hook *this, *next;
8299
8300         list_for_each_entry_safe(this, next, &flowtable_hook->list, list) {
8301                 list_del(&this->list);
8302                 kfree(this);
8303         }
8304 }
8305
8306 static int nft_delflowtable_hook(struct nft_ctx *ctx,
8307                                  struct nft_flowtable *flowtable,
8308                                  struct netlink_ext_ack *extack)
8309 {
8310         const struct nlattr * const *nla = ctx->nla;
8311         struct nft_flowtable_hook flowtable_hook;
8312         LIST_HEAD(flowtable_del_list);
8313         struct nft_hook *this, *hook;
8314         struct nft_trans *trans;
8315         int err;
8316
8317         err = nft_flowtable_parse_hook(ctx, nla, &flowtable_hook, flowtable,
8318                                        extack, false);
8319         if (err < 0)
8320                 return err;
8321
8322         list_for_each_entry(this, &flowtable_hook.list, list) {
8323                 hook = nft_hook_list_find(&flowtable->hook_list, this);
8324                 if (!hook) {
8325                         err = -ENOENT;
8326                         goto err_flowtable_del_hook;
8327                 }
8328                 list_move(&hook->list, &flowtable_del_list);
8329         }
8330
8331         trans = nft_trans_alloc(ctx, NFT_MSG_DELFLOWTABLE,
8332                                 sizeof(struct nft_trans_flowtable));
8333         if (!trans) {
8334                 err = -ENOMEM;
8335                 goto err_flowtable_del_hook;
8336         }
8337
8338         nft_trans_flowtable(trans) = flowtable;
8339         nft_trans_flowtable_update(trans) = true;
8340         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
8341         list_splice(&flowtable_del_list, &nft_trans_flowtable_hooks(trans));
8342         nft_flowtable_hook_release(&flowtable_hook);
8343
8344         nft_trans_commit_list_add_tail(ctx->net, trans);
8345
8346         return 0;
8347
8348 err_flowtable_del_hook:
8349         list_splice(&flowtable_del_list, &flowtable->hook_list);
8350         nft_flowtable_hook_release(&flowtable_hook);
8351
8352         return err;
8353 }
8354
8355 static int nf_tables_delflowtable(struct sk_buff *skb,
8356                                   const struct nfnl_info *info,
8357                                   const struct nlattr * const nla[])
8358 {
8359         struct netlink_ext_ack *extack = info->extack;
8360         u8 genmask = nft_genmask_next(info->net);
8361         u8 family = info->nfmsg->nfgen_family;
8362         struct nft_flowtable *flowtable;
8363         struct net *net = info->net;
8364         const struct nlattr *attr;
8365         struct nft_table *table;
8366         struct nft_ctx ctx;
8367
8368         if (!nla[NFTA_FLOWTABLE_TABLE] ||
8369             (!nla[NFTA_FLOWTABLE_NAME] &&
8370              !nla[NFTA_FLOWTABLE_HANDLE]))
8371                 return -EINVAL;
8372
8373         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8374                                  genmask, NETLINK_CB(skb).portid);
8375         if (IS_ERR(table)) {
8376                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
8377                 return PTR_ERR(table);
8378         }
8379
8380         if (nla[NFTA_FLOWTABLE_HANDLE]) {
8381                 attr = nla[NFTA_FLOWTABLE_HANDLE];
8382                 flowtable = nft_flowtable_lookup_byhandle(table, attr, genmask);
8383         } else {
8384                 attr = nla[NFTA_FLOWTABLE_NAME];
8385                 flowtable = nft_flowtable_lookup(table, attr, genmask);
8386         }
8387
8388         if (IS_ERR(flowtable)) {
8389                 if (PTR_ERR(flowtable) == -ENOENT &&
8390                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYFLOWTABLE)
8391                         return 0;
8392
8393                 NL_SET_BAD_ATTR(extack, attr);
8394                 return PTR_ERR(flowtable);
8395         }
8396
8397         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8398
8399         if (nla[NFTA_FLOWTABLE_HOOK])
8400                 return nft_delflowtable_hook(&ctx, flowtable, extack);
8401
8402         if (flowtable->use > 0) {
8403                 NL_SET_BAD_ATTR(extack, attr);
8404                 return -EBUSY;
8405         }
8406
8407         return nft_delflowtable(&ctx, flowtable);
8408 }
8409
8410 static int nf_tables_fill_flowtable_info(struct sk_buff *skb, struct net *net,
8411                                          u32 portid, u32 seq, int event,
8412                                          u32 flags, int family,
8413                                          struct nft_flowtable *flowtable,
8414                                          struct list_head *hook_list)
8415 {
8416         struct nlattr *nest, *nest_devs;
8417         struct nft_hook *hook;
8418         struct nlmsghdr *nlh;
8419
8420         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
8421         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
8422                            NFNETLINK_V0, nft_base_seq(net));
8423         if (!nlh)
8424                 goto nla_put_failure;
8425
8426         if (nla_put_string(skb, NFTA_FLOWTABLE_TABLE, flowtable->table->name) ||
8427             nla_put_string(skb, NFTA_FLOWTABLE_NAME, flowtable->name) ||
8428             nla_put_be64(skb, NFTA_FLOWTABLE_HANDLE, cpu_to_be64(flowtable->handle),
8429                          NFTA_FLOWTABLE_PAD))
8430                 goto nla_put_failure;
8431
8432         if (event == NFT_MSG_DELFLOWTABLE && !hook_list) {
8433                 nlmsg_end(skb, nlh);
8434                 return 0;
8435         }
8436
8437         if (nla_put_be32(skb, NFTA_FLOWTABLE_USE, htonl(flowtable->use)) ||
8438             nla_put_be32(skb, NFTA_FLOWTABLE_FLAGS, htonl(flowtable->data.flags)))
8439                 goto nla_put_failure;
8440
8441         nest = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK);
8442         if (!nest)
8443                 goto nla_put_failure;
8444         if (nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_NUM, htonl(flowtable->hooknum)) ||
8445             nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_PRIORITY, htonl(flowtable->data.priority)))
8446                 goto nla_put_failure;
8447
8448         nest_devs = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK_DEVS);
8449         if (!nest_devs)
8450                 goto nla_put_failure;
8451
8452         if (!hook_list)
8453                 hook_list = &flowtable->hook_list;
8454
8455         list_for_each_entry_rcu(hook, hook_list, list) {
8456                 if (nla_put_string(skb, NFTA_DEVICE_NAME, hook->ops.dev->name))
8457                         goto nla_put_failure;
8458         }
8459         nla_nest_end(skb, nest_devs);
8460         nla_nest_end(skb, nest);
8461
8462         nlmsg_end(skb, nlh);
8463         return 0;
8464
8465 nla_put_failure:
8466         nlmsg_trim(skb, nlh);
8467         return -1;
8468 }
8469
8470 struct nft_flowtable_filter {
8471         char            *table;
8472 };
8473
8474 static int nf_tables_dump_flowtable(struct sk_buff *skb,
8475                                     struct netlink_callback *cb)
8476 {
8477         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
8478         struct nft_flowtable_filter *filter = cb->data;
8479         unsigned int idx = 0, s_idx = cb->args[0];
8480         struct net *net = sock_net(skb->sk);
8481         int family = nfmsg->nfgen_family;
8482         struct nft_flowtable *flowtable;
8483         struct nftables_pernet *nft_net;
8484         const struct nft_table *table;
8485
8486         rcu_read_lock();
8487         nft_net = nft_pernet(net);
8488         cb->seq = READ_ONCE(nft_net->base_seq);
8489
8490         list_for_each_entry_rcu(table, &nft_net->tables, list) {
8491                 if (family != NFPROTO_UNSPEC && family != table->family)
8492                         continue;
8493
8494                 list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
8495                         if (!nft_is_active(net, flowtable))
8496                                 goto cont;
8497                         if (idx < s_idx)
8498                                 goto cont;
8499                         if (idx > s_idx)
8500                                 memset(&cb->args[1], 0,
8501                                        sizeof(cb->args) - sizeof(cb->args[0]));
8502                         if (filter && filter->table &&
8503                             strcmp(filter->table, table->name))
8504                                 goto cont;
8505
8506                         if (nf_tables_fill_flowtable_info(skb, net, NETLINK_CB(cb->skb).portid,
8507                                                           cb->nlh->nlmsg_seq,
8508                                                           NFT_MSG_NEWFLOWTABLE,
8509                                                           NLM_F_MULTI | NLM_F_APPEND,
8510                                                           table->family,
8511                                                           flowtable, NULL) < 0)
8512                                 goto done;
8513
8514                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
8515 cont:
8516                         idx++;
8517                 }
8518         }
8519 done:
8520         rcu_read_unlock();
8521
8522         cb->args[0] = idx;
8523         return skb->len;
8524 }
8525
8526 static int nf_tables_dump_flowtable_start(struct netlink_callback *cb)
8527 {
8528         const struct nlattr * const *nla = cb->data;
8529         struct nft_flowtable_filter *filter = NULL;
8530
8531         if (nla[NFTA_FLOWTABLE_TABLE]) {
8532                 filter = kzalloc(sizeof(*filter), GFP_ATOMIC);
8533                 if (!filter)
8534                         return -ENOMEM;
8535
8536                 filter->table = nla_strdup(nla[NFTA_FLOWTABLE_TABLE],
8537                                            GFP_ATOMIC);
8538                 if (!filter->table) {
8539                         kfree(filter);
8540                         return -ENOMEM;
8541                 }
8542         }
8543
8544         cb->data = filter;
8545         return 0;
8546 }
8547
8548 static int nf_tables_dump_flowtable_done(struct netlink_callback *cb)
8549 {
8550         struct nft_flowtable_filter *filter = cb->data;
8551
8552         if (!filter)
8553                 return 0;
8554
8555         kfree(filter->table);
8556         kfree(filter);
8557
8558         return 0;
8559 }
8560
8561 /* called with rcu_read_lock held */
8562 static int nf_tables_getflowtable(struct sk_buff *skb,
8563                                   const struct nfnl_info *info,
8564                                   const struct nlattr * const nla[])
8565 {
8566         u8 genmask = nft_genmask_cur(info->net);
8567         u8 family = info->nfmsg->nfgen_family;
8568         struct nft_flowtable *flowtable;
8569         const struct nft_table *table;
8570         struct net *net = info->net;
8571         struct sk_buff *skb2;
8572         int err;
8573
8574         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
8575                 struct netlink_dump_control c = {
8576                         .start = nf_tables_dump_flowtable_start,
8577                         .dump = nf_tables_dump_flowtable,
8578                         .done = nf_tables_dump_flowtable_done,
8579                         .module = THIS_MODULE,
8580                         .data = (void *)nla,
8581                 };
8582
8583                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
8584         }
8585
8586         if (!nla[NFTA_FLOWTABLE_NAME])
8587                 return -EINVAL;
8588
8589         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8590                                  genmask, 0);
8591         if (IS_ERR(table))
8592                 return PTR_ERR(table);
8593
8594         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
8595                                          genmask);
8596         if (IS_ERR(flowtable))
8597                 return PTR_ERR(flowtable);
8598
8599         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
8600         if (!skb2)
8601                 return -ENOMEM;
8602
8603         err = nf_tables_fill_flowtable_info(skb2, net, NETLINK_CB(skb).portid,
8604                                             info->nlh->nlmsg_seq,
8605                                             NFT_MSG_NEWFLOWTABLE, 0, family,
8606                                             flowtable, NULL);
8607         if (err < 0)
8608                 goto err_fill_flowtable_info;
8609
8610         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
8611
8612 err_fill_flowtable_info:
8613         kfree_skb(skb2);
8614         return err;
8615 }
8616
8617 static void nf_tables_flowtable_notify(struct nft_ctx *ctx,
8618                                        struct nft_flowtable *flowtable,
8619                                        struct list_head *hook_list, int event)
8620 {
8621         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
8622         struct sk_buff *skb;
8623         u16 flags = 0;
8624         int err;
8625
8626         if (!ctx->report &&
8627             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
8628                 return;
8629
8630         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
8631         if (skb == NULL)
8632                 goto err;
8633
8634         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
8635                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
8636
8637         err = nf_tables_fill_flowtable_info(skb, ctx->net, ctx->portid,
8638                                             ctx->seq, event, flags,
8639                                             ctx->family, flowtable, hook_list);
8640         if (err < 0) {
8641                 kfree_skb(skb);
8642                 goto err;
8643         }
8644
8645         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
8646         return;
8647 err:
8648         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
8649 }
8650
8651 static void nf_tables_flowtable_destroy(struct nft_flowtable *flowtable)
8652 {
8653         struct nft_hook *hook, *next;
8654
8655         flowtable->data.type->free(&flowtable->data);
8656         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
8657                 flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
8658                                             FLOW_BLOCK_UNBIND);
8659                 list_del_rcu(&hook->list);
8660                 kfree(hook);
8661         }
8662         kfree(flowtable->name);
8663         module_put(flowtable->data.type->owner);
8664         kfree(flowtable);
8665 }
8666
8667 static int nf_tables_fill_gen_info(struct sk_buff *skb, struct net *net,
8668                                    u32 portid, u32 seq)
8669 {
8670         struct nftables_pernet *nft_net = nft_pernet(net);
8671         struct nlmsghdr *nlh;
8672         char buf[TASK_COMM_LEN];
8673         int event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWGEN);
8674
8675         nlh = nfnl_msg_put(skb, portid, seq, event, 0, AF_UNSPEC,
8676                            NFNETLINK_V0, nft_base_seq(net));
8677         if (!nlh)
8678                 goto nla_put_failure;
8679
8680         if (nla_put_be32(skb, NFTA_GEN_ID, htonl(nft_net->base_seq)) ||
8681             nla_put_be32(skb, NFTA_GEN_PROC_PID, htonl(task_pid_nr(current))) ||
8682             nla_put_string(skb, NFTA_GEN_PROC_NAME, get_task_comm(buf, current)))
8683                 goto nla_put_failure;
8684
8685         nlmsg_end(skb, nlh);
8686         return 0;
8687
8688 nla_put_failure:
8689         nlmsg_trim(skb, nlh);
8690         return -EMSGSIZE;
8691 }
8692
8693 static void nft_flowtable_event(unsigned long event, struct net_device *dev,
8694                                 struct nft_flowtable *flowtable)
8695 {
8696         struct nft_hook *hook;
8697
8698         list_for_each_entry(hook, &flowtable->hook_list, list) {
8699                 if (hook->ops.dev != dev)
8700                         continue;
8701
8702                 /* flow_offload_netdev_event() cleans up entries for us. */
8703                 nft_unregister_flowtable_hook(dev_net(dev), flowtable, hook);
8704                 list_del_rcu(&hook->list);
8705                 kfree_rcu(hook, rcu);
8706                 break;
8707         }
8708 }
8709
8710 static int nf_tables_flowtable_event(struct notifier_block *this,
8711                                      unsigned long event, void *ptr)
8712 {
8713         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
8714         struct nft_flowtable *flowtable;
8715         struct nftables_pernet *nft_net;
8716         struct nft_table *table;
8717         struct net *net;
8718
8719         if (event != NETDEV_UNREGISTER)
8720                 return 0;
8721
8722         net = dev_net(dev);
8723         nft_net = nft_pernet(net);
8724         mutex_lock(&nft_net->commit_mutex);
8725         list_for_each_entry(table, &nft_net->tables, list) {
8726                 list_for_each_entry(flowtable, &table->flowtables, list) {
8727                         nft_flowtable_event(event, dev, flowtable);
8728                 }
8729         }
8730         mutex_unlock(&nft_net->commit_mutex);
8731
8732         return NOTIFY_DONE;
8733 }
8734
8735 static struct notifier_block nf_tables_flowtable_notifier = {
8736         .notifier_call  = nf_tables_flowtable_event,
8737 };
8738
8739 static void nf_tables_gen_notify(struct net *net, struct sk_buff *skb,
8740                                  int event)
8741 {
8742         struct nlmsghdr *nlh = nlmsg_hdr(skb);
8743         struct sk_buff *skb2;
8744         int err;
8745
8746         if (!nlmsg_report(nlh) &&
8747             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
8748                 return;
8749
8750         skb2 = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
8751         if (skb2 == NULL)
8752                 goto err;
8753
8754         err = nf_tables_fill_gen_info(skb2, net, NETLINK_CB(skb).portid,
8755                                       nlh->nlmsg_seq);
8756         if (err < 0) {
8757                 kfree_skb(skb2);
8758                 goto err;
8759         }
8760
8761         nfnetlink_send(skb2, net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
8762                        nlmsg_report(nlh), GFP_KERNEL);
8763         return;
8764 err:
8765         nfnetlink_set_err(net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
8766                           -ENOBUFS);
8767 }
8768
8769 static int nf_tables_getgen(struct sk_buff *skb, const struct nfnl_info *info,
8770                             const struct nlattr * const nla[])
8771 {
8772         struct sk_buff *skb2;
8773         int err;
8774
8775         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
8776         if (skb2 == NULL)
8777                 return -ENOMEM;
8778
8779         err = nf_tables_fill_gen_info(skb2, info->net, NETLINK_CB(skb).portid,
8780                                       info->nlh->nlmsg_seq);
8781         if (err < 0)
8782                 goto err_fill_gen_info;
8783
8784         return nfnetlink_unicast(skb2, info->net, NETLINK_CB(skb).portid);
8785
8786 err_fill_gen_info:
8787         kfree_skb(skb2);
8788         return err;
8789 }
8790
8791 static const struct nfnl_callback nf_tables_cb[NFT_MSG_MAX] = {
8792         [NFT_MSG_NEWTABLE] = {
8793                 .call           = nf_tables_newtable,
8794                 .type           = NFNL_CB_BATCH,
8795                 .attr_count     = NFTA_TABLE_MAX,
8796                 .policy         = nft_table_policy,
8797         },
8798         [NFT_MSG_GETTABLE] = {
8799                 .call           = nf_tables_gettable,
8800                 .type           = NFNL_CB_RCU,
8801                 .attr_count     = NFTA_TABLE_MAX,
8802                 .policy         = nft_table_policy,
8803         },
8804         [NFT_MSG_DELTABLE] = {
8805                 .call           = nf_tables_deltable,
8806                 .type           = NFNL_CB_BATCH,
8807                 .attr_count     = NFTA_TABLE_MAX,
8808                 .policy         = nft_table_policy,
8809         },
8810         [NFT_MSG_DESTROYTABLE] = {
8811                 .call           = nf_tables_deltable,
8812                 .type           = NFNL_CB_BATCH,
8813                 .attr_count     = NFTA_TABLE_MAX,
8814                 .policy         = nft_table_policy,
8815         },
8816         [NFT_MSG_NEWCHAIN] = {
8817                 .call           = nf_tables_newchain,
8818                 .type           = NFNL_CB_BATCH,
8819                 .attr_count     = NFTA_CHAIN_MAX,
8820                 .policy         = nft_chain_policy,
8821         },
8822         [NFT_MSG_GETCHAIN] = {
8823                 .call           = nf_tables_getchain,
8824                 .type           = NFNL_CB_RCU,
8825                 .attr_count     = NFTA_CHAIN_MAX,
8826                 .policy         = nft_chain_policy,
8827         },
8828         [NFT_MSG_DELCHAIN] = {
8829                 .call           = nf_tables_delchain,
8830                 .type           = NFNL_CB_BATCH,
8831                 .attr_count     = NFTA_CHAIN_MAX,
8832                 .policy         = nft_chain_policy,
8833         },
8834         [NFT_MSG_DESTROYCHAIN] = {
8835                 .call           = nf_tables_delchain,
8836                 .type           = NFNL_CB_BATCH,
8837                 .attr_count     = NFTA_CHAIN_MAX,
8838                 .policy         = nft_chain_policy,
8839         },
8840         [NFT_MSG_NEWRULE] = {
8841                 .call           = nf_tables_newrule,
8842                 .type           = NFNL_CB_BATCH,
8843                 .attr_count     = NFTA_RULE_MAX,
8844                 .policy         = nft_rule_policy,
8845         },
8846         [NFT_MSG_GETRULE] = {
8847                 .call           = nf_tables_getrule,
8848                 .type           = NFNL_CB_RCU,
8849                 .attr_count     = NFTA_RULE_MAX,
8850                 .policy         = nft_rule_policy,
8851         },
8852         [NFT_MSG_GETRULE_RESET] = {
8853                 .call           = nf_tables_getrule,
8854                 .type           = NFNL_CB_RCU,
8855                 .attr_count     = NFTA_RULE_MAX,
8856                 .policy         = nft_rule_policy,
8857         },
8858         [NFT_MSG_DELRULE] = {
8859                 .call           = nf_tables_delrule,
8860                 .type           = NFNL_CB_BATCH,
8861                 .attr_count     = NFTA_RULE_MAX,
8862                 .policy         = nft_rule_policy,
8863         },
8864         [NFT_MSG_DESTROYRULE] = {
8865                 .call           = nf_tables_delrule,
8866                 .type           = NFNL_CB_BATCH,
8867                 .attr_count     = NFTA_RULE_MAX,
8868                 .policy         = nft_rule_policy,
8869         },
8870         [NFT_MSG_NEWSET] = {
8871                 .call           = nf_tables_newset,
8872                 .type           = NFNL_CB_BATCH,
8873                 .attr_count     = NFTA_SET_MAX,
8874                 .policy         = nft_set_policy,
8875         },
8876         [NFT_MSG_GETSET] = {
8877                 .call           = nf_tables_getset,
8878                 .type           = NFNL_CB_RCU,
8879                 .attr_count     = NFTA_SET_MAX,
8880                 .policy         = nft_set_policy,
8881         },
8882         [NFT_MSG_DELSET] = {
8883                 .call           = nf_tables_delset,
8884                 .type           = NFNL_CB_BATCH,
8885                 .attr_count     = NFTA_SET_MAX,
8886                 .policy         = nft_set_policy,
8887         },
8888         [NFT_MSG_DESTROYSET] = {
8889                 .call           = nf_tables_delset,
8890                 .type           = NFNL_CB_BATCH,
8891                 .attr_count     = NFTA_SET_MAX,
8892                 .policy         = nft_set_policy,
8893         },
8894         [NFT_MSG_NEWSETELEM] = {
8895                 .call           = nf_tables_newsetelem,
8896                 .type           = NFNL_CB_BATCH,
8897                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
8898                 .policy         = nft_set_elem_list_policy,
8899         },
8900         [NFT_MSG_GETSETELEM] = {
8901                 .call           = nf_tables_getsetelem,
8902                 .type           = NFNL_CB_RCU,
8903                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
8904                 .policy         = nft_set_elem_list_policy,
8905         },
8906         [NFT_MSG_DELSETELEM] = {
8907                 .call           = nf_tables_delsetelem,
8908                 .type           = NFNL_CB_BATCH,
8909                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
8910                 .policy         = nft_set_elem_list_policy,
8911         },
8912         [NFT_MSG_DESTROYSETELEM] = {
8913                 .call           = nf_tables_delsetelem,
8914                 .type           = NFNL_CB_BATCH,
8915                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
8916                 .policy         = nft_set_elem_list_policy,
8917         },
8918         [NFT_MSG_GETGEN] = {
8919                 .call           = nf_tables_getgen,
8920                 .type           = NFNL_CB_RCU,
8921         },
8922         [NFT_MSG_NEWOBJ] = {
8923                 .call           = nf_tables_newobj,
8924                 .type           = NFNL_CB_BATCH,
8925                 .attr_count     = NFTA_OBJ_MAX,
8926                 .policy         = nft_obj_policy,
8927         },
8928         [NFT_MSG_GETOBJ] = {
8929                 .call           = nf_tables_getobj,
8930                 .type           = NFNL_CB_RCU,
8931                 .attr_count     = NFTA_OBJ_MAX,
8932                 .policy         = nft_obj_policy,
8933         },
8934         [NFT_MSG_DELOBJ] = {
8935                 .call           = nf_tables_delobj,
8936                 .type           = NFNL_CB_BATCH,
8937                 .attr_count     = NFTA_OBJ_MAX,
8938                 .policy         = nft_obj_policy,
8939         },
8940         [NFT_MSG_DESTROYOBJ] = {
8941                 .call           = nf_tables_delobj,
8942                 .type           = NFNL_CB_BATCH,
8943                 .attr_count     = NFTA_OBJ_MAX,
8944                 .policy         = nft_obj_policy,
8945         },
8946         [NFT_MSG_GETOBJ_RESET] = {
8947                 .call           = nf_tables_getobj,
8948                 .type           = NFNL_CB_RCU,
8949                 .attr_count     = NFTA_OBJ_MAX,
8950                 .policy         = nft_obj_policy,
8951         },
8952         [NFT_MSG_NEWFLOWTABLE] = {
8953                 .call           = nf_tables_newflowtable,
8954                 .type           = NFNL_CB_BATCH,
8955                 .attr_count     = NFTA_FLOWTABLE_MAX,
8956                 .policy         = nft_flowtable_policy,
8957         },
8958         [NFT_MSG_GETFLOWTABLE] = {
8959                 .call           = nf_tables_getflowtable,
8960                 .type           = NFNL_CB_RCU,
8961                 .attr_count     = NFTA_FLOWTABLE_MAX,
8962                 .policy         = nft_flowtable_policy,
8963         },
8964         [NFT_MSG_DELFLOWTABLE] = {
8965                 .call           = nf_tables_delflowtable,
8966                 .type           = NFNL_CB_BATCH,
8967                 .attr_count     = NFTA_FLOWTABLE_MAX,
8968                 .policy         = nft_flowtable_policy,
8969         },
8970         [NFT_MSG_DESTROYFLOWTABLE] = {
8971                 .call           = nf_tables_delflowtable,
8972                 .type           = NFNL_CB_BATCH,
8973                 .attr_count     = NFTA_FLOWTABLE_MAX,
8974                 .policy         = nft_flowtable_policy,
8975         },
8976 };
8977
8978 static int nf_tables_validate(struct net *net)
8979 {
8980         struct nftables_pernet *nft_net = nft_pernet(net);
8981         struct nft_table *table;
8982
8983         list_for_each_entry(table, &nft_net->tables, list) {
8984                 switch (table->validate_state) {
8985                 case NFT_VALIDATE_SKIP:
8986                         continue;
8987                 case NFT_VALIDATE_NEED:
8988                         nft_validate_state_update(table, NFT_VALIDATE_DO);
8989                         fallthrough;
8990                 case NFT_VALIDATE_DO:
8991                         if (nft_table_validate(net, table) < 0)
8992                                 return -EAGAIN;
8993
8994                         nft_validate_state_update(table, NFT_VALIDATE_SKIP);
8995                 }
8996
8997                 break;
8998         }
8999
9000         return 0;
9001 }
9002
9003 /* a drop policy has to be deferred until all rules have been activated,
9004  * otherwise a large ruleset that contains a drop-policy base chain will
9005  * cause all packets to get dropped until the full transaction has been
9006  * processed.
9007  *
9008  * We defer the drop policy until the transaction has been finalized.
9009  */
9010 static void nft_chain_commit_drop_policy(struct nft_trans *trans)
9011 {
9012         struct nft_base_chain *basechain;
9013
9014         if (nft_trans_chain_policy(trans) != NF_DROP)
9015                 return;
9016
9017         if (!nft_is_base_chain(trans->ctx.chain))
9018                 return;
9019
9020         basechain = nft_base_chain(trans->ctx.chain);
9021         basechain->policy = NF_DROP;
9022 }
9023
9024 static void nft_chain_commit_update(struct nft_trans *trans)
9025 {
9026         struct nft_base_chain *basechain;
9027
9028         if (nft_trans_chain_name(trans)) {
9029                 rhltable_remove(&trans->ctx.table->chains_ht,
9030                                 &trans->ctx.chain->rhlhead,
9031                                 nft_chain_ht_params);
9032                 swap(trans->ctx.chain->name, nft_trans_chain_name(trans));
9033                 rhltable_insert_key(&trans->ctx.table->chains_ht,
9034                                     trans->ctx.chain->name,
9035                                     &trans->ctx.chain->rhlhead,
9036                                     nft_chain_ht_params);
9037         }
9038
9039         if (!nft_is_base_chain(trans->ctx.chain))
9040                 return;
9041
9042         nft_chain_stats_replace(trans);
9043
9044         basechain = nft_base_chain(trans->ctx.chain);
9045
9046         switch (nft_trans_chain_policy(trans)) {
9047         case NF_DROP:
9048         case NF_ACCEPT:
9049                 basechain->policy = nft_trans_chain_policy(trans);
9050                 break;
9051         }
9052 }
9053
9054 static void nft_obj_commit_update(struct nft_trans *trans)
9055 {
9056         struct nft_object *newobj;
9057         struct nft_object *obj;
9058
9059         obj = nft_trans_obj(trans);
9060         newobj = nft_trans_obj_newobj(trans);
9061
9062         if (obj->ops->update)
9063                 obj->ops->update(obj, newobj);
9064
9065         nft_obj_destroy(&trans->ctx, newobj);
9066 }
9067
9068 static void nft_commit_release(struct nft_trans *trans)
9069 {
9070         switch (trans->msg_type) {
9071         case NFT_MSG_DELTABLE:
9072         case NFT_MSG_DESTROYTABLE:
9073                 nf_tables_table_destroy(&trans->ctx);
9074                 break;
9075         case NFT_MSG_NEWCHAIN:
9076                 free_percpu(nft_trans_chain_stats(trans));
9077                 kfree(nft_trans_chain_name(trans));
9078                 break;
9079         case NFT_MSG_DELCHAIN:
9080         case NFT_MSG_DESTROYCHAIN:
9081                 if (nft_trans_chain_update(trans))
9082                         nft_hooks_destroy(&nft_trans_chain_hooks(trans));
9083                 else
9084                         nf_tables_chain_destroy(&trans->ctx);
9085                 break;
9086         case NFT_MSG_DELRULE:
9087         case NFT_MSG_DESTROYRULE:
9088                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
9089                 break;
9090         case NFT_MSG_DELSET:
9091         case NFT_MSG_DESTROYSET:
9092                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
9093                 break;
9094         case NFT_MSG_DELSETELEM:
9095         case NFT_MSG_DESTROYSETELEM:
9096                 nf_tables_set_elem_destroy(&trans->ctx,
9097                                            nft_trans_elem_set(trans),
9098                                            nft_trans_elem(trans).priv);
9099                 break;
9100         case NFT_MSG_DELOBJ:
9101         case NFT_MSG_DESTROYOBJ:
9102                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
9103                 break;
9104         case NFT_MSG_DELFLOWTABLE:
9105         case NFT_MSG_DESTROYFLOWTABLE:
9106                 if (nft_trans_flowtable_update(trans))
9107                         nft_hooks_destroy(&nft_trans_flowtable_hooks(trans));
9108                 else
9109                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
9110                 break;
9111         }
9112
9113         if (trans->put_net)
9114                 put_net(trans->ctx.net);
9115
9116         kfree(trans);
9117 }
9118
9119 static void nf_tables_trans_destroy_work(struct work_struct *w)
9120 {
9121         struct nft_trans *trans, *next;
9122         LIST_HEAD(head);
9123
9124         spin_lock(&nf_tables_destroy_list_lock);
9125         list_splice_init(&nf_tables_destroy_list, &head);
9126         spin_unlock(&nf_tables_destroy_list_lock);
9127
9128         if (list_empty(&head))
9129                 return;
9130
9131         synchronize_rcu();
9132
9133         list_for_each_entry_safe(trans, next, &head, list) {
9134                 nft_trans_list_del(trans);
9135                 nft_commit_release(trans);
9136         }
9137 }
9138
9139 void nf_tables_trans_destroy_flush_work(void)
9140 {
9141         flush_work(&trans_destroy_work);
9142 }
9143 EXPORT_SYMBOL_GPL(nf_tables_trans_destroy_flush_work);
9144
9145 static bool nft_expr_reduce(struct nft_regs_track *track,
9146                             const struct nft_expr *expr)
9147 {
9148         return false;
9149 }
9150
9151 static int nf_tables_commit_chain_prepare(struct net *net, struct nft_chain *chain)
9152 {
9153         const struct nft_expr *expr, *last;
9154         struct nft_regs_track track = {};
9155         unsigned int size, data_size;
9156         void *data, *data_boundary;
9157         struct nft_rule_dp *prule;
9158         struct nft_rule *rule;
9159
9160         /* already handled or inactive chain? */
9161         if (chain->blob_next || !nft_is_active_next(net, chain))
9162                 return 0;
9163
9164         data_size = 0;
9165         list_for_each_entry(rule, &chain->rules, list) {
9166                 if (nft_is_active_next(net, rule)) {
9167                         data_size += sizeof(*prule) + rule->dlen;
9168                         if (data_size > INT_MAX)
9169                                 return -ENOMEM;
9170                 }
9171         }
9172
9173         chain->blob_next = nf_tables_chain_alloc_rules(chain, data_size);
9174         if (!chain->blob_next)
9175                 return -ENOMEM;
9176
9177         data = (void *)chain->blob_next->data;
9178         data_boundary = data + data_size;
9179         size = 0;
9180
9181         list_for_each_entry(rule, &chain->rules, list) {
9182                 if (!nft_is_active_next(net, rule))
9183                         continue;
9184
9185                 prule = (struct nft_rule_dp *)data;
9186                 data += offsetof(struct nft_rule_dp, data);
9187                 if (WARN_ON_ONCE(data > data_boundary))
9188                         return -ENOMEM;
9189
9190                 size = 0;
9191                 track.last = nft_expr_last(rule);
9192                 nft_rule_for_each_expr(expr, last, rule) {
9193                         track.cur = expr;
9194
9195                         if (nft_expr_reduce(&track, expr)) {
9196                                 expr = track.cur;
9197                                 continue;
9198                         }
9199
9200                         if (WARN_ON_ONCE(data + size + expr->ops->size > data_boundary))
9201                                 return -ENOMEM;
9202
9203                         memcpy(data + size, expr, expr->ops->size);
9204                         size += expr->ops->size;
9205                 }
9206                 if (WARN_ON_ONCE(size >= 1 << 12))
9207                         return -ENOMEM;
9208
9209                 prule->handle = rule->handle;
9210                 prule->dlen = size;
9211                 prule->is_last = 0;
9212
9213                 data += size;
9214                 size = 0;
9215                 chain->blob_next->size += (unsigned long)(data - (void *)prule);
9216         }
9217
9218         if (WARN_ON_ONCE(data > data_boundary))
9219                 return -ENOMEM;
9220
9221         prule = (struct nft_rule_dp *)data;
9222         nft_last_rule(chain, prule);
9223
9224         return 0;
9225 }
9226
9227 static void nf_tables_commit_chain_prepare_cancel(struct net *net)
9228 {
9229         struct nftables_pernet *nft_net = nft_pernet(net);
9230         struct nft_trans *trans, *next;
9231
9232         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
9233                 struct nft_chain *chain = trans->ctx.chain;
9234
9235                 if (trans->msg_type == NFT_MSG_NEWRULE ||
9236                     trans->msg_type == NFT_MSG_DELRULE) {
9237                         kvfree(chain->blob_next);
9238                         chain->blob_next = NULL;
9239                 }
9240         }
9241 }
9242
9243 static void __nf_tables_commit_chain_free_rules(struct rcu_head *h)
9244 {
9245         struct nft_rule_dp_last *l = container_of(h, struct nft_rule_dp_last, h);
9246
9247         kvfree(l->blob);
9248 }
9249
9250 static void nf_tables_commit_chain_free_rules_old(struct nft_rule_blob *blob)
9251 {
9252         struct nft_rule_dp_last *last;
9253
9254         /* last rule trailer is after end marker */
9255         last = (void *)blob + sizeof(*blob) + blob->size;
9256         last->blob = blob;
9257
9258         call_rcu(&last->h, __nf_tables_commit_chain_free_rules);
9259 }
9260
9261 static void nf_tables_commit_chain(struct net *net, struct nft_chain *chain)
9262 {
9263         struct nft_rule_blob *g0, *g1;
9264         bool next_genbit;
9265
9266         next_genbit = nft_gencursor_next(net);
9267
9268         g0 = rcu_dereference_protected(chain->blob_gen_0,
9269                                        lockdep_commit_lock_is_held(net));
9270         g1 = rcu_dereference_protected(chain->blob_gen_1,
9271                                        lockdep_commit_lock_is_held(net));
9272
9273         /* No changes to this chain? */
9274         if (chain->blob_next == NULL) {
9275                 /* chain had no change in last or next generation */
9276                 if (g0 == g1)
9277                         return;
9278                 /*
9279                  * chain had no change in this generation; make sure next
9280                  * one uses same rules as current generation.
9281                  */
9282                 if (next_genbit) {
9283                         rcu_assign_pointer(chain->blob_gen_1, g0);
9284                         nf_tables_commit_chain_free_rules_old(g1);
9285                 } else {
9286                         rcu_assign_pointer(chain->blob_gen_0, g1);
9287                         nf_tables_commit_chain_free_rules_old(g0);
9288                 }
9289
9290                 return;
9291         }
9292
9293         if (next_genbit)
9294                 rcu_assign_pointer(chain->blob_gen_1, chain->blob_next);
9295         else
9296                 rcu_assign_pointer(chain->blob_gen_0, chain->blob_next);
9297
9298         chain->blob_next = NULL;
9299
9300         if (g0 == g1)
9301                 return;
9302
9303         if (next_genbit)
9304                 nf_tables_commit_chain_free_rules_old(g1);
9305         else
9306                 nf_tables_commit_chain_free_rules_old(g0);
9307 }
9308
9309 static void nft_obj_del(struct nft_object *obj)
9310 {
9311         rhltable_remove(&nft_objname_ht, &obj->rhlhead, nft_objname_ht_params);
9312         list_del_rcu(&obj->list);
9313 }
9314
9315 void nft_chain_del(struct nft_chain *chain)
9316 {
9317         struct nft_table *table = chain->table;
9318
9319         WARN_ON_ONCE(rhltable_remove(&table->chains_ht, &chain->rhlhead,
9320                                      nft_chain_ht_params));
9321         list_del_rcu(&chain->list);
9322 }
9323
9324 static void nf_tables_module_autoload_cleanup(struct net *net)
9325 {
9326         struct nftables_pernet *nft_net = nft_pernet(net);
9327         struct nft_module_request *req, *next;
9328
9329         WARN_ON_ONCE(!list_empty(&nft_net->commit_list));
9330         list_for_each_entry_safe(req, next, &nft_net->module_list, list) {
9331                 WARN_ON_ONCE(!req->done);
9332                 list_del(&req->list);
9333                 kfree(req);
9334         }
9335 }
9336
9337 static void nf_tables_commit_release(struct net *net)
9338 {
9339         struct nftables_pernet *nft_net = nft_pernet(net);
9340         struct nft_trans *trans;
9341
9342         /* all side effects have to be made visible.
9343          * For example, if a chain named 'foo' has been deleted, a
9344          * new transaction must not find it anymore.
9345          *
9346          * Memory reclaim happens asynchronously from work queue
9347          * to prevent expensive synchronize_rcu() in commit phase.
9348          */
9349         if (list_empty(&nft_net->commit_list)) {
9350                 nf_tables_module_autoload_cleanup(net);
9351                 mutex_unlock(&nft_net->commit_mutex);
9352                 return;
9353         }
9354
9355         trans = list_last_entry(&nft_net->commit_list,
9356                                 struct nft_trans, list);
9357         get_net(trans->ctx.net);
9358         WARN_ON_ONCE(trans->put_net);
9359
9360         trans->put_net = true;
9361         spin_lock(&nf_tables_destroy_list_lock);
9362         list_splice_tail_init(&nft_net->commit_list, &nf_tables_destroy_list);
9363         spin_unlock(&nf_tables_destroy_list_lock);
9364
9365         nf_tables_module_autoload_cleanup(net);
9366         schedule_work(&trans_destroy_work);
9367
9368         mutex_unlock(&nft_net->commit_mutex);
9369 }
9370
9371 static void nft_commit_notify(struct net *net, u32 portid)
9372 {
9373         struct nftables_pernet *nft_net = nft_pernet(net);
9374         struct sk_buff *batch_skb = NULL, *nskb, *skb;
9375         unsigned char *data;
9376         int len;
9377
9378         list_for_each_entry_safe(skb, nskb, &nft_net->notify_list, list) {
9379                 if (!batch_skb) {
9380 new_batch:
9381                         batch_skb = skb;
9382                         len = NLMSG_GOODSIZE - skb->len;
9383                         list_del(&skb->list);
9384                         continue;
9385                 }
9386                 len -= skb->len;
9387                 if (len > 0 && NFT_CB(skb).report == NFT_CB(batch_skb).report) {
9388                         data = skb_put(batch_skb, skb->len);
9389                         memcpy(data, skb->data, skb->len);
9390                         list_del(&skb->list);
9391                         kfree_skb(skb);
9392                         continue;
9393                 }
9394                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
9395                                NFT_CB(batch_skb).report, GFP_KERNEL);
9396                 goto new_batch;
9397         }
9398
9399         if (batch_skb) {
9400                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
9401                                NFT_CB(batch_skb).report, GFP_KERNEL);
9402         }
9403
9404         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
9405 }
9406
9407 static int nf_tables_commit_audit_alloc(struct list_head *adl,
9408                                         struct nft_table *table)
9409 {
9410         struct nft_audit_data *adp;
9411
9412         list_for_each_entry(adp, adl, list) {
9413                 if (adp->table == table)
9414                         return 0;
9415         }
9416         adp = kzalloc(sizeof(*adp), GFP_KERNEL);
9417         if (!adp)
9418                 return -ENOMEM;
9419         adp->table = table;
9420         list_add(&adp->list, adl);
9421         return 0;
9422 }
9423
9424 static void nf_tables_commit_audit_free(struct list_head *adl)
9425 {
9426         struct nft_audit_data *adp, *adn;
9427
9428         list_for_each_entry_safe(adp, adn, adl, list) {
9429                 list_del(&adp->list);
9430                 kfree(adp);
9431         }
9432 }
9433
9434 static void nf_tables_commit_audit_collect(struct list_head *adl,
9435                                            struct nft_table *table, u32 op)
9436 {
9437         struct nft_audit_data *adp;
9438
9439         list_for_each_entry(adp, adl, list) {
9440                 if (adp->table == table)
9441                         goto found;
9442         }
9443         WARN_ONCE(1, "table=%s not expected in commit list", table->name);
9444         return;
9445 found:
9446         adp->entries++;
9447         if (!adp->op || adp->op > op)
9448                 adp->op = op;
9449 }
9450
9451 #define AUNFTABLENAMELEN (NFT_TABLE_MAXNAMELEN + 22)
9452
9453 static void nf_tables_commit_audit_log(struct list_head *adl, u32 generation)
9454 {
9455         struct nft_audit_data *adp, *adn;
9456         char aubuf[AUNFTABLENAMELEN];
9457
9458         list_for_each_entry_safe(adp, adn, adl, list) {
9459                 snprintf(aubuf, AUNFTABLENAMELEN, "%s:%u", adp->table->name,
9460                          generation);
9461                 audit_log_nfcfg(aubuf, adp->table->family, adp->entries,
9462                                 nft2audit_op[adp->op], GFP_KERNEL);
9463                 list_del(&adp->list);
9464                 kfree(adp);
9465         }
9466 }
9467
9468 static void nft_set_commit_update(struct list_head *set_update_list)
9469 {
9470         struct nft_set *set, *next;
9471
9472         list_for_each_entry_safe(set, next, set_update_list, pending_update) {
9473                 list_del_init(&set->pending_update);
9474
9475                 if (!set->ops->commit)
9476                         continue;
9477
9478                 set->ops->commit(set);
9479         }
9480 }
9481
9482 static int nf_tables_commit(struct net *net, struct sk_buff *skb)
9483 {
9484         struct nftables_pernet *nft_net = nft_pernet(net);
9485         struct nft_trans *trans, *next;
9486         LIST_HEAD(set_update_list);
9487         struct nft_trans_elem *te;
9488         struct nft_chain *chain;
9489         struct nft_table *table;
9490         unsigned int base_seq;
9491         LIST_HEAD(adl);
9492         int err;
9493
9494         if (list_empty(&nft_net->commit_list)) {
9495                 mutex_unlock(&nft_net->commit_mutex);
9496                 return 0;
9497         }
9498
9499         list_for_each_entry(trans, &nft_net->binding_list, binding_list) {
9500                 switch (trans->msg_type) {
9501                 case NFT_MSG_NEWSET:
9502                         if (!nft_trans_set_update(trans) &&
9503                             nft_set_is_anonymous(nft_trans_set(trans)) &&
9504                             !nft_trans_set_bound(trans)) {
9505                                 pr_warn_once("nftables ruleset with unbound set\n");
9506                                 return -EINVAL;
9507                         }
9508                         break;
9509                 case NFT_MSG_NEWCHAIN:
9510                         if (!nft_trans_chain_update(trans) &&
9511                             nft_chain_binding(nft_trans_chain(trans)) &&
9512                             !nft_trans_chain_bound(trans)) {
9513                                 pr_warn_once("nftables ruleset with unbound chain\n");
9514                                 return -EINVAL;
9515                         }
9516                         break;
9517                 }
9518         }
9519
9520         /* 0. Validate ruleset, otherwise roll back for error reporting. */
9521         if (nf_tables_validate(net) < 0)
9522                 return -EAGAIN;
9523
9524         err = nft_flow_rule_offload_commit(net);
9525         if (err < 0)
9526                 return err;
9527
9528         /* 1.  Allocate space for next generation rules_gen_X[] */
9529         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
9530                 int ret;
9531
9532                 ret = nf_tables_commit_audit_alloc(&adl, trans->ctx.table);
9533                 if (ret) {
9534                         nf_tables_commit_chain_prepare_cancel(net);
9535                         nf_tables_commit_audit_free(&adl);
9536                         return ret;
9537                 }
9538                 if (trans->msg_type == NFT_MSG_NEWRULE ||
9539                     trans->msg_type == NFT_MSG_DELRULE) {
9540                         chain = trans->ctx.chain;
9541
9542                         ret = nf_tables_commit_chain_prepare(net, chain);
9543                         if (ret < 0) {
9544                                 nf_tables_commit_chain_prepare_cancel(net);
9545                                 nf_tables_commit_audit_free(&adl);
9546                                 return ret;
9547                         }
9548                 }
9549         }
9550
9551         /* step 2.  Make rules_gen_X visible to packet path */
9552         list_for_each_entry(table, &nft_net->tables, list) {
9553                 list_for_each_entry(chain, &table->chains, list)
9554                         nf_tables_commit_chain(net, chain);
9555         }
9556
9557         /*
9558          * Bump generation counter, invalidate any dump in progress.
9559          * Cannot fail after this point.
9560          */
9561         base_seq = READ_ONCE(nft_net->base_seq);
9562         while (++base_seq == 0)
9563                 ;
9564
9565         WRITE_ONCE(nft_net->base_seq, base_seq);
9566
9567         /* step 3. Start new generation, rules_gen_X now in use. */
9568         net->nft.gencursor = nft_gencursor_next(net);
9569
9570         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
9571                 nf_tables_commit_audit_collect(&adl, trans->ctx.table,
9572                                                trans->msg_type);
9573                 switch (trans->msg_type) {
9574                 case NFT_MSG_NEWTABLE:
9575                         if (nft_trans_table_update(trans)) {
9576                                 if (!(trans->ctx.table->flags & __NFT_TABLE_F_UPDATE)) {
9577                                         nft_trans_destroy(trans);
9578                                         break;
9579                                 }
9580                                 if (trans->ctx.table->flags & NFT_TABLE_F_DORMANT)
9581                                         nf_tables_table_disable(net, trans->ctx.table);
9582
9583                                 trans->ctx.table->flags &= ~__NFT_TABLE_F_UPDATE;
9584                         } else {
9585                                 nft_clear(net, trans->ctx.table);
9586                         }
9587                         nf_tables_table_notify(&trans->ctx, NFT_MSG_NEWTABLE);
9588                         nft_trans_destroy(trans);
9589                         break;
9590                 case NFT_MSG_DELTABLE:
9591                 case NFT_MSG_DESTROYTABLE:
9592                         list_del_rcu(&trans->ctx.table->list);
9593                         nf_tables_table_notify(&trans->ctx, trans->msg_type);
9594                         break;
9595                 case NFT_MSG_NEWCHAIN:
9596                         if (nft_trans_chain_update(trans)) {
9597                                 nft_chain_commit_update(trans);
9598                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN,
9599                                                        &nft_trans_chain_hooks(trans));
9600                                 list_splice(&nft_trans_chain_hooks(trans),
9601                                             &nft_trans_basechain(trans)->hook_list);
9602                                 /* trans destroyed after rcu grace period */
9603                         } else {
9604                                 nft_chain_commit_drop_policy(trans);
9605                                 nft_clear(net, trans->ctx.chain);
9606                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN, NULL);
9607                                 nft_trans_destroy(trans);
9608                         }
9609                         break;
9610                 case NFT_MSG_DELCHAIN:
9611                 case NFT_MSG_DESTROYCHAIN:
9612                         if (nft_trans_chain_update(trans)) {
9613                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_DELCHAIN,
9614                                                        &nft_trans_chain_hooks(trans));
9615                                 nft_netdev_unregister_hooks(net,
9616                                                             &nft_trans_chain_hooks(trans),
9617                                                             true);
9618                         } else {
9619                                 nft_chain_del(trans->ctx.chain);
9620                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_DELCHAIN,
9621                                                        NULL);
9622                                 nf_tables_unregister_hook(trans->ctx.net,
9623                                                           trans->ctx.table,
9624                                                           trans->ctx.chain);
9625                         }
9626                         break;
9627                 case NFT_MSG_NEWRULE:
9628                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
9629                         nf_tables_rule_notify(&trans->ctx,
9630                                               nft_trans_rule(trans),
9631                                               NFT_MSG_NEWRULE);
9632                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
9633                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
9634
9635                         nft_trans_destroy(trans);
9636                         break;
9637                 case NFT_MSG_DELRULE:
9638                 case NFT_MSG_DESTROYRULE:
9639                         list_del_rcu(&nft_trans_rule(trans)->list);
9640                         nf_tables_rule_notify(&trans->ctx,
9641                                               nft_trans_rule(trans),
9642                                               trans->msg_type);
9643                         nft_rule_expr_deactivate(&trans->ctx,
9644                                                  nft_trans_rule(trans),
9645                                                  NFT_TRANS_COMMIT);
9646
9647                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
9648                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
9649                         break;
9650                 case NFT_MSG_NEWSET:
9651                         if (nft_trans_set_update(trans)) {
9652                                 struct nft_set *set = nft_trans_set(trans);
9653
9654                                 WRITE_ONCE(set->timeout, nft_trans_set_timeout(trans));
9655                                 WRITE_ONCE(set->gc_int, nft_trans_set_gc_int(trans));
9656                         } else {
9657                                 nft_clear(net, nft_trans_set(trans));
9658                                 /* This avoids hitting -EBUSY when deleting the table
9659                                  * from the transaction.
9660                                  */
9661                                 if (nft_set_is_anonymous(nft_trans_set(trans)) &&
9662                                     !list_empty(&nft_trans_set(trans)->bindings))
9663                                         trans->ctx.table->use--;
9664                         }
9665                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
9666                                              NFT_MSG_NEWSET, GFP_KERNEL);
9667                         nft_trans_destroy(trans);
9668                         break;
9669                 case NFT_MSG_DELSET:
9670                 case NFT_MSG_DESTROYSET:
9671                         list_del_rcu(&nft_trans_set(trans)->list);
9672                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
9673                                              trans->msg_type, GFP_KERNEL);
9674                         break;
9675                 case NFT_MSG_NEWSETELEM:
9676                         te = (struct nft_trans_elem *)trans->data;
9677
9678                         nft_setelem_activate(net, te->set, &te->elem);
9679                         nf_tables_setelem_notify(&trans->ctx, te->set,
9680                                                  &te->elem,
9681                                                  NFT_MSG_NEWSETELEM);
9682                         if (te->set->ops->commit &&
9683                             list_empty(&te->set->pending_update)) {
9684                                 list_add_tail(&te->set->pending_update,
9685                                               &set_update_list);
9686                         }
9687                         nft_trans_destroy(trans);
9688                         break;
9689                 case NFT_MSG_DELSETELEM:
9690                 case NFT_MSG_DESTROYSETELEM:
9691                         te = (struct nft_trans_elem *)trans->data;
9692
9693                         nf_tables_setelem_notify(&trans->ctx, te->set,
9694                                                  &te->elem,
9695                                                  trans->msg_type);
9696                         nft_setelem_remove(net, te->set, &te->elem);
9697                         if (!nft_setelem_is_catchall(te->set, &te->elem)) {
9698                                 atomic_dec(&te->set->nelems);
9699                                 te->set->ndeact--;
9700                         }
9701                         if (te->set->ops->commit &&
9702                             list_empty(&te->set->pending_update)) {
9703                                 list_add_tail(&te->set->pending_update,
9704                                               &set_update_list);
9705                         }
9706                         break;
9707                 case NFT_MSG_NEWOBJ:
9708                         if (nft_trans_obj_update(trans)) {
9709                                 nft_obj_commit_update(trans);
9710                                 nf_tables_obj_notify(&trans->ctx,
9711                                                      nft_trans_obj(trans),
9712                                                      NFT_MSG_NEWOBJ);
9713                         } else {
9714                                 nft_clear(net, nft_trans_obj(trans));
9715                                 nf_tables_obj_notify(&trans->ctx,
9716                                                      nft_trans_obj(trans),
9717                                                      NFT_MSG_NEWOBJ);
9718                                 nft_trans_destroy(trans);
9719                         }
9720                         break;
9721                 case NFT_MSG_DELOBJ:
9722                 case NFT_MSG_DESTROYOBJ:
9723                         nft_obj_del(nft_trans_obj(trans));
9724                         nf_tables_obj_notify(&trans->ctx, nft_trans_obj(trans),
9725                                              trans->msg_type);
9726                         break;
9727                 case NFT_MSG_NEWFLOWTABLE:
9728                         if (nft_trans_flowtable_update(trans)) {
9729                                 nft_trans_flowtable(trans)->data.flags =
9730                                         nft_trans_flowtable_flags(trans);
9731                                 nf_tables_flowtable_notify(&trans->ctx,
9732                                                            nft_trans_flowtable(trans),
9733                                                            &nft_trans_flowtable_hooks(trans),
9734                                                            NFT_MSG_NEWFLOWTABLE);
9735                                 list_splice(&nft_trans_flowtable_hooks(trans),
9736                                             &nft_trans_flowtable(trans)->hook_list);
9737                         } else {
9738                                 nft_clear(net, nft_trans_flowtable(trans));
9739                                 nf_tables_flowtable_notify(&trans->ctx,
9740                                                            nft_trans_flowtable(trans),
9741                                                            NULL,
9742                                                            NFT_MSG_NEWFLOWTABLE);
9743                         }
9744                         nft_trans_destroy(trans);
9745                         break;
9746                 case NFT_MSG_DELFLOWTABLE:
9747                 case NFT_MSG_DESTROYFLOWTABLE:
9748                         if (nft_trans_flowtable_update(trans)) {
9749                                 nf_tables_flowtable_notify(&trans->ctx,
9750                                                            nft_trans_flowtable(trans),
9751                                                            &nft_trans_flowtable_hooks(trans),
9752                                                            trans->msg_type);
9753                                 nft_unregister_flowtable_net_hooks(net,
9754                                                                    &nft_trans_flowtable_hooks(trans));
9755                         } else {
9756                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
9757                                 nf_tables_flowtable_notify(&trans->ctx,
9758                                                            nft_trans_flowtable(trans),
9759                                                            NULL,
9760                                                            trans->msg_type);
9761                                 nft_unregister_flowtable_net_hooks(net,
9762                                                 &nft_trans_flowtable(trans)->hook_list);
9763                         }
9764                         break;
9765                 }
9766         }
9767
9768         nft_set_commit_update(&set_update_list);
9769
9770         nft_commit_notify(net, NETLINK_CB(skb).portid);
9771         nf_tables_gen_notify(net, skb, NFT_MSG_NEWGEN);
9772         nf_tables_commit_audit_log(&adl, nft_net->base_seq);
9773         nf_tables_commit_release(net);
9774
9775         return 0;
9776 }
9777
9778 static void nf_tables_module_autoload(struct net *net)
9779 {
9780         struct nftables_pernet *nft_net = nft_pernet(net);
9781         struct nft_module_request *req, *next;
9782         LIST_HEAD(module_list);
9783
9784         list_splice_init(&nft_net->module_list, &module_list);
9785         mutex_unlock(&nft_net->commit_mutex);
9786         list_for_each_entry_safe(req, next, &module_list, list) {
9787                 request_module("%s", req->module);
9788                 req->done = true;
9789         }
9790         mutex_lock(&nft_net->commit_mutex);
9791         list_splice(&module_list, &nft_net->module_list);
9792 }
9793
9794 static void nf_tables_abort_release(struct nft_trans *trans)
9795 {
9796         switch (trans->msg_type) {
9797         case NFT_MSG_NEWTABLE:
9798                 nf_tables_table_destroy(&trans->ctx);
9799                 break;
9800         case NFT_MSG_NEWCHAIN:
9801                 if (nft_trans_chain_update(trans))
9802                         nft_hooks_destroy(&nft_trans_chain_hooks(trans));
9803                 else
9804                         nf_tables_chain_destroy(&trans->ctx);
9805                 break;
9806         case NFT_MSG_NEWRULE:
9807                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
9808                 break;
9809         case NFT_MSG_NEWSET:
9810                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
9811                 break;
9812         case NFT_MSG_NEWSETELEM:
9813                 nft_set_elem_destroy(nft_trans_elem_set(trans),
9814                                      nft_trans_elem(trans).priv, true);
9815                 break;
9816         case NFT_MSG_NEWOBJ:
9817                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
9818                 break;
9819         case NFT_MSG_NEWFLOWTABLE:
9820                 if (nft_trans_flowtable_update(trans))
9821                         nft_hooks_destroy(&nft_trans_flowtable_hooks(trans));
9822                 else
9823                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
9824                 break;
9825         }
9826         kfree(trans);
9827 }
9828
9829 static void nft_set_abort_update(struct list_head *set_update_list)
9830 {
9831         struct nft_set *set, *next;
9832
9833         list_for_each_entry_safe(set, next, set_update_list, pending_update) {
9834                 list_del_init(&set->pending_update);
9835
9836                 if (!set->ops->abort)
9837                         continue;
9838
9839                 set->ops->abort(set);
9840         }
9841 }
9842
9843 static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action)
9844 {
9845         struct nftables_pernet *nft_net = nft_pernet(net);
9846         struct nft_trans *trans, *next;
9847         LIST_HEAD(set_update_list);
9848         struct nft_trans_elem *te;
9849
9850         if (action == NFNL_ABORT_VALIDATE &&
9851             nf_tables_validate(net) < 0)
9852                 return -EAGAIN;
9853
9854         list_for_each_entry_safe_reverse(trans, next, &nft_net->commit_list,
9855                                          list) {
9856                 switch (trans->msg_type) {
9857                 case NFT_MSG_NEWTABLE:
9858                         if (nft_trans_table_update(trans)) {
9859                                 if (!(trans->ctx.table->flags & __NFT_TABLE_F_UPDATE)) {
9860                                         nft_trans_destroy(trans);
9861                                         break;
9862                                 }
9863                                 if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_DORMANT) {
9864                                         nf_tables_table_disable(net, trans->ctx.table);
9865                                         trans->ctx.table->flags |= NFT_TABLE_F_DORMANT;
9866                                 } else if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_AWAKEN) {
9867                                         trans->ctx.table->flags &= ~NFT_TABLE_F_DORMANT;
9868                                 }
9869                                 trans->ctx.table->flags &= ~__NFT_TABLE_F_UPDATE;
9870                                 nft_trans_destroy(trans);
9871                         } else {
9872                                 list_del_rcu(&trans->ctx.table->list);
9873                         }
9874                         break;
9875                 case NFT_MSG_DELTABLE:
9876                 case NFT_MSG_DESTROYTABLE:
9877                         nft_clear(trans->ctx.net, trans->ctx.table);
9878                         nft_trans_destroy(trans);
9879                         break;
9880                 case NFT_MSG_NEWCHAIN:
9881                         if (nft_trans_chain_update(trans)) {
9882                                 nft_netdev_unregister_hooks(net,
9883                                                             &nft_trans_chain_hooks(trans),
9884                                                             true);
9885                                 free_percpu(nft_trans_chain_stats(trans));
9886                                 kfree(nft_trans_chain_name(trans));
9887                                 nft_trans_destroy(trans);
9888                         } else {
9889                                 if (nft_trans_chain_bound(trans)) {
9890                                         nft_trans_destroy(trans);
9891                                         break;
9892                                 }
9893                                 trans->ctx.table->use--;
9894                                 nft_chain_del(trans->ctx.chain);
9895                                 nf_tables_unregister_hook(trans->ctx.net,
9896                                                           trans->ctx.table,
9897                                                           trans->ctx.chain);
9898                         }
9899                         break;
9900                 case NFT_MSG_DELCHAIN:
9901                 case NFT_MSG_DESTROYCHAIN:
9902                         if (nft_trans_chain_update(trans)) {
9903                                 list_splice(&nft_trans_chain_hooks(trans),
9904                                             &nft_trans_basechain(trans)->hook_list);
9905                         } else {
9906                                 trans->ctx.table->use++;
9907                                 nft_clear(trans->ctx.net, trans->ctx.chain);
9908                         }
9909                         nft_trans_destroy(trans);
9910                         break;
9911                 case NFT_MSG_NEWRULE:
9912                         if (nft_trans_rule_bound(trans)) {
9913                                 nft_trans_destroy(trans);
9914                                 break;
9915                         }
9916                         trans->ctx.chain->use--;
9917                         list_del_rcu(&nft_trans_rule(trans)->list);
9918                         nft_rule_expr_deactivate(&trans->ctx,
9919                                                  nft_trans_rule(trans),
9920                                                  NFT_TRANS_ABORT);
9921                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
9922                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
9923                         break;
9924                 case NFT_MSG_DELRULE:
9925                 case NFT_MSG_DESTROYRULE:
9926                         trans->ctx.chain->use++;
9927                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
9928                         nft_rule_expr_activate(&trans->ctx, nft_trans_rule(trans));
9929                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
9930                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
9931
9932                         nft_trans_destroy(trans);
9933                         break;
9934                 case NFT_MSG_NEWSET:
9935                         if (nft_trans_set_update(trans)) {
9936                                 nft_trans_destroy(trans);
9937                                 break;
9938                         }
9939                         trans->ctx.table->use--;
9940                         if (nft_trans_set_bound(trans)) {
9941                                 nft_trans_destroy(trans);
9942                                 break;
9943                         }
9944                         list_del_rcu(&nft_trans_set(trans)->list);
9945                         break;
9946                 case NFT_MSG_DELSET:
9947                 case NFT_MSG_DESTROYSET:
9948                         trans->ctx.table->use++;
9949                         nft_clear(trans->ctx.net, nft_trans_set(trans));
9950                         if (nft_trans_set(trans)->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
9951                                 nft_map_activate(&trans->ctx, nft_trans_set(trans));
9952
9953                         nft_trans_destroy(trans);
9954                         break;
9955                 case NFT_MSG_NEWSETELEM:
9956                         if (nft_trans_elem_set_bound(trans)) {
9957                                 nft_trans_destroy(trans);
9958                                 break;
9959                         }
9960                         te = (struct nft_trans_elem *)trans->data;
9961                         nft_setelem_remove(net, te->set, &te->elem);
9962                         if (!nft_setelem_is_catchall(te->set, &te->elem))
9963                                 atomic_dec(&te->set->nelems);
9964
9965                         if (te->set->ops->abort &&
9966                             list_empty(&te->set->pending_update)) {
9967                                 list_add_tail(&te->set->pending_update,
9968                                               &set_update_list);
9969                         }
9970                         break;
9971                 case NFT_MSG_DELSETELEM:
9972                 case NFT_MSG_DESTROYSETELEM:
9973                         te = (struct nft_trans_elem *)trans->data;
9974
9975                         nft_setelem_data_activate(net, te->set, &te->elem);
9976                         nft_setelem_activate(net, te->set, &te->elem);
9977                         if (!nft_setelem_is_catchall(te->set, &te->elem))
9978                                 te->set->ndeact--;
9979
9980                         if (te->set->ops->abort &&
9981                             list_empty(&te->set->pending_update)) {
9982                                 list_add_tail(&te->set->pending_update,
9983                                               &set_update_list);
9984                         }
9985                         nft_trans_destroy(trans);
9986                         break;
9987                 case NFT_MSG_NEWOBJ:
9988                         if (nft_trans_obj_update(trans)) {
9989                                 nft_obj_destroy(&trans->ctx, nft_trans_obj_newobj(trans));
9990                                 nft_trans_destroy(trans);
9991                         } else {
9992                                 trans->ctx.table->use--;
9993                                 nft_obj_del(nft_trans_obj(trans));
9994                         }
9995                         break;
9996                 case NFT_MSG_DELOBJ:
9997                 case NFT_MSG_DESTROYOBJ:
9998                         trans->ctx.table->use++;
9999                         nft_clear(trans->ctx.net, nft_trans_obj(trans));
10000                         nft_trans_destroy(trans);
10001                         break;
10002                 case NFT_MSG_NEWFLOWTABLE:
10003                         if (nft_trans_flowtable_update(trans)) {
10004                                 nft_unregister_flowtable_net_hooks(net,
10005                                                 &nft_trans_flowtable_hooks(trans));
10006                         } else {
10007                                 trans->ctx.table->use--;
10008                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
10009                                 nft_unregister_flowtable_net_hooks(net,
10010                                                 &nft_trans_flowtable(trans)->hook_list);
10011                         }
10012                         break;
10013                 case NFT_MSG_DELFLOWTABLE:
10014                 case NFT_MSG_DESTROYFLOWTABLE:
10015                         if (nft_trans_flowtable_update(trans)) {
10016                                 list_splice(&nft_trans_flowtable_hooks(trans),
10017                                             &nft_trans_flowtable(trans)->hook_list);
10018                         } else {
10019                                 trans->ctx.table->use++;
10020                                 nft_clear(trans->ctx.net, nft_trans_flowtable(trans));
10021                         }
10022                         nft_trans_destroy(trans);
10023                         break;
10024                 }
10025         }
10026
10027         nft_set_abort_update(&set_update_list);
10028
10029         synchronize_rcu();
10030
10031         list_for_each_entry_safe_reverse(trans, next,
10032                                          &nft_net->commit_list, list) {
10033                 nft_trans_list_del(trans);
10034                 nf_tables_abort_release(trans);
10035         }
10036
10037         if (action == NFNL_ABORT_AUTOLOAD)
10038                 nf_tables_module_autoload(net);
10039         else
10040                 nf_tables_module_autoload_cleanup(net);
10041
10042         return 0;
10043 }
10044
10045 static int nf_tables_abort(struct net *net, struct sk_buff *skb,
10046                            enum nfnl_abort_action action)
10047 {
10048         struct nftables_pernet *nft_net = nft_pernet(net);
10049         int ret = __nf_tables_abort(net, action);
10050
10051         mutex_unlock(&nft_net->commit_mutex);
10052
10053         return ret;
10054 }
10055
10056 static bool nf_tables_valid_genid(struct net *net, u32 genid)
10057 {
10058         struct nftables_pernet *nft_net = nft_pernet(net);
10059         bool genid_ok;
10060
10061         mutex_lock(&nft_net->commit_mutex);
10062
10063         genid_ok = genid == 0 || nft_net->base_seq == genid;
10064         if (!genid_ok)
10065                 mutex_unlock(&nft_net->commit_mutex);
10066
10067         /* else, commit mutex has to be released by commit or abort function */
10068         return genid_ok;
10069 }
10070
10071 static const struct nfnetlink_subsystem nf_tables_subsys = {
10072         .name           = "nf_tables",
10073         .subsys_id      = NFNL_SUBSYS_NFTABLES,
10074         .cb_count       = NFT_MSG_MAX,
10075         .cb             = nf_tables_cb,
10076         .commit         = nf_tables_commit,
10077         .abort          = nf_tables_abort,
10078         .valid_genid    = nf_tables_valid_genid,
10079         .owner          = THIS_MODULE,
10080 };
10081
10082 int nft_chain_validate_dependency(const struct nft_chain *chain,
10083                                   enum nft_chain_types type)
10084 {
10085         const struct nft_base_chain *basechain;
10086
10087         if (nft_is_base_chain(chain)) {
10088                 basechain = nft_base_chain(chain);
10089                 if (basechain->type->type != type)
10090                         return -EOPNOTSUPP;
10091         }
10092         return 0;
10093 }
10094 EXPORT_SYMBOL_GPL(nft_chain_validate_dependency);
10095
10096 int nft_chain_validate_hooks(const struct nft_chain *chain,
10097                              unsigned int hook_flags)
10098 {
10099         struct nft_base_chain *basechain;
10100
10101         if (nft_is_base_chain(chain)) {
10102                 basechain = nft_base_chain(chain);
10103
10104                 if ((1 << basechain->ops.hooknum) & hook_flags)
10105                         return 0;
10106
10107                 return -EOPNOTSUPP;
10108         }
10109
10110         return 0;
10111 }
10112 EXPORT_SYMBOL_GPL(nft_chain_validate_hooks);
10113
10114 /*
10115  * Loop detection - walk through the ruleset beginning at the destination chain
10116  * of a new jump until either the source chain is reached (loop) or all
10117  * reachable chains have been traversed.
10118  *
10119  * The loop check is performed whenever a new jump verdict is added to an
10120  * expression or verdict map or a verdict map is bound to a new chain.
10121  */
10122
10123 static int nf_tables_check_loops(const struct nft_ctx *ctx,
10124                                  const struct nft_chain *chain);
10125
10126 static int nft_check_loops(const struct nft_ctx *ctx,
10127                            const struct nft_set_ext *ext)
10128 {
10129         const struct nft_data *data;
10130         int ret;
10131
10132         data = nft_set_ext_data(ext);
10133         switch (data->verdict.code) {
10134         case NFT_JUMP:
10135         case NFT_GOTO:
10136                 ret = nf_tables_check_loops(ctx, data->verdict.chain);
10137                 break;
10138         default:
10139                 ret = 0;
10140                 break;
10141         }
10142
10143         return ret;
10144 }
10145
10146 static int nf_tables_loop_check_setelem(const struct nft_ctx *ctx,
10147                                         struct nft_set *set,
10148                                         const struct nft_set_iter *iter,
10149                                         struct nft_set_elem *elem)
10150 {
10151         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
10152
10153         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
10154             *nft_set_ext_flags(ext) & NFT_SET_ELEM_INTERVAL_END)
10155                 return 0;
10156
10157         return nft_check_loops(ctx, ext);
10158 }
10159
10160 static int nft_set_catchall_loops(const struct nft_ctx *ctx,
10161                                   struct nft_set *set)
10162 {
10163         u8 genmask = nft_genmask_next(ctx->net);
10164         struct nft_set_elem_catchall *catchall;
10165         struct nft_set_ext *ext;
10166         int ret = 0;
10167
10168         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
10169                 ext = nft_set_elem_ext(set, catchall->elem);
10170                 if (!nft_set_elem_active(ext, genmask))
10171                         continue;
10172
10173                 ret = nft_check_loops(ctx, ext);
10174                 if (ret < 0)
10175                         return ret;
10176         }
10177
10178         return ret;
10179 }
10180
10181 static int nf_tables_check_loops(const struct nft_ctx *ctx,
10182                                  const struct nft_chain *chain)
10183 {
10184         const struct nft_rule *rule;
10185         const struct nft_expr *expr, *last;
10186         struct nft_set *set;
10187         struct nft_set_binding *binding;
10188         struct nft_set_iter iter;
10189
10190         if (ctx->chain == chain)
10191                 return -ELOOP;
10192
10193         list_for_each_entry(rule, &chain->rules, list) {
10194                 nft_rule_for_each_expr(expr, last, rule) {
10195                         struct nft_immediate_expr *priv;
10196                         const struct nft_data *data;
10197                         int err;
10198
10199                         if (strcmp(expr->ops->type->name, "immediate"))
10200                                 continue;
10201
10202                         priv = nft_expr_priv(expr);
10203                         if (priv->dreg != NFT_REG_VERDICT)
10204                                 continue;
10205
10206                         data = &priv->data;
10207                         switch (data->verdict.code) {
10208                         case NFT_JUMP:
10209                         case NFT_GOTO:
10210                                 err = nf_tables_check_loops(ctx,
10211                                                         data->verdict.chain);
10212                                 if (err < 0)
10213                                         return err;
10214                                 break;
10215                         default:
10216                                 break;
10217                         }
10218                 }
10219         }
10220
10221         list_for_each_entry(set, &ctx->table->sets, list) {
10222                 if (!nft_is_active_next(ctx->net, set))
10223                         continue;
10224                 if (!(set->flags & NFT_SET_MAP) ||
10225                     set->dtype != NFT_DATA_VERDICT)
10226                         continue;
10227
10228                 list_for_each_entry(binding, &set->bindings, list) {
10229                         if (!(binding->flags & NFT_SET_MAP) ||
10230                             binding->chain != chain)
10231                                 continue;
10232
10233                         iter.genmask    = nft_genmask_next(ctx->net);
10234                         iter.skip       = 0;
10235                         iter.count      = 0;
10236                         iter.err        = 0;
10237                         iter.fn         = nf_tables_loop_check_setelem;
10238
10239                         set->ops->walk(ctx, set, &iter);
10240                         if (!iter.err)
10241                                 iter.err = nft_set_catchall_loops(ctx, set);
10242
10243                         if (iter.err < 0)
10244                                 return iter.err;
10245                 }
10246         }
10247
10248         return 0;
10249 }
10250
10251 /**
10252  *      nft_parse_u32_check - fetch u32 attribute and check for maximum value
10253  *
10254  *      @attr: netlink attribute to fetch value from
10255  *      @max: maximum value to be stored in dest
10256  *      @dest: pointer to the variable
10257  *
10258  *      Parse, check and store a given u32 netlink attribute into variable.
10259  *      This function returns -ERANGE if the value goes over maximum value.
10260  *      Otherwise a 0 is returned and the attribute value is stored in the
10261  *      destination variable.
10262  */
10263 int nft_parse_u32_check(const struct nlattr *attr, int max, u32 *dest)
10264 {
10265         u32 val;
10266
10267         val = ntohl(nla_get_be32(attr));
10268         if (val > max)
10269                 return -ERANGE;
10270
10271         *dest = val;
10272         return 0;
10273 }
10274 EXPORT_SYMBOL_GPL(nft_parse_u32_check);
10275
10276 static int nft_parse_register(const struct nlattr *attr, u32 *preg)
10277 {
10278         unsigned int reg;
10279
10280         reg = ntohl(nla_get_be32(attr));
10281         switch (reg) {
10282         case NFT_REG_VERDICT...NFT_REG_4:
10283                 *preg = reg * NFT_REG_SIZE / NFT_REG32_SIZE;
10284                 break;
10285         case NFT_REG32_00...NFT_REG32_15:
10286                 *preg = reg + NFT_REG_SIZE / NFT_REG32_SIZE - NFT_REG32_00;
10287                 break;
10288         default:
10289                 return -ERANGE;
10290         }
10291
10292         return 0;
10293 }
10294
10295 /**
10296  *      nft_dump_register - dump a register value to a netlink attribute
10297  *
10298  *      @skb: socket buffer
10299  *      @attr: attribute number
10300  *      @reg: register number
10301  *
10302  *      Construct a netlink attribute containing the register number. For
10303  *      compatibility reasons, register numbers being a multiple of 4 are
10304  *      translated to the corresponding 128 bit register numbers.
10305  */
10306 int nft_dump_register(struct sk_buff *skb, unsigned int attr, unsigned int reg)
10307 {
10308         if (reg % (NFT_REG_SIZE / NFT_REG32_SIZE) == 0)
10309                 reg = reg / (NFT_REG_SIZE / NFT_REG32_SIZE);
10310         else
10311                 reg = reg - NFT_REG_SIZE / NFT_REG32_SIZE + NFT_REG32_00;
10312
10313         return nla_put_be32(skb, attr, htonl(reg));
10314 }
10315 EXPORT_SYMBOL_GPL(nft_dump_register);
10316
10317 static int nft_validate_register_load(enum nft_registers reg, unsigned int len)
10318 {
10319         if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
10320                 return -EINVAL;
10321         if (len == 0)
10322                 return -EINVAL;
10323         if (reg * NFT_REG32_SIZE + len > sizeof_field(struct nft_regs, data))
10324                 return -ERANGE;
10325
10326         return 0;
10327 }
10328
10329 int nft_parse_register_load(const struct nlattr *attr, u8 *sreg, u32 len)
10330 {
10331         u32 reg;
10332         int err;
10333
10334         err = nft_parse_register(attr, &reg);
10335         if (err < 0)
10336                 return err;
10337
10338         err = nft_validate_register_load(reg, len);
10339         if (err < 0)
10340                 return err;
10341
10342         *sreg = reg;
10343         return 0;
10344 }
10345 EXPORT_SYMBOL_GPL(nft_parse_register_load);
10346
10347 static int nft_validate_register_store(const struct nft_ctx *ctx,
10348                                        enum nft_registers reg,
10349                                        const struct nft_data *data,
10350                                        enum nft_data_types type,
10351                                        unsigned int len)
10352 {
10353         int err;
10354
10355         switch (reg) {
10356         case NFT_REG_VERDICT:
10357                 if (type != NFT_DATA_VERDICT)
10358                         return -EINVAL;
10359
10360                 if (data != NULL &&
10361                     (data->verdict.code == NFT_GOTO ||
10362                      data->verdict.code == NFT_JUMP)) {
10363                         err = nf_tables_check_loops(ctx, data->verdict.chain);
10364                         if (err < 0)
10365                                 return err;
10366                 }
10367
10368                 return 0;
10369         default:
10370                 if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
10371                         return -EINVAL;
10372                 if (len == 0)
10373                         return -EINVAL;
10374                 if (reg * NFT_REG32_SIZE + len >
10375                     sizeof_field(struct nft_regs, data))
10376                         return -ERANGE;
10377
10378                 if (data != NULL && type != NFT_DATA_VALUE)
10379                         return -EINVAL;
10380                 return 0;
10381         }
10382 }
10383
10384 int nft_parse_register_store(const struct nft_ctx *ctx,
10385                              const struct nlattr *attr, u8 *dreg,
10386                              const struct nft_data *data,
10387                              enum nft_data_types type, unsigned int len)
10388 {
10389         int err;
10390         u32 reg;
10391
10392         err = nft_parse_register(attr, &reg);
10393         if (err < 0)
10394                 return err;
10395
10396         err = nft_validate_register_store(ctx, reg, data, type, len);
10397         if (err < 0)
10398                 return err;
10399
10400         *dreg = reg;
10401         return 0;
10402 }
10403 EXPORT_SYMBOL_GPL(nft_parse_register_store);
10404
10405 static const struct nla_policy nft_verdict_policy[NFTA_VERDICT_MAX + 1] = {
10406         [NFTA_VERDICT_CODE]     = { .type = NLA_U32 },
10407         [NFTA_VERDICT_CHAIN]    = { .type = NLA_STRING,
10408                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
10409         [NFTA_VERDICT_CHAIN_ID] = { .type = NLA_U32 },
10410 };
10411
10412 static int nft_verdict_init(const struct nft_ctx *ctx, struct nft_data *data,
10413                             struct nft_data_desc *desc, const struct nlattr *nla)
10414 {
10415         u8 genmask = nft_genmask_next(ctx->net);
10416         struct nlattr *tb[NFTA_VERDICT_MAX + 1];
10417         struct nft_chain *chain;
10418         int err;
10419
10420         err = nla_parse_nested_deprecated(tb, NFTA_VERDICT_MAX, nla,
10421                                           nft_verdict_policy, NULL);
10422         if (err < 0)
10423                 return err;
10424
10425         if (!tb[NFTA_VERDICT_CODE])
10426                 return -EINVAL;
10427         data->verdict.code = ntohl(nla_get_be32(tb[NFTA_VERDICT_CODE]));
10428
10429         switch (data->verdict.code) {
10430         default:
10431                 switch (data->verdict.code & NF_VERDICT_MASK) {
10432                 case NF_ACCEPT:
10433                 case NF_DROP:
10434                 case NF_QUEUE:
10435                         break;
10436                 default:
10437                         return -EINVAL;
10438                 }
10439                 fallthrough;
10440         case NFT_CONTINUE:
10441         case NFT_BREAK:
10442         case NFT_RETURN:
10443                 break;
10444         case NFT_JUMP:
10445         case NFT_GOTO:
10446                 if (tb[NFTA_VERDICT_CHAIN]) {
10447                         chain = nft_chain_lookup(ctx->net, ctx->table,
10448                                                  tb[NFTA_VERDICT_CHAIN],
10449                                                  genmask);
10450                 } else if (tb[NFTA_VERDICT_CHAIN_ID]) {
10451                         chain = nft_chain_lookup_byid(ctx->net, ctx->table,
10452                                                       tb[NFTA_VERDICT_CHAIN_ID]);
10453                         if (IS_ERR(chain))
10454                                 return PTR_ERR(chain);
10455                 } else {
10456                         return -EINVAL;
10457                 }
10458
10459                 if (IS_ERR(chain))
10460                         return PTR_ERR(chain);
10461                 if (nft_is_base_chain(chain))
10462                         return -EOPNOTSUPP;
10463                 if (nft_chain_is_bound(chain))
10464                         return -EINVAL;
10465                 if (desc->flags & NFT_DATA_DESC_SETELEM &&
10466                     chain->flags & NFT_CHAIN_BINDING)
10467                         return -EINVAL;
10468
10469                 chain->use++;
10470                 data->verdict.chain = chain;
10471                 break;
10472         }
10473
10474         desc->len = sizeof(data->verdict);
10475
10476         return 0;
10477 }
10478
10479 static void nft_verdict_uninit(const struct nft_data *data)
10480 {
10481         struct nft_chain *chain;
10482
10483         switch (data->verdict.code) {
10484         case NFT_JUMP:
10485         case NFT_GOTO:
10486                 chain = data->verdict.chain;
10487                 chain->use--;
10488                 break;
10489         }
10490 }
10491
10492 int nft_verdict_dump(struct sk_buff *skb, int type, const struct nft_verdict *v)
10493 {
10494         struct nlattr *nest;
10495
10496         nest = nla_nest_start_noflag(skb, type);
10497         if (!nest)
10498                 goto nla_put_failure;
10499
10500         if (nla_put_be32(skb, NFTA_VERDICT_CODE, htonl(v->code)))
10501                 goto nla_put_failure;
10502
10503         switch (v->code) {
10504         case NFT_JUMP:
10505         case NFT_GOTO:
10506                 if (nla_put_string(skb, NFTA_VERDICT_CHAIN,
10507                                    v->chain->name))
10508                         goto nla_put_failure;
10509         }
10510         nla_nest_end(skb, nest);
10511         return 0;
10512
10513 nla_put_failure:
10514         return -1;
10515 }
10516
10517 static int nft_value_init(const struct nft_ctx *ctx,
10518                           struct nft_data *data, struct nft_data_desc *desc,
10519                           const struct nlattr *nla)
10520 {
10521         unsigned int len;
10522
10523         len = nla_len(nla);
10524         if (len == 0)
10525                 return -EINVAL;
10526         if (len > desc->size)
10527                 return -EOVERFLOW;
10528         if (desc->len) {
10529                 if (len != desc->len)
10530                         return -EINVAL;
10531         } else {
10532                 desc->len = len;
10533         }
10534
10535         nla_memcpy(data->data, nla, len);
10536
10537         return 0;
10538 }
10539
10540 static int nft_value_dump(struct sk_buff *skb, const struct nft_data *data,
10541                           unsigned int len)
10542 {
10543         return nla_put(skb, NFTA_DATA_VALUE, len, data->data);
10544 }
10545
10546 static const struct nla_policy nft_data_policy[NFTA_DATA_MAX + 1] = {
10547         [NFTA_DATA_VALUE]       = { .type = NLA_BINARY },
10548         [NFTA_DATA_VERDICT]     = { .type = NLA_NESTED },
10549 };
10550
10551 /**
10552  *      nft_data_init - parse nf_tables data netlink attributes
10553  *
10554  *      @ctx: context of the expression using the data
10555  *      @data: destination struct nft_data
10556  *      @desc: data description
10557  *      @nla: netlink attribute containing data
10558  *
10559  *      Parse the netlink data attributes and initialize a struct nft_data.
10560  *      The type and length of data are returned in the data description.
10561  *
10562  *      The caller can indicate that it only wants to accept data of type
10563  *      NFT_DATA_VALUE by passing NULL for the ctx argument.
10564  */
10565 int nft_data_init(const struct nft_ctx *ctx, struct nft_data *data,
10566                   struct nft_data_desc *desc, const struct nlattr *nla)
10567 {
10568         struct nlattr *tb[NFTA_DATA_MAX + 1];
10569         int err;
10570
10571         if (WARN_ON_ONCE(!desc->size))
10572                 return -EINVAL;
10573
10574         err = nla_parse_nested_deprecated(tb, NFTA_DATA_MAX, nla,
10575                                           nft_data_policy, NULL);
10576         if (err < 0)
10577                 return err;
10578
10579         if (tb[NFTA_DATA_VALUE]) {
10580                 if (desc->type != NFT_DATA_VALUE)
10581                         return -EINVAL;
10582
10583                 err = nft_value_init(ctx, data, desc, tb[NFTA_DATA_VALUE]);
10584         } else if (tb[NFTA_DATA_VERDICT] && ctx != NULL) {
10585                 if (desc->type != NFT_DATA_VERDICT)
10586                         return -EINVAL;
10587
10588                 err = nft_verdict_init(ctx, data, desc, tb[NFTA_DATA_VERDICT]);
10589         } else {
10590                 err = -EINVAL;
10591         }
10592
10593         return err;
10594 }
10595 EXPORT_SYMBOL_GPL(nft_data_init);
10596
10597 /**
10598  *      nft_data_release - release a nft_data item
10599  *
10600  *      @data: struct nft_data to release
10601  *      @type: type of data
10602  *
10603  *      Release a nft_data item. NFT_DATA_VALUE types can be silently discarded,
10604  *      all others need to be released by calling this function.
10605  */
10606 void nft_data_release(const struct nft_data *data, enum nft_data_types type)
10607 {
10608         if (type < NFT_DATA_VERDICT)
10609                 return;
10610         switch (type) {
10611         case NFT_DATA_VERDICT:
10612                 return nft_verdict_uninit(data);
10613         default:
10614                 WARN_ON(1);
10615         }
10616 }
10617 EXPORT_SYMBOL_GPL(nft_data_release);
10618
10619 int nft_data_dump(struct sk_buff *skb, int attr, const struct nft_data *data,
10620                   enum nft_data_types type, unsigned int len)
10621 {
10622         struct nlattr *nest;
10623         int err;
10624
10625         nest = nla_nest_start_noflag(skb, attr);
10626         if (nest == NULL)
10627                 return -1;
10628
10629         switch (type) {
10630         case NFT_DATA_VALUE:
10631                 err = nft_value_dump(skb, data, len);
10632                 break;
10633         case NFT_DATA_VERDICT:
10634                 err = nft_verdict_dump(skb, NFTA_DATA_VERDICT, &data->verdict);
10635                 break;
10636         default:
10637                 err = -EINVAL;
10638                 WARN_ON(1);
10639         }
10640
10641         nla_nest_end(skb, nest);
10642         return err;
10643 }
10644 EXPORT_SYMBOL_GPL(nft_data_dump);
10645
10646 int __nft_release_basechain(struct nft_ctx *ctx)
10647 {
10648         struct nft_rule *rule, *nr;
10649
10650         if (WARN_ON(!nft_is_base_chain(ctx->chain)))
10651                 return 0;
10652
10653         nf_tables_unregister_hook(ctx->net, ctx->chain->table, ctx->chain);
10654         list_for_each_entry_safe(rule, nr, &ctx->chain->rules, list) {
10655                 list_del(&rule->list);
10656                 ctx->chain->use--;
10657                 nf_tables_rule_release(ctx, rule);
10658         }
10659         nft_chain_del(ctx->chain);
10660         ctx->table->use--;
10661         nf_tables_chain_destroy(ctx);
10662
10663         return 0;
10664 }
10665 EXPORT_SYMBOL_GPL(__nft_release_basechain);
10666
10667 static void __nft_release_hook(struct net *net, struct nft_table *table)
10668 {
10669         struct nft_flowtable *flowtable;
10670         struct nft_chain *chain;
10671
10672         list_for_each_entry(chain, &table->chains, list)
10673                 __nf_tables_unregister_hook(net, table, chain, true);
10674         list_for_each_entry(flowtable, &table->flowtables, list)
10675                 __nft_unregister_flowtable_net_hooks(net, &flowtable->hook_list,
10676                                                      true);
10677 }
10678
10679 static void __nft_release_hooks(struct net *net)
10680 {
10681         struct nftables_pernet *nft_net = nft_pernet(net);
10682         struct nft_table *table;
10683
10684         list_for_each_entry(table, &nft_net->tables, list) {
10685                 if (nft_table_has_owner(table))
10686                         continue;
10687
10688                 __nft_release_hook(net, table);
10689         }
10690 }
10691
10692 static void __nft_release_table(struct net *net, struct nft_table *table)
10693 {
10694         struct nft_flowtable *flowtable, *nf;
10695         struct nft_chain *chain, *nc;
10696         struct nft_object *obj, *ne;
10697         struct nft_rule *rule, *nr;
10698         struct nft_set *set, *ns;
10699         struct nft_ctx ctx = {
10700                 .net    = net,
10701                 .family = NFPROTO_NETDEV,
10702         };
10703
10704         ctx.family = table->family;
10705         ctx.table = table;
10706         list_for_each_entry(chain, &table->chains, list) {
10707                 ctx.chain = chain;
10708                 list_for_each_entry_safe(rule, nr, &chain->rules, list) {
10709                         list_del(&rule->list);
10710                         chain->use--;
10711                         nf_tables_rule_release(&ctx, rule);
10712                 }
10713         }
10714         list_for_each_entry_safe(flowtable, nf, &table->flowtables, list) {
10715                 list_del(&flowtable->list);
10716                 table->use--;
10717                 nf_tables_flowtable_destroy(flowtable);
10718         }
10719         list_for_each_entry_safe(set, ns, &table->sets, list) {
10720                 list_del(&set->list);
10721                 table->use--;
10722                 if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
10723                         nft_map_deactivate(&ctx, set);
10724
10725                 nft_set_destroy(&ctx, set);
10726         }
10727         list_for_each_entry_safe(obj, ne, &table->objects, list) {
10728                 nft_obj_del(obj);
10729                 table->use--;
10730                 nft_obj_destroy(&ctx, obj);
10731         }
10732         list_for_each_entry_safe(chain, nc, &table->chains, list) {
10733                 ctx.chain = chain;
10734                 nft_chain_del(chain);
10735                 table->use--;
10736                 nf_tables_chain_destroy(&ctx);
10737         }
10738         nf_tables_table_destroy(&ctx);
10739 }
10740
10741 static void __nft_release_tables(struct net *net)
10742 {
10743         struct nftables_pernet *nft_net = nft_pernet(net);
10744         struct nft_table *table, *nt;
10745
10746         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
10747                 if (nft_table_has_owner(table))
10748                         continue;
10749
10750                 list_del(&table->list);
10751
10752                 __nft_release_table(net, table);
10753         }
10754 }
10755
10756 static int nft_rcv_nl_event(struct notifier_block *this, unsigned long event,
10757                             void *ptr)
10758 {
10759         struct nft_table *table, *to_delete[8];
10760         struct nftables_pernet *nft_net;
10761         struct netlink_notify *n = ptr;
10762         struct net *net = n->net;
10763         unsigned int deleted;
10764         bool restart = false;
10765
10766         if (event != NETLINK_URELEASE || n->protocol != NETLINK_NETFILTER)
10767                 return NOTIFY_DONE;
10768
10769         nft_net = nft_pernet(net);
10770         deleted = 0;
10771         mutex_lock(&nft_net->commit_mutex);
10772         if (!list_empty(&nf_tables_destroy_list))
10773                 rcu_barrier();
10774 again:
10775         list_for_each_entry(table, &nft_net->tables, list) {
10776                 if (nft_table_has_owner(table) &&
10777                     n->portid == table->nlpid) {
10778                         __nft_release_hook(net, table);
10779                         list_del_rcu(&table->list);
10780                         to_delete[deleted++] = table;
10781                         if (deleted >= ARRAY_SIZE(to_delete))
10782                                 break;
10783                 }
10784         }
10785         if (deleted) {
10786                 restart = deleted >= ARRAY_SIZE(to_delete);
10787                 synchronize_rcu();
10788                 while (deleted)
10789                         __nft_release_table(net, to_delete[--deleted]);
10790
10791                 if (restart)
10792                         goto again;
10793         }
10794         mutex_unlock(&nft_net->commit_mutex);
10795
10796         return NOTIFY_DONE;
10797 }
10798
10799 static struct notifier_block nft_nl_notifier = {
10800         .notifier_call  = nft_rcv_nl_event,
10801 };
10802
10803 static int __net_init nf_tables_init_net(struct net *net)
10804 {
10805         struct nftables_pernet *nft_net = nft_pernet(net);
10806
10807         INIT_LIST_HEAD(&nft_net->tables);
10808         INIT_LIST_HEAD(&nft_net->commit_list);
10809         INIT_LIST_HEAD(&nft_net->binding_list);
10810         INIT_LIST_HEAD(&nft_net->module_list);
10811         INIT_LIST_HEAD(&nft_net->notify_list);
10812         mutex_init(&nft_net->commit_mutex);
10813         nft_net->base_seq = 1;
10814
10815         return 0;
10816 }
10817
10818 static void __net_exit nf_tables_pre_exit_net(struct net *net)
10819 {
10820         struct nftables_pernet *nft_net = nft_pernet(net);
10821
10822         mutex_lock(&nft_net->commit_mutex);
10823         __nft_release_hooks(net);
10824         mutex_unlock(&nft_net->commit_mutex);
10825 }
10826
10827 static void __net_exit nf_tables_exit_net(struct net *net)
10828 {
10829         struct nftables_pernet *nft_net = nft_pernet(net);
10830
10831         mutex_lock(&nft_net->commit_mutex);
10832         if (!list_empty(&nft_net->commit_list) ||
10833             !list_empty(&nft_net->module_list))
10834                 __nf_tables_abort(net, NFNL_ABORT_NONE);
10835         __nft_release_tables(net);
10836         mutex_unlock(&nft_net->commit_mutex);
10837         WARN_ON_ONCE(!list_empty(&nft_net->tables));
10838         WARN_ON_ONCE(!list_empty(&nft_net->module_list));
10839         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
10840 }
10841
10842 static struct pernet_operations nf_tables_net_ops = {
10843         .init           = nf_tables_init_net,
10844         .pre_exit       = nf_tables_pre_exit_net,
10845         .exit           = nf_tables_exit_net,
10846         .id             = &nf_tables_net_id,
10847         .size           = sizeof(struct nftables_pernet),
10848 };
10849
10850 static int __init nf_tables_module_init(void)
10851 {
10852         int err;
10853
10854         err = register_pernet_subsys(&nf_tables_net_ops);
10855         if (err < 0)
10856                 return err;
10857
10858         err = nft_chain_filter_init();
10859         if (err < 0)
10860                 goto err_chain_filter;
10861
10862         err = nf_tables_core_module_init();
10863         if (err < 0)
10864                 goto err_core_module;
10865
10866         err = register_netdevice_notifier(&nf_tables_flowtable_notifier);
10867         if (err < 0)
10868                 goto err_netdev_notifier;
10869
10870         err = rhltable_init(&nft_objname_ht, &nft_objname_ht_params);
10871         if (err < 0)
10872                 goto err_rht_objname;
10873
10874         err = nft_offload_init();
10875         if (err < 0)
10876                 goto err_offload;
10877
10878         err = netlink_register_notifier(&nft_nl_notifier);
10879         if (err < 0)
10880                 goto err_netlink_notifier;
10881
10882         /* must be last */
10883         err = nfnetlink_subsys_register(&nf_tables_subsys);
10884         if (err < 0)
10885                 goto err_nfnl_subsys;
10886
10887         nft_chain_route_init();
10888
10889         return err;
10890
10891 err_nfnl_subsys:
10892         netlink_unregister_notifier(&nft_nl_notifier);
10893 err_netlink_notifier:
10894         nft_offload_exit();
10895 err_offload:
10896         rhltable_destroy(&nft_objname_ht);
10897 err_rht_objname:
10898         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
10899 err_netdev_notifier:
10900         nf_tables_core_module_exit();
10901 err_core_module:
10902         nft_chain_filter_fini();
10903 err_chain_filter:
10904         unregister_pernet_subsys(&nf_tables_net_ops);
10905         return err;
10906 }
10907
10908 static void __exit nf_tables_module_exit(void)
10909 {
10910         nfnetlink_subsys_unregister(&nf_tables_subsys);
10911         netlink_unregister_notifier(&nft_nl_notifier);
10912         nft_offload_exit();
10913         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
10914         nft_chain_filter_fini();
10915         nft_chain_route_fini();
10916         unregister_pernet_subsys(&nf_tables_net_ops);
10917         cancel_work_sync(&trans_destroy_work);
10918         rcu_barrier();
10919         rhltable_destroy(&nft_objname_ht);
10920         nf_tables_core_module_exit();
10921 }
10922
10923 module_init(nf_tables_module_init);
10924 module_exit(nf_tables_module_exit);
10925
10926 MODULE_LICENSE("GPL");
10927 MODULE_AUTHOR("Patrick McHardy <kaber@trash.net>");
10928 MODULE_ALIAS_NFNL_SUBSYS(NFNL_SUBSYS_NFTABLES);