net/netfilter/nf_tables_api.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2007-2009 Patrick McHardy <kaber@trash.net>
   4  *
   5  * Development of this code funded by Astaro AG (http://www.astaro.com/)
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/init.h>
  10 #include <linux/list.h>
  11 #include <linux/skbuff.h>
  12 #include <linux/netlink.h>
  13 #include <linux/vmalloc.h>
  14 #include <linux/rhashtable.h>
  15 #include <linux/audit.h>
  16 #include <linux/netfilter.h>
  17 #include <linux/netfilter/nfnetlink.h>
  18 #include <linux/netfilter/nf_tables.h>
  19 #include <net/netfilter/nf_flow_table.h>
  20 #include <net/netfilter/nf_tables_core.h>
  21 #include <net/netfilter/nf_tables.h>
  22 #include <net/netfilter/nf_tables_offload.h>
  23 #include <net/net_namespace.h>
  24 #include <net/sock.h>
  25
  26 #define NFT_MODULE_AUTOLOAD_LIMIT (MODULE_NAME_LEN - sizeof("nft-expr-255-"))
  27
  28 unsigned int nf_tables_net_id __read_mostly;
  29
  30 static LIST_HEAD(nf_tables_expressions);
  31 static LIST_HEAD(nf_tables_objects);
  32 static LIST_HEAD(nf_tables_flowtables);
  33 static LIST_HEAD(nf_tables_destroy_list);
  34 static DEFINE_SPINLOCK(nf_tables_destroy_list_lock);
  35
  36 enum {
  37         NFT_VALIDATE_SKIP       = 0,
  38         NFT_VALIDATE_NEED,
  39         NFT_VALIDATE_DO,
  40 };
  41
  42 static struct rhltable nft_objname_ht;
  43
  44 static u32 nft_chain_hash(const void *data, u32 len, u32 seed);
  45 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed);
  46 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *, const void *);
  47
  48 static u32 nft_objname_hash(const void *data, u32 len, u32 seed);
  49 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed);
  50 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *, const void *);
  51
  52 static const struct rhashtable_params nft_chain_ht_params = {
  53         .head_offset            = offsetof(struct nft_chain, rhlhead),
  54         .key_offset             = offsetof(struct nft_chain, name),
  55         .hashfn                 = nft_chain_hash,
  56         .obj_hashfn             = nft_chain_hash_obj,
  57         .obj_cmpfn              = nft_chain_hash_cmp,
  58         .automatic_shrinking    = true,
  59 };
  60
  61 static const struct rhashtable_params nft_objname_ht_params = {
  62         .head_offset            = offsetof(struct nft_object, rhlhead),
  63         .key_offset             = offsetof(struct nft_object, key),
  64         .hashfn                 = nft_objname_hash,
  65         .obj_hashfn             = nft_objname_hash_obj,
  66         .obj_cmpfn              = nft_objname_hash_cmp,
  67         .automatic_shrinking    = true,
  68 };
  69
  70 struct nft_audit_data {
  71         struct nft_table *table;
  72         int entries;
  73         int op;
  74         struct list_head list;
  75 };
  76
  77 static const u8 nft2audit_op[NFT_MSG_MAX] = { // enum nf_tables_msg_types
  78         [NFT_MSG_NEWTABLE]      = AUDIT_NFT_OP_TABLE_REGISTER,
  79         [NFT_MSG_GETTABLE]      = AUDIT_NFT_OP_INVALID,
  80         [NFT_MSG_DELTABLE]      = AUDIT_NFT_OP_TABLE_UNREGISTER,
  81         [NFT_MSG_NEWCHAIN]      = AUDIT_NFT_OP_CHAIN_REGISTER,
  82         [NFT_MSG_GETCHAIN]      = AUDIT_NFT_OP_INVALID,
  83         [NFT_MSG_DELCHAIN]      = AUDIT_NFT_OP_CHAIN_UNREGISTER,
  84         [NFT_MSG_NEWRULE]       = AUDIT_NFT_OP_RULE_REGISTER,
  85         [NFT_MSG_GETRULE]       = AUDIT_NFT_OP_INVALID,
  86         [NFT_MSG_DELRULE]       = AUDIT_NFT_OP_RULE_UNREGISTER,
  87         [NFT_MSG_NEWSET]        = AUDIT_NFT_OP_SET_REGISTER,
  88         [NFT_MSG_GETSET]        = AUDIT_NFT_OP_INVALID,
  89         [NFT_MSG_DELSET]        = AUDIT_NFT_OP_SET_UNREGISTER,
  90         [NFT_MSG_NEWSETELEM]    = AUDIT_NFT_OP_SETELEM_REGISTER,
  91         [NFT_MSG_GETSETELEM]    = AUDIT_NFT_OP_INVALID,
  92         [NFT_MSG_DELSETELEM]    = AUDIT_NFT_OP_SETELEM_UNREGISTER,
  93         [NFT_MSG_NEWGEN]        = AUDIT_NFT_OP_GEN_REGISTER,
  94         [NFT_MSG_GETGEN]        = AUDIT_NFT_OP_INVALID,
  95         [NFT_MSG_TRACE]         = AUDIT_NFT_OP_INVALID,
  96         [NFT_MSG_NEWOBJ]        = AUDIT_NFT_OP_OBJ_REGISTER,
  97         [NFT_MSG_GETOBJ]        = AUDIT_NFT_OP_INVALID,
  98         [NFT_MSG_DELOBJ]        = AUDIT_NFT_OP_OBJ_UNREGISTER,
  99         [NFT_MSG_GETOBJ_RESET]  = AUDIT_NFT_OP_OBJ_RESET,
 100         [NFT_MSG_NEWFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_REGISTER,
 101         [NFT_MSG_GETFLOWTABLE]  = AUDIT_NFT_OP_INVALID,
 102         [NFT_MSG_DELFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_UNREGISTER,
 103 };
 104
 105 static void nft_validate_state_update(struct nft_table *table, u8 new_validate_state)
 106 {
 107         switch (table->validate_state) {
 108         case NFT_VALIDATE_SKIP:
 109                 WARN_ON_ONCE(new_validate_state == NFT_VALIDATE_DO);
 110                 break;
 111         case NFT_VALIDATE_NEED:
 112                 break;
 113         case NFT_VALIDATE_DO:
 114                 if (new_validate_state == NFT_VALIDATE_NEED)
 115                         return;
 116         }
 117
 118         table->validate_state = new_validate_state;
 119 }
 120 static void nf_tables_trans_destroy_work(struct work_struct *w);
 121 static DECLARE_WORK(trans_destroy_work, nf_tables_trans_destroy_work);
 122
 123 static void nft_ctx_init(struct nft_ctx *ctx,
 124                          struct net *net,
 125                          const struct sk_buff *skb,
 126                          const struct nlmsghdr *nlh,
 127                          u8 family,
 128                          struct nft_table *table,
 129                          struct nft_chain *chain,
 130                          const struct nlattr * const *nla)
 131 {
 132         ctx->net        = net;
 133         ctx->family     = family;
 134         ctx->level      = 0;
 135         ctx->table      = table;
 136         ctx->chain      = chain;
 137         ctx->nla        = nla;
 138         ctx->portid     = NETLINK_CB(skb).portid;
 139         ctx->report     = nlmsg_report(nlh);
 140         ctx->flags      = nlh->nlmsg_flags;
 141         ctx->seq        = nlh->nlmsg_seq;
 142 }
 143
 144 static struct nft_trans *nft_trans_alloc_gfp(const struct nft_ctx *ctx,
 145                                              int msg_type, u32 size, gfp_t gfp)
 146 {
 147         struct nft_trans *trans;
 148
 149         trans = kzalloc(sizeof(struct nft_trans) + size, gfp);
 150         if (trans == NULL)
 151                 return NULL;
 152
 153         INIT_LIST_HEAD(&trans->list);
 154         trans->msg_type = msg_type;
 155         trans->ctx      = *ctx;
 156
 157         return trans;
 158 }
 159
 160 static struct nft_trans *nft_trans_alloc(const struct nft_ctx *ctx,
 161                                          int msg_type, u32 size)
 162 {
 163         return nft_trans_alloc_gfp(ctx, msg_type, size, GFP_KERNEL);
 164 }
 165
 166 static void nft_trans_destroy(struct nft_trans *trans)
 167 {
 168         list_del(&trans->list);
 169         kfree(trans);
 170 }
 171
 172 static void __nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set,
 173                                  bool bind)
 174 {
 175         struct nftables_pernet *nft_net;
 176         struct net *net = ctx->net;
 177         struct nft_trans *trans;
 178
 179         if (!nft_set_is_anonymous(set))
 180                 return;
 181
 182         nft_net = nft_pernet(net);
 183         list_for_each_entry_reverse(trans, &nft_net->commit_list, list) {
 184                 switch (trans->msg_type) {
 185                 case NFT_MSG_NEWSET:
 186                         if (nft_trans_set(trans) == set)
 187                                 nft_trans_set_bound(trans) = bind;
 188                         break;
 189                 case NFT_MSG_NEWSETELEM:
 190                         if (nft_trans_elem_set(trans) == set)
 191                                 nft_trans_elem_set_bound(trans) = bind;
 192                         break;
 193                 }
 194         }
 195 }
 196
 197 static void nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set)
 198 {
 199         return __nft_set_trans_bind(ctx, set, true);
 200 }
 201
 202 static void nft_set_trans_unbind(const struct nft_ctx *ctx, struct nft_set *set)
 203 {
 204         return __nft_set_trans_bind(ctx, set, false);
 205 }
 206
 207 static void __nft_chain_trans_bind(const struct nft_ctx *ctx,
 208                                    struct nft_chain *chain, bool bind)
 209 {
 210         struct nftables_pernet *nft_net;
 211         struct net *net = ctx->net;
 212         struct nft_trans *trans;
 213
 214         if (!nft_chain_binding(chain))
 215                 return;
 216
 217         nft_net = nft_pernet(net);
 218         list_for_each_entry_reverse(trans, &nft_net->commit_list, list) {
 219                 switch (trans->msg_type) {
 220                 case NFT_MSG_NEWCHAIN:
 221                         if (nft_trans_chain(trans) == chain)
 222                                 nft_trans_chain_bound(trans) = bind;
 223                         break;
 224                 case NFT_MSG_NEWRULE:
 225                         if (trans->ctx.chain == chain)
 226                                 nft_trans_rule_bound(trans) = bind;
 227                         break;
 228                 }
 229         }
 230 }
 231
 232 static void nft_chain_trans_bind(const struct nft_ctx *ctx,
 233                                  struct nft_chain *chain)
 234 {
 235         __nft_chain_trans_bind(ctx, chain, true);
 236 }
 237
 238 int nf_tables_bind_chain(const struct nft_ctx *ctx, struct nft_chain *chain)
 239 {
 240         if (!nft_chain_binding(chain))
 241                 return 0;
 242
 243         if (nft_chain_binding(ctx->chain))
 244                 return -EOPNOTSUPP;
 245
 246         if (chain->bound)
 247                 return -EBUSY;
 248
 249         chain->bound = true;
 250         chain->use++;
 251         nft_chain_trans_bind(ctx, chain);
 252
 253         return 0;
 254 }
 255
 256 void nf_tables_unbind_chain(const struct nft_ctx *ctx, struct nft_chain *chain)
 257 {
 258         __nft_chain_trans_bind(ctx, chain, false);
 259 }
 260
 261 static int nft_netdev_register_hooks(struct net *net,
 262                                      struct list_head *hook_list)
 263 {
 264         struct nft_hook *hook;
 265         int err, j;
 266
 267         j = 0;
 268         list_for_each_entry(hook, hook_list, list) {
 269                 err = nf_register_net_hook(net, &hook->ops);
 270                 if (err < 0)
 271                         goto err_register;
 272
 273                 j++;
 274         }
 275         return 0;
 276
 277 err_register:
 278         list_for_each_entry(hook, hook_list, list) {
 279                 if (j-- <= 0)
 280                         break;
 281
 282                 nf_unregister_net_hook(net, &hook->ops);
 283         }
 284         return err;
 285 }
 286
 287 static void nft_netdev_unregister_hooks(struct net *net,
 288                                         struct list_head *hook_list,
 289                                         bool release_netdev)
 290 {
 291         struct nft_hook *hook, *next;
 292
 293         list_for_each_entry_safe(hook, next, hook_list, list) {
 294                 nf_unregister_net_hook(net, &hook->ops);
 295                 if (release_netdev) {
 296                         list_del(&hook->list);
 297                         kfree_rcu(hook, rcu);
 298                 }
 299         }
 300 }
 301
 302 static int nf_tables_register_hook(struct net *net,
 303                                    const struct nft_table *table,
 304                                    struct nft_chain *chain)
 305 {
 306         struct nft_base_chain *basechain;
 307         const struct nf_hook_ops *ops;
 308
 309         if (table->flags & NFT_TABLE_F_DORMANT ||
 310             !nft_is_base_chain(chain))
 311                 return 0;
 312
 313         basechain = nft_base_chain(chain);
 314         ops = &basechain->ops;
 315
 316         if (basechain->type->ops_register)
 317                 return basechain->type->ops_register(net, ops);
 318
 319         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 320                 return nft_netdev_register_hooks(net, &basechain->hook_list);
 321
 322         return nf_register_net_hook(net, &basechain->ops);
 323 }
 324
 325 static void __nf_tables_unregister_hook(struct net *net,
 326                                         const struct nft_table *table,
 327                                         struct nft_chain *chain,
 328                                         bool release_netdev)
 329 {
 330         struct nft_base_chain *basechain;
 331         const struct nf_hook_ops *ops;
 332
 333         if (table->flags & NFT_TABLE_F_DORMANT ||
 334             !nft_is_base_chain(chain))
 335                 return;
 336         basechain = nft_base_chain(chain);
 337         ops = &basechain->ops;
 338
 339         if (basechain->type->ops_unregister)
 340                 return basechain->type->ops_unregister(net, ops);
 341
 342         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 343                 nft_netdev_unregister_hooks(net, &basechain->hook_list,
 344                                             release_netdev);
 345         else
 346                 nf_unregister_net_hook(net, &basechain->ops);
 347 }
 348
 349 static void nf_tables_unregister_hook(struct net *net,
 350                                       const struct nft_table *table,
 351                                       struct nft_chain *chain)
 352 {
 353         return __nf_tables_unregister_hook(net, table, chain, false);
 354 }
 355
 356 static void nft_trans_commit_list_add_tail(struct net *net, struct nft_trans *trans)
 357 {
 358         struct nftables_pernet *nft_net = nft_pernet(net);
 359
 360         list_add_tail(&trans->list, &nft_net->commit_list);
 361 }
 362
 363 static int nft_trans_table_add(struct nft_ctx *ctx, int msg_type)
 364 {
 365         struct nft_trans *trans;
 366
 367         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_table));
 368         if (trans == NULL)
 369                 return -ENOMEM;
 370
 371         if (msg_type == NFT_MSG_NEWTABLE)
 372                 nft_activate_next(ctx->net, ctx->table);
 373
 374         nft_trans_commit_list_add_tail(ctx->net, trans);
 375         return 0;
 376 }
 377
 378 static int nft_deltable(struct nft_ctx *ctx)
 379 {
 380         int err;
 381
 382         err = nft_trans_table_add(ctx, NFT_MSG_DELTABLE);
 383         if (err < 0)
 384                 return err;
 385
 386         nft_deactivate_next(ctx->net, ctx->table);
 387         return err;
 388 }
 389
 390 static struct nft_trans *nft_trans_chain_add(struct nft_ctx *ctx, int msg_type)
 391 {
 392         struct nft_trans *trans;
 393
 394         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_chain));
 395         if (trans == NULL)
 396                 return ERR_PTR(-ENOMEM);
 397
 398         if (msg_type == NFT_MSG_NEWCHAIN) {
 399                 nft_activate_next(ctx->net, ctx->chain);
 400
 401                 if (ctx->nla[NFTA_CHAIN_ID]) {
 402                         nft_trans_chain_id(trans) =
 403                                 ntohl(nla_get_be32(ctx->nla[NFTA_CHAIN_ID]));
 404                 }
 405         }
 406         nft_trans_chain(trans) = ctx->chain;
 407         nft_trans_commit_list_add_tail(ctx->net, trans);
 408
 409         return trans;
 410 }
 411
 412 static int nft_delchain(struct nft_ctx *ctx)
 413 {
 414         struct nft_trans *trans;
 415
 416         trans = nft_trans_chain_add(ctx, NFT_MSG_DELCHAIN);
 417         if (IS_ERR(trans))
 418                 return PTR_ERR(trans);
 419
 420         ctx->table->use--;
 421         nft_deactivate_next(ctx->net, ctx->chain);
 422
 423         return 0;
 424 }
 425
 426 void nft_rule_expr_activate(const struct nft_ctx *ctx, struct nft_rule *rule)
 427 {
 428         struct nft_expr *expr;
 429
 430         expr = nft_expr_first(rule);
 431         while (nft_expr_more(rule, expr)) {
 432                 if (expr->ops->activate)
 433                         expr->ops->activate(ctx, expr);
 434
 435                 expr = nft_expr_next(expr);
 436         }
 437 }
 438
 439 void nft_rule_expr_deactivate(const struct nft_ctx *ctx, struct nft_rule *rule,
 440                               enum nft_trans_phase phase)
 441 {
 442         struct nft_expr *expr;
 443
 444         expr = nft_expr_first(rule);
 445         while (nft_expr_more(rule, expr)) {
 446                 if (expr->ops->deactivate)
 447                         expr->ops->deactivate(ctx, expr, phase);
 448
 449                 expr = nft_expr_next(expr);
 450         }
 451 }
 452
 453 static int
 454 nf_tables_delrule_deactivate(struct nft_ctx *ctx, struct nft_rule *rule)
 455 {
 456         /* You cannot delete the same rule twice */
 457         if (nft_is_active_next(ctx->net, rule)) {
 458                 nft_deactivate_next(ctx->net, rule);
 459                 ctx->chain->use--;
 460                 return 0;
 461         }
 462         return -ENOENT;
 463 }
 464
 465 static struct nft_trans *nft_trans_rule_add(struct nft_ctx *ctx, int msg_type,
 466                                             struct nft_rule *rule)
 467 {
 468         struct nft_trans *trans;
 469
 470         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_rule));
 471         if (trans == NULL)
 472                 return NULL;
 473
 474         if (msg_type == NFT_MSG_NEWRULE && ctx->nla[NFTA_RULE_ID] != NULL) {
 475                 nft_trans_rule_id(trans) =
 476                         ntohl(nla_get_be32(ctx->nla[NFTA_RULE_ID]));
 477         }
 478         nft_trans_rule(trans) = rule;
 479         nft_trans_commit_list_add_tail(ctx->net, trans);
 480
 481         return trans;
 482 }
 483
 484 static int nft_delrule(struct nft_ctx *ctx, struct nft_rule *rule)
 485 {
 486         struct nft_flow_rule *flow;
 487         struct nft_trans *trans;
 488         int err;
 489
 490         trans = nft_trans_rule_add(ctx, NFT_MSG_DELRULE, rule);
 491         if (trans == NULL)
 492                 return -ENOMEM;
 493
 494         if (ctx->chain->flags & NFT_CHAIN_HW_OFFLOAD) {
 495                 flow = nft_flow_rule_create(ctx->net, rule);
 496                 if (IS_ERR(flow)) {
 497                         nft_trans_destroy(trans);
 498                         return PTR_ERR(flow);
 499                 }
 500
 501                 nft_trans_flow_rule(trans) = flow;
 502         }
 503
 504         err = nf_tables_delrule_deactivate(ctx, rule);
 505         if (err < 0) {
 506                 nft_trans_destroy(trans);
 507                 return err;
 508         }
 509         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_PREPARE);
 510
 511         return 0;
 512 }
 513
 514 static int nft_delrule_by_chain(struct nft_ctx *ctx)
 515 {
 516         struct nft_rule *rule;
 517         int err;
 518
 519         list_for_each_entry(rule, &ctx->chain->rules, list) {
 520                 if (!nft_is_active_next(ctx->net, rule))
 521                         continue;
 522
 523                 err = nft_delrule(ctx, rule);
 524                 if (err < 0)
 525                         return err;
 526         }
 527         return 0;
 528 }
 529
 530 static int __nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
 531                                struct nft_set *set,
 532                                const struct nft_set_desc *desc)
 533 {
 534         struct nft_trans *trans;
 535
 536         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_set));
 537         if (trans == NULL)
 538                 return -ENOMEM;
 539
 540         if (msg_type == NFT_MSG_NEWSET && ctx->nla[NFTA_SET_ID] && !desc) {
 541                 nft_trans_set_id(trans) =
 542                         ntohl(nla_get_be32(ctx->nla[NFTA_SET_ID]));
 543                 nft_activate_next(ctx->net, set);
 544         }
 545         nft_trans_set(trans) = set;
 546         if (desc) {
 547                 nft_trans_set_update(trans) = true;
 548                 nft_trans_set_gc_int(trans) = desc->gc_int;
 549                 nft_trans_set_timeout(trans) = desc->timeout;
 550         }
 551         nft_trans_commit_list_add_tail(ctx->net, trans);
 552
 553         return 0;
 554 }
 555
 556 static int nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
 557                              struct nft_set *set)
 558 {
 559         return __nft_trans_set_add(ctx, msg_type, set, NULL);
 560 }
 561
 562 static int nft_delset(const struct nft_ctx *ctx, struct nft_set *set)
 563 {
 564         int err;
 565
 566         err = nft_trans_set_add(ctx, NFT_MSG_DELSET, set);
 567         if (err < 0)
 568                 return err;
 569
 570         nft_deactivate_next(ctx->net, set);
 571         ctx->table->use--;
 572
 573         return err;
 574 }
 575
 576 static int nft_trans_obj_add(struct nft_ctx *ctx, int msg_type,
 577                              struct nft_object *obj)
 578 {
 579         struct nft_trans *trans;
 580
 581         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_obj));
 582         if (trans == NULL)
 583                 return -ENOMEM;
 584
 585         if (msg_type == NFT_MSG_NEWOBJ)
 586                 nft_activate_next(ctx->net, obj);
 587
 588         nft_trans_obj(trans) = obj;
 589         nft_trans_commit_list_add_tail(ctx->net, trans);
 590
 591         return 0;
 592 }
 593
 594 static int nft_delobj(struct nft_ctx *ctx, struct nft_object *obj)
 595 {
 596         int err;
 597
 598         err = nft_trans_obj_add(ctx, NFT_MSG_DELOBJ, obj);
 599         if (err < 0)
 600                 return err;
 601
 602         nft_deactivate_next(ctx->net, obj);
 603         ctx->table->use--;
 604
 605         return err;
 606 }
 607
 608 static int nft_trans_flowtable_add(struct nft_ctx *ctx, int msg_type,
 609                                    struct nft_flowtable *flowtable)
 610 {
 611         struct nft_trans *trans;
 612
 613         trans = nft_trans_alloc(ctx, msg_type,
 614                                 sizeof(struct nft_trans_flowtable));
 615         if (trans == NULL)
 616                 return -ENOMEM;
 617
 618         if (msg_type == NFT_MSG_NEWFLOWTABLE)
 619                 nft_activate_next(ctx->net, flowtable);
 620
 621         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
 622         nft_trans_flowtable(trans) = flowtable;
 623         nft_trans_commit_list_add_tail(ctx->net, trans);
 624
 625         return 0;
 626 }
 627
 628 static int nft_delflowtable(struct nft_ctx *ctx,
 629                             struct nft_flowtable *flowtable)
 630 {
 631         int err;
 632
 633         err = nft_trans_flowtable_add(ctx, NFT_MSG_DELFLOWTABLE, flowtable);
 634         if (err < 0)
 635                 return err;
 636
 637         nft_deactivate_next(ctx->net, flowtable);
 638         ctx->table->use--;
 639
 640         return err;
 641 }
 642
 643 static void __nft_reg_track_clobber(struct nft_regs_track *track, u8 dreg)
 644 {
 645         int i;
 646
 647         for (i = track->regs[dreg].num_reg; i > 0; i--)
 648                 __nft_reg_track_cancel(track, dreg - i);
 649 }
 650
 651 static void __nft_reg_track_update(struct nft_regs_track *track,
 652                                    const struct nft_expr *expr,
 653                                    u8 dreg, u8 num_reg)
 654 {
 655         track->regs[dreg].selector = expr;
 656         track->regs[dreg].bitwise = NULL;
 657         track->regs[dreg].num_reg = num_reg;
 658 }
 659
 660 void nft_reg_track_update(struct nft_regs_track *track,
 661                           const struct nft_expr *expr, u8 dreg, u8 len)
 662 {
 663         unsigned int regcount;
 664         int i;
 665
 666         __nft_reg_track_clobber(track, dreg);
 667
 668         regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
 669         for (i = 0; i < regcount; i++, dreg++)
 670                 __nft_reg_track_update(track, expr, dreg, i);
 671 }
 672 EXPORT_SYMBOL_GPL(nft_reg_track_update);
 673
 674 void nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg, u8 len)
 675 {
 676         unsigned int regcount;
 677         int i;
 678
 679         __nft_reg_track_clobber(track, dreg);
 680
 681         regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
 682         for (i = 0; i < regcount; i++, dreg++)
 683                 __nft_reg_track_cancel(track, dreg);
 684 }
 685 EXPORT_SYMBOL_GPL(nft_reg_track_cancel);
 686
 687 void __nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg)
 688 {
 689         track->regs[dreg].selector = NULL;
 690         track->regs[dreg].bitwise = NULL;
 691         track->regs[dreg].num_reg = 0;
 692 }
 693 EXPORT_SYMBOL_GPL(__nft_reg_track_cancel);
 694
 695 /*
 696  * Tables
 697  */
 698
 699 static struct nft_table *nft_table_lookup(const struct net *net,
 700                                           const struct nlattr *nla,
 701                                           u8 family, u8 genmask, u32 nlpid)
 702 {
 703         struct nftables_pernet *nft_net;
 704         struct nft_table *table;
 705
 706         if (nla == NULL)
 707                 return ERR_PTR(-EINVAL);
 708
 709         nft_net = nft_pernet(net);
 710         list_for_each_entry_rcu(table, &nft_net->tables, list,
 711                                 lockdep_is_held(&nft_net->commit_mutex)) {
 712                 if (!nla_strcmp(nla, table->name) &&
 713                     table->family == family &&
 714                     nft_active_genmask(table, genmask)) {
 715                         if (nft_table_has_owner(table) &&
 716                             nlpid && table->nlpid != nlpid)
 717                                 return ERR_PTR(-EPERM);
 718
 719                         return table;
 720                 }
 721         }
 722
 723         return ERR_PTR(-ENOENT);
 724 }
 725
 726 static struct nft_table *nft_table_lookup_byhandle(const struct net *net,
 727                                                    const struct nlattr *nla,
 728                                                    u8 genmask, u32 nlpid)
 729 {
 730         struct nftables_pernet *nft_net;
 731         struct nft_table *table;
 732
 733         nft_net = nft_pernet(net);
 734         list_for_each_entry(table, &nft_net->tables, list) {
 735                 if (be64_to_cpu(nla_get_be64(nla)) == table->handle &&
 736                     nft_active_genmask(table, genmask)) {
 737                         if (nft_table_has_owner(table) &&
 738                             nlpid && table->nlpid != nlpid)
 739                                 return ERR_PTR(-EPERM);
 740
 741                         return table;
 742                 }
 743         }
 744
 745         return ERR_PTR(-ENOENT);
 746 }
 747
 748 static inline u64 nf_tables_alloc_handle(struct nft_table *table)
 749 {
 750         return ++table->hgenerator;
 751 }
 752
 753 static const struct nft_chain_type *chain_type[NFPROTO_NUMPROTO][NFT_CHAIN_T_MAX];
 754
 755 static const struct nft_chain_type *
 756 __nft_chain_type_get(u8 family, enum nft_chain_types type)
 757 {
 758         if (family >= NFPROTO_NUMPROTO ||
 759             type >= NFT_CHAIN_T_MAX)
 760                 return NULL;
 761
 762         return chain_type[family][type];
 763 }
 764
 765 static const struct nft_chain_type *
 766 __nf_tables_chain_type_lookup(const struct nlattr *nla, u8 family)
 767 {
 768         const struct nft_chain_type *type;
 769         int i;
 770
 771         for (i = 0; i < NFT_CHAIN_T_MAX; i++) {
 772                 type = __nft_chain_type_get(family, i);
 773                 if (!type)
 774                         continue;
 775                 if (!nla_strcmp(nla, type->name))
 776                         return type;
 777         }
 778         return NULL;
 779 }
 780
 781 struct nft_module_request {
 782         struct list_head        list;
 783         char                    module[MODULE_NAME_LEN];
 784         bool                    done;
 785 };
 786
 787 #ifdef CONFIG_MODULES
 788 __printf(2, 3) int nft_request_module(struct net *net, const char *fmt,
 789                                       ...)
 790 {
 791         char module_name[MODULE_NAME_LEN];
 792         struct nftables_pernet *nft_net;
 793         struct nft_module_request *req;
 794         va_list args;
 795         int ret;
 796
 797         va_start(args, fmt);
 798         ret = vsnprintf(module_name, MODULE_NAME_LEN, fmt, args);
 799         va_end(args);
 800         if (ret >= MODULE_NAME_LEN)
 801                 return 0;
 802
 803         nft_net = nft_pernet(net);
 804         list_for_each_entry(req, &nft_net->module_list, list) {
 805                 if (!strcmp(req->module, module_name)) {
 806                         if (req->done)
 807                                 return 0;
 808
 809                         /* A request to load this module already exists. */
 810                         return -EAGAIN;
 811                 }
 812         }
 813
 814         req = kmalloc(sizeof(*req), GFP_KERNEL);
 815         if (!req)
 816                 return -ENOMEM;
 817
 818         req->done = false;
 819         strscpy(req->module, module_name, MODULE_NAME_LEN);
 820         list_add_tail(&req->list, &nft_net->module_list);
 821
 822         return -EAGAIN;
 823 }
 824 EXPORT_SYMBOL_GPL(nft_request_module);
 825 #endif
 826
 827 static void lockdep_nfnl_nft_mutex_not_held(void)
 828 {
 829 #ifdef CONFIG_PROVE_LOCKING
 830         if (debug_locks)
 831                 WARN_ON_ONCE(lockdep_nfnl_is_held(NFNL_SUBSYS_NFTABLES));
 832 #endif
 833 }
 834
 835 static const struct nft_chain_type *
 836 nf_tables_chain_type_lookup(struct net *net, const struct nlattr *nla,
 837                             u8 family, bool autoload)
 838 {
 839         const struct nft_chain_type *type;
 840
 841         type = __nf_tables_chain_type_lookup(nla, family);
 842         if (type != NULL)
 843                 return type;
 844
 845         lockdep_nfnl_nft_mutex_not_held();
 846 #ifdef CONFIG_MODULES
 847         if (autoload) {
 848                 if (nft_request_module(net, "nft-chain-%u-%.*s", family,
 849                                        nla_len(nla),
 850                                        (const char *)nla_data(nla)) == -EAGAIN)
 851                         return ERR_PTR(-EAGAIN);
 852         }
 853 #endif
 854         return ERR_PTR(-ENOENT);
 855 }
 856
 857 static __be16 nft_base_seq(const struct net *net)
 858 {
 859         struct nftables_pernet *nft_net = nft_pernet(net);
 860
 861         return htons(nft_net->base_seq & 0xffff);
 862 }
 863
 864 static const struct nla_policy nft_table_policy[NFTA_TABLE_MAX + 1] = {
 865         [NFTA_TABLE_NAME]       = { .type = NLA_STRING,
 866                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
 867         [NFTA_TABLE_FLAGS]      = { .type = NLA_U32 },
 868         [NFTA_TABLE_HANDLE]     = { .type = NLA_U64 },
 869         [NFTA_TABLE_USERDATA]   = { .type = NLA_BINARY,
 870                                     .len = NFT_USERDATA_MAXLEN }
 871 };
 872
 873 static int nf_tables_fill_table_info(struct sk_buff *skb, struct net *net,
 874                                      u32 portid, u32 seq, int event, u32 flags,
 875                                      int family, const struct nft_table *table)
 876 {
 877         struct nlmsghdr *nlh;
 878
 879         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
 880         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
 881                            NFNETLINK_V0, nft_base_seq(net));
 882         if (!nlh)
 883                 goto nla_put_failure;
 884
 885         if (nla_put_string(skb, NFTA_TABLE_NAME, table->name) ||
 886             nla_put_be32(skb, NFTA_TABLE_USE, htonl(table->use)) ||
 887             nla_put_be64(skb, NFTA_TABLE_HANDLE, cpu_to_be64(table->handle),
 888                          NFTA_TABLE_PAD))
 889                 goto nla_put_failure;
 890
 891         if (event == NFT_MSG_DELTABLE) {
 892                 nlmsg_end(skb, nlh);
 893                 return 0;
 894         }
 895
 896         if (nla_put_be32(skb, NFTA_TABLE_FLAGS,
 897                          htonl(table->flags & NFT_TABLE_F_MASK)))
 898                 goto nla_put_failure;
 899
 900         if (nft_table_has_owner(table) &&
 901             nla_put_be32(skb, NFTA_TABLE_OWNER, htonl(table->nlpid)))
 902                 goto nla_put_failure;
 903
 904         if (table->udata) {
 905                 if (nla_put(skb, NFTA_TABLE_USERDATA, table->udlen, table->udata))
 906                         goto nla_put_failure;
 907         }
 908
 909         nlmsg_end(skb, nlh);
 910         return 0;
 911
 912 nla_put_failure:
 913         nlmsg_trim(skb, nlh);
 914         return -1;
 915 }
 916
 917 struct nftnl_skb_parms {
 918         bool report;
 919 };
 920 #define NFT_CB(skb)     (*(struct nftnl_skb_parms*)&((skb)->cb))
 921
 922 static void nft_notify_enqueue(struct sk_buff *skb, bool report,
 923                                struct list_head *notify_list)
 924 {
 925         NFT_CB(skb).report = report;
 926         list_add_tail(&skb->list, notify_list);
 927 }
 928
 929 static void nf_tables_table_notify(const struct nft_ctx *ctx, int event)
 930 {
 931         struct nftables_pernet *nft_net;
 932         struct sk_buff *skb;
 933         u16 flags = 0;
 934         int err;
 935
 936         if (!ctx->report &&
 937             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
 938                 return;
 939
 940         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
 941         if (skb == NULL)
 942                 goto err;
 943
 944         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
 945                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
 946
 947         err = nf_tables_fill_table_info(skb, ctx->net, ctx->portid, ctx->seq,
 948                                         event, flags, ctx->family, ctx->table);
 949         if (err < 0) {
 950                 kfree_skb(skb);
 951                 goto err;
 952         }
 953
 954         nft_net = nft_pernet(ctx->net);
 955         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
 956         return;
 957 err:
 958         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
 959 }
 960
 961 static int nf_tables_dump_tables(struct sk_buff *skb,
 962                                  struct netlink_callback *cb)
 963 {
 964         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
 965         struct nftables_pernet *nft_net;
 966         const struct nft_table *table;
 967         unsigned int idx = 0, s_idx = cb->args[0];
 968         struct net *net = sock_net(skb->sk);
 969         int family = nfmsg->nfgen_family;
 970
 971         rcu_read_lock();
 972         nft_net = nft_pernet(net);
 973         cb->seq = READ_ONCE(nft_net->base_seq);
 974
 975         list_for_each_entry_rcu(table, &nft_net->tables, list) {
 976                 if (family != NFPROTO_UNSPEC && family != table->family)
 977                         continue;
 978
 979                 if (idx < s_idx)
 980                         goto cont;
 981                 if (idx > s_idx)
 982                         memset(&cb->args[1], 0,
 983                                sizeof(cb->args) - sizeof(cb->args[0]));
 984                 if (!nft_is_active(net, table))
 985                         continue;
 986                 if (nf_tables_fill_table_info(skb, net,
 987                                               NETLINK_CB(cb->skb).portid,
 988                                               cb->nlh->nlmsg_seq,
 989                                               NFT_MSG_NEWTABLE, NLM_F_MULTI,
 990                                               table->family, table) < 0)
 991                         goto done;
 992
 993                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
 994 cont:
 995                 idx++;
 996         }
 997 done:
 998         rcu_read_unlock();
 999         cb->args[0] = idx;
1000         return skb->len;
1001 }
1002
1003 static int nft_netlink_dump_start_rcu(struct sock *nlsk, struct sk_buff *skb,
1004                                       const struct nlmsghdr *nlh,
1005                                       struct netlink_dump_control *c)
1006 {
1007         int err;
1008
1009         if (!try_module_get(THIS_MODULE))
1010                 return -EINVAL;
1011
1012         rcu_read_unlock();
1013         err = netlink_dump_start(nlsk, skb, nlh, c);
1014         rcu_read_lock();
1015         module_put(THIS_MODULE);
1016
1017         return err;
1018 }
1019
1020 /* called with rcu_read_lock held */
1021 static int nf_tables_gettable(struct sk_buff *skb, const struct nfnl_info *info,
1022                               const struct nlattr * const nla[])
1023 {
1024         struct netlink_ext_ack *extack = info->extack;
1025         u8 genmask = nft_genmask_cur(info->net);
1026         u8 family = info->nfmsg->nfgen_family;
1027         const struct nft_table *table;
1028         struct net *net = info->net;
1029         struct sk_buff *skb2;
1030         int err;
1031
1032         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
1033                 struct netlink_dump_control c = {
1034                         .dump = nf_tables_dump_tables,
1035                         .module = THIS_MODULE,
1036                 };
1037
1038                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
1039         }
1040
1041         table = nft_table_lookup(net, nla[NFTA_TABLE_NAME], family, genmask, 0);
1042         if (IS_ERR(table)) {
1043                 NL_SET_BAD_ATTR(extack, nla[NFTA_TABLE_NAME]);
1044                 return PTR_ERR(table);
1045         }
1046
1047         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
1048         if (!skb2)
1049                 return -ENOMEM;
1050
1051         err = nf_tables_fill_table_info(skb2, net, NETLINK_CB(skb).portid,
1052                                         info->nlh->nlmsg_seq, NFT_MSG_NEWTABLE,
1053                                         0, family, table);
1054         if (err < 0)
1055                 goto err_fill_table_info;
1056
1057         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
1058
1059 err_fill_table_info:
1060         kfree_skb(skb2);
1061         return err;
1062 }
1063
1064 static void nft_table_disable(struct net *net, struct nft_table *table, u32 cnt)
1065 {
1066         struct nft_chain *chain;
1067         u32 i = 0;
1068
1069         list_for_each_entry(chain, &table->chains, list) {
1070                 if (!nft_is_active_next(net, chain))
1071                         continue;
1072                 if (!nft_is_base_chain(chain))
1073                         continue;
1074
1075                 if (cnt && i++ == cnt)
1076                         break;
1077
1078                 nf_tables_unregister_hook(net, table, chain);
1079         }
1080 }
1081
1082 static int nf_tables_table_enable(struct net *net, struct nft_table *table)
1083 {
1084         struct nft_chain *chain;
1085         int err, i = 0;
1086
1087         list_for_each_entry(chain, &table->chains, list) {
1088                 if (!nft_is_active_next(net, chain))
1089                         continue;
1090                 if (!nft_is_base_chain(chain))
1091                         continue;
1092
1093                 err = nf_tables_register_hook(net, table, chain);
1094                 if (err < 0)
1095                         goto err_register_hooks;
1096
1097                 i++;
1098         }
1099         return 0;
1100
1101 err_register_hooks:
1102         if (i)
1103                 nft_table_disable(net, table, i);
1104         return err;
1105 }
1106
1107 static void nf_tables_table_disable(struct net *net, struct nft_table *table)
1108 {
1109         table->flags &= ~NFT_TABLE_F_DORMANT;
1110         nft_table_disable(net, table, 0);
1111         table->flags |= NFT_TABLE_F_DORMANT;
1112 }
1113
1114 #define __NFT_TABLE_F_INTERNAL          (NFT_TABLE_F_MASK + 1)
1115 #define __NFT_TABLE_F_WAS_DORMANT       (__NFT_TABLE_F_INTERNAL << 0)
1116 #define __NFT_TABLE_F_WAS_AWAKEN        (__NFT_TABLE_F_INTERNAL << 1)
1117 #define __NFT_TABLE_F_UPDATE            (__NFT_TABLE_F_WAS_DORMANT | \
1118                                          __NFT_TABLE_F_WAS_AWAKEN)
1119
1120 static int nf_tables_updtable(struct nft_ctx *ctx)
1121 {
1122         struct nft_trans *trans;
1123         u32 flags;
1124         int ret;
1125
1126         if (!ctx->nla[NFTA_TABLE_FLAGS])
1127                 return 0;
1128
1129         flags = ntohl(nla_get_be32(ctx->nla[NFTA_TABLE_FLAGS]));
1130         if (flags & ~NFT_TABLE_F_MASK)
1131                 return -EOPNOTSUPP;
1132
1133         if (flags == ctx->table->flags)
1134                 return 0;
1135
1136         if ((nft_table_has_owner(ctx->table) &&
1137              !(flags & NFT_TABLE_F_OWNER)) ||
1138             (!nft_table_has_owner(ctx->table) &&
1139              flags & NFT_TABLE_F_OWNER))
1140                 return -EOPNOTSUPP;
1141
1142         trans = nft_trans_alloc(ctx, NFT_MSG_NEWTABLE,
1143                                 sizeof(struct nft_trans_table));
1144         if (trans == NULL)
1145                 return -ENOMEM;
1146
1147         if ((flags & NFT_TABLE_F_DORMANT) &&
1148             !(ctx->table->flags & NFT_TABLE_F_DORMANT)) {
1149                 ctx->table->flags |= NFT_TABLE_F_DORMANT;
1150                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE))
1151                         ctx->table->flags |= __NFT_TABLE_F_WAS_AWAKEN;
1152         } else if (!(flags & NFT_TABLE_F_DORMANT) &&
1153                    ctx->table->flags & NFT_TABLE_F_DORMANT) {
1154                 ctx->table->flags &= ~NFT_TABLE_F_DORMANT;
1155                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE)) {
1156                         ret = nf_tables_table_enable(ctx->net, ctx->table);
1157                         if (ret < 0)
1158                                 goto err_register_hooks;
1159
1160                         ctx->table->flags |= __NFT_TABLE_F_WAS_DORMANT;
1161                 }
1162         }
1163
1164         nft_trans_table_update(trans) = true;
1165         nft_trans_commit_list_add_tail(ctx->net, trans);
1166
1167         return 0;
1168
1169 err_register_hooks:
1170         nft_trans_destroy(trans);
1171         return ret;
1172 }
1173
1174 static u32 nft_chain_hash(const void *data, u32 len, u32 seed)
1175 {
1176         const char *name = data;
1177
1178         return jhash(name, strlen(name), seed);
1179 }
1180
1181 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed)
1182 {
1183         const struct nft_chain *chain = data;
1184
1185         return nft_chain_hash(chain->name, 0, seed);
1186 }
1187
1188 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *arg,
1189                               const void *ptr)
1190 {
1191         const struct nft_chain *chain = ptr;
1192         const char *name = arg->key;
1193
1194         return strcmp(chain->name, name);
1195 }
1196
1197 static u32 nft_objname_hash(const void *data, u32 len, u32 seed)
1198 {
1199         const struct nft_object_hash_key *k = data;
1200
1201         seed ^= hash_ptr(k->table, 32);
1202
1203         return jhash(k->name, strlen(k->name), seed);
1204 }
1205
1206 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed)
1207 {
1208         const struct nft_object *obj = data;
1209
1210         return nft_objname_hash(&obj->key, 0, seed);
1211 }
1212
1213 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *arg,
1214                                 const void *ptr)
1215 {
1216         const struct nft_object_hash_key *k = arg->key;
1217         const struct nft_object *obj = ptr;
1218
1219         if (obj->key.table != k->table)
1220                 return -1;
1221
1222         return strcmp(obj->key.name, k->name);
1223 }
1224
1225 static bool nft_supported_family(u8 family)
1226 {
1227         return false
1228 #ifdef CONFIG_NF_TABLES_INET
1229                 || family == NFPROTO_INET
1230 #endif
1231 #ifdef CONFIG_NF_TABLES_IPV4
1232                 || family == NFPROTO_IPV4
1233 #endif
1234 #ifdef CONFIG_NF_TABLES_ARP
1235                 || family == NFPROTO_ARP
1236 #endif
1237 #ifdef CONFIG_NF_TABLES_NETDEV
1238                 || family == NFPROTO_NETDEV
1239 #endif
1240 #if IS_ENABLED(CONFIG_NF_TABLES_BRIDGE)
1241                 || family == NFPROTO_BRIDGE
1242 #endif
1243 #ifdef CONFIG_NF_TABLES_IPV6
1244                 || family == NFPROTO_IPV6
1245 #endif
1246                 ;
1247 }
1248
1249 static int nf_tables_newtable(struct sk_buff *skb, const struct nfnl_info *info,
1250                               const struct nlattr * const nla[])
1251 {
1252         struct nftables_pernet *nft_net = nft_pernet(info->net);
1253         struct netlink_ext_ack *extack = info->extack;
1254         u8 genmask = nft_genmask_next(info->net);
1255         u8 family = info->nfmsg->nfgen_family;
1256         struct net *net = info->net;
1257         const struct nlattr *attr;
1258         struct nft_table *table;
1259         struct nft_ctx ctx;
1260         u32 flags = 0;
1261         int err;
1262
1263         if (!nft_supported_family(family))
1264                 return -EOPNOTSUPP;
1265
1266         lockdep_assert_held(&nft_net->commit_mutex);
1267         attr = nla[NFTA_TABLE_NAME];
1268         table = nft_table_lookup(net, attr, family, genmask,
1269                                  NETLINK_CB(skb).portid);
1270         if (IS_ERR(table)) {
1271                 if (PTR_ERR(table) != -ENOENT)
1272                         return PTR_ERR(table);
1273         } else {
1274                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
1275                         NL_SET_BAD_ATTR(extack, attr);
1276                         return -EEXIST;
1277                 }
1278                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
1279                         return -EOPNOTSUPP;
1280
1281                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1282
1283                 return nf_tables_updtable(&ctx);
1284         }
1285
1286         if (nla[NFTA_TABLE_FLAGS]) {
1287                 flags = ntohl(nla_get_be32(nla[NFTA_TABLE_FLAGS]));
1288                 if (flags & ~NFT_TABLE_F_MASK)
1289                         return -EOPNOTSUPP;
1290         }
1291
1292         err = -ENOMEM;
1293         table = kzalloc(sizeof(*table), GFP_KERNEL_ACCOUNT);
1294         if (table == NULL)
1295                 goto err_kzalloc;
1296
1297         table->validate_state = NFT_VALIDATE_SKIP;
1298         table->name = nla_strdup(attr, GFP_KERNEL_ACCOUNT);
1299         if (table->name == NULL)
1300                 goto err_strdup;
1301
1302         if (nla[NFTA_TABLE_USERDATA]) {
1303                 table->udata = nla_memdup(nla[NFTA_TABLE_USERDATA], GFP_KERNEL_ACCOUNT);
1304                 if (table->udata == NULL)
1305                         goto err_table_udata;
1306
1307                 table->udlen = nla_len(nla[NFTA_TABLE_USERDATA]);
1308         }
1309
1310         err = rhltable_init(&table->chains_ht, &nft_chain_ht_params);
1311         if (err)
1312                 goto err_chain_ht;
1313
1314         INIT_LIST_HEAD(&table->chains);
1315         INIT_LIST_HEAD(&table->sets);
1316         INIT_LIST_HEAD(&table->objects);
1317         INIT_LIST_HEAD(&table->flowtables);
1318         table->family = family;
1319         table->flags = flags;
1320         table->handle = ++nft_net->table_handle;
1321         if (table->flags & NFT_TABLE_F_OWNER)
1322                 table->nlpid = NETLINK_CB(skb).portid;
1323
1324         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1325         err = nft_trans_table_add(&ctx, NFT_MSG_NEWTABLE);
1326         if (err < 0)
1327                 goto err_trans;
1328
1329         list_add_tail_rcu(&table->list, &nft_net->tables);
1330         return 0;
1331 err_trans:
1332         rhltable_destroy(&table->chains_ht);
1333 err_chain_ht:
1334         kfree(table->udata);
1335 err_table_udata:
1336         kfree(table->name);
1337 err_strdup:
1338         kfree(table);
1339 err_kzalloc:
1340         return err;
1341 }
1342
1343 static int nft_flush_table(struct nft_ctx *ctx)
1344 {
1345         struct nft_flowtable *flowtable, *nft;
1346         struct nft_chain *chain, *nc;
1347         struct nft_object *obj, *ne;
1348         struct nft_set *set, *ns;
1349         int err;
1350
1351         list_for_each_entry(chain, &ctx->table->chains, list) {
1352                 if (!nft_is_active_next(ctx->net, chain))
1353                         continue;
1354
1355                 if (nft_chain_is_bound(chain))
1356                         continue;
1357
1358                 ctx->chain = chain;
1359
1360                 err = nft_delrule_by_chain(ctx);
1361                 if (err < 0)
1362                         goto out;
1363         }
1364
1365         list_for_each_entry_safe(set, ns, &ctx->table->sets, list) {
1366                 if (!nft_is_active_next(ctx->net, set))
1367                         continue;
1368
1369                 if (nft_set_is_anonymous(set) &&
1370                     !list_empty(&set->bindings))
1371                         continue;
1372
1373                 err = nft_delset(ctx, set);
1374                 if (err < 0)
1375                         goto out;
1376         }
1377
1378         list_for_each_entry_safe(flowtable, nft, &ctx->table->flowtables, list) {
1379                 if (!nft_is_active_next(ctx->net, flowtable))
1380                         continue;
1381
1382                 err = nft_delflowtable(ctx, flowtable);
1383                 if (err < 0)
1384                         goto out;
1385         }
1386
1387         list_for_each_entry_safe(obj, ne, &ctx->table->objects, list) {
1388                 if (!nft_is_active_next(ctx->net, obj))
1389                         continue;
1390
1391                 err = nft_delobj(ctx, obj);
1392                 if (err < 0)
1393                         goto out;
1394         }
1395
1396         list_for_each_entry_safe(chain, nc, &ctx->table->chains, list) {
1397                 if (!nft_is_active_next(ctx->net, chain))
1398                         continue;
1399
1400                 if (nft_chain_is_bound(chain))
1401                         continue;
1402
1403                 ctx->chain = chain;
1404
1405                 err = nft_delchain(ctx);
1406                 if (err < 0)
1407                         goto out;
1408         }
1409
1410         err = nft_deltable(ctx);
1411 out:
1412         return err;
1413 }
1414
1415 static int nft_flush(struct nft_ctx *ctx, int family)
1416 {
1417         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
1418         const struct nlattr * const *nla = ctx->nla;
1419         struct nft_table *table, *nt;
1420         int err = 0;
1421
1422         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
1423                 if (family != AF_UNSPEC && table->family != family)
1424                         continue;
1425
1426                 ctx->family = table->family;
1427
1428                 if (!nft_is_active_next(ctx->net, table))
1429                         continue;
1430
1431                 if (nft_table_has_owner(table) && table->nlpid != ctx->portid)
1432                         continue;
1433
1434                 if (nla[NFTA_TABLE_NAME] &&
1435                     nla_strcmp(nla[NFTA_TABLE_NAME], table->name) != 0)
1436                         continue;
1437
1438                 ctx->table = table;
1439
1440                 err = nft_flush_table(ctx);
1441                 if (err < 0)
1442                         goto out;
1443         }
1444 out:
1445         return err;
1446 }
1447
1448 static int nf_tables_deltable(struct sk_buff *skb, const struct nfnl_info *info,
1449                               const struct nlattr * const nla[])
1450 {
1451         struct netlink_ext_ack *extack = info->extack;
1452         u8 genmask = nft_genmask_next(info->net);
1453         u8 family = info->nfmsg->nfgen_family;
1454         struct net *net = info->net;
1455         const struct nlattr *attr;
1456         struct nft_table *table;
1457         struct nft_ctx ctx;
1458
1459         nft_ctx_init(&ctx, net, skb, info->nlh, 0, NULL, NULL, nla);
1460         if (family == AF_UNSPEC ||
1461             (!nla[NFTA_TABLE_NAME] && !nla[NFTA_TABLE_HANDLE]))
1462                 return nft_flush(&ctx, family);
1463
1464         if (nla[NFTA_TABLE_HANDLE]) {
1465                 attr = nla[NFTA_TABLE_HANDLE];
1466                 table = nft_table_lookup_byhandle(net, attr, genmask,
1467                                                   NETLINK_CB(skb).portid);
1468         } else {
1469                 attr = nla[NFTA_TABLE_NAME];
1470                 table = nft_table_lookup(net, attr, family, genmask,
1471                                          NETLINK_CB(skb).portid);
1472         }
1473
1474         if (IS_ERR(table)) {
1475                 if (PTR_ERR(table) == -ENOENT &&
1476                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYTABLE)
1477                         return 0;
1478
1479                 NL_SET_BAD_ATTR(extack, attr);
1480                 return PTR_ERR(table);
1481         }
1482
1483         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
1484             table->use > 0)
1485                 return -EBUSY;
1486
1487         ctx.family = family;
1488         ctx.table = table;
1489
1490         return nft_flush_table(&ctx);
1491 }
1492
1493 static void nf_tables_table_destroy(struct nft_ctx *ctx)
1494 {
1495         if (WARN_ON(ctx->table->use > 0))
1496                 return;
1497
1498         rhltable_destroy(&ctx->table->chains_ht);
1499         kfree(ctx->table->name);
1500         kfree(ctx->table->udata);
1501         kfree(ctx->table);
1502 }
1503
1504 void nft_register_chain_type(const struct nft_chain_type *ctype)
1505 {
1506         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1507         if (WARN_ON(__nft_chain_type_get(ctype->family, ctype->type))) {
1508                 nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1509                 return;
1510         }
1511         chain_type[ctype->family][ctype->type] = ctype;
1512         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1513 }
1514 EXPORT_SYMBOL_GPL(nft_register_chain_type);
1515
1516 void nft_unregister_chain_type(const struct nft_chain_type *ctype)
1517 {
1518         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1519         chain_type[ctype->family][ctype->type] = NULL;
1520         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1521 }
1522 EXPORT_SYMBOL_GPL(nft_unregister_chain_type);
1523
1524 /*
1525  * Chains
1526  */
1527
1528 static struct nft_chain *
1529 nft_chain_lookup_byhandle(const struct nft_table *table, u64 handle, u8 genmask)
1530 {
1531         struct nft_chain *chain;
1532
1533         list_for_each_entry(chain, &table->chains, list) {
1534                 if (chain->handle == handle &&
1535                     nft_active_genmask(chain, genmask))
1536                         return chain;
1537         }
1538
1539         return ERR_PTR(-ENOENT);
1540 }
1541
1542 static bool lockdep_commit_lock_is_held(const struct net *net)
1543 {
1544 #ifdef CONFIG_PROVE_LOCKING
1545         struct nftables_pernet *nft_net = nft_pernet(net);
1546
1547         return lockdep_is_held(&nft_net->commit_mutex);
1548 #else
1549         return true;
1550 #endif
1551 }
1552
1553 static struct nft_chain *nft_chain_lookup(struct net *net,
1554                                           struct nft_table *table,
1555                                           const struct nlattr *nla, u8 genmask)
1556 {
1557         char search[NFT_CHAIN_MAXNAMELEN + 1];
1558         struct rhlist_head *tmp, *list;
1559         struct nft_chain *chain;
1560
1561         if (nla == NULL)
1562                 return ERR_PTR(-EINVAL);
1563
1564         nla_strscpy(search, nla, sizeof(search));
1565
1566         WARN_ON(!rcu_read_lock_held() &&
1567                 !lockdep_commit_lock_is_held(net));
1568
1569         chain = ERR_PTR(-ENOENT);
1570         rcu_read_lock();
1571         list = rhltable_lookup(&table->chains_ht, search, nft_chain_ht_params);
1572         if (!list)
1573                 goto out_unlock;
1574
1575         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
1576                 if (nft_active_genmask(chain, genmask))
1577                         goto out_unlock;
1578         }
1579         chain = ERR_PTR(-ENOENT);
1580 out_unlock:
1581         rcu_read_unlock();
1582         return chain;
1583 }
1584
1585 static const struct nla_policy nft_chain_policy[NFTA_CHAIN_MAX + 1] = {
1586         [NFTA_CHAIN_TABLE]      = { .type = NLA_STRING,
1587                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
1588         [NFTA_CHAIN_HANDLE]     = { .type = NLA_U64 },
1589         [NFTA_CHAIN_NAME]       = { .type = NLA_STRING,
1590                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
1591         [NFTA_CHAIN_HOOK]       = { .type = NLA_NESTED },
1592         [NFTA_CHAIN_POLICY]     = { .type = NLA_U32 },
1593         [NFTA_CHAIN_TYPE]       = { .type = NLA_STRING,
1594                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
1595         [NFTA_CHAIN_COUNTERS]   = { .type = NLA_NESTED },
1596         [NFTA_CHAIN_FLAGS]      = { .type = NLA_U32 },
1597         [NFTA_CHAIN_ID]         = { .type = NLA_U32 },
1598         [NFTA_CHAIN_USERDATA]   = { .type = NLA_BINARY,
1599                                     .len = NFT_USERDATA_MAXLEN },
1600 };
1601
1602 static const struct nla_policy nft_hook_policy[NFTA_HOOK_MAX + 1] = {
1603         [NFTA_HOOK_HOOKNUM]     = { .type = NLA_U32 },
1604         [NFTA_HOOK_PRIORITY]    = { .type = NLA_U32 },
1605         [NFTA_HOOK_DEV]         = { .type = NLA_STRING,
1606                                     .len = IFNAMSIZ - 1 },
1607 };
1608
1609 static int nft_dump_stats(struct sk_buff *skb, struct nft_stats __percpu *stats)
1610 {
1611         struct nft_stats *cpu_stats, total;
1612         struct nlattr *nest;
1613         unsigned int seq;
1614         u64 pkts, bytes;
1615         int cpu;
1616
1617         if (!stats)
1618                 return 0;
1619
1620         memset(&total, 0, sizeof(total));
1621         for_each_possible_cpu(cpu) {
1622                 cpu_stats = per_cpu_ptr(stats, cpu);
1623                 do {
1624                         seq = u64_stats_fetch_begin(&cpu_stats->syncp);
1625                         pkts = cpu_stats->pkts;
1626                         bytes = cpu_stats->bytes;
1627                 } while (u64_stats_fetch_retry(&cpu_stats->syncp, seq));
1628                 total.pkts += pkts;
1629                 total.bytes += bytes;
1630         }
1631         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_COUNTERS);
1632         if (nest == NULL)
1633                 goto nla_put_failure;
1634
1635         if (nla_put_be64(skb, NFTA_COUNTER_PACKETS, cpu_to_be64(total.pkts),
1636                          NFTA_COUNTER_PAD) ||
1637             nla_put_be64(skb, NFTA_COUNTER_BYTES, cpu_to_be64(total.bytes),
1638                          NFTA_COUNTER_PAD))
1639                 goto nla_put_failure;
1640
1641         nla_nest_end(skb, nest);
1642         return 0;
1643
1644 nla_put_failure:
1645         return -ENOSPC;
1646 }
1647
1648 static int nft_dump_basechain_hook(struct sk_buff *skb, int family,
1649                                    const struct nft_base_chain *basechain,
1650                                    const struct list_head *hook_list)
1651 {
1652         const struct nf_hook_ops *ops = &basechain->ops;
1653         struct nft_hook *hook, *first = NULL;
1654         struct nlattr *nest, *nest_devs;
1655         int n = 0;
1656
1657         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_HOOK);
1658         if (nest == NULL)
1659                 goto nla_put_failure;
1660         if (nla_put_be32(skb, NFTA_HOOK_HOOKNUM, htonl(ops->hooknum)))
1661                 goto nla_put_failure;
1662         if (nla_put_be32(skb, NFTA_HOOK_PRIORITY, htonl(ops->priority)))
1663                 goto nla_put_failure;
1664
1665         if (nft_base_chain_netdev(family, ops->hooknum)) {
1666                 nest_devs = nla_nest_start_noflag(skb, NFTA_HOOK_DEVS);
1667                 if (!nest_devs)
1668                         goto nla_put_failure;
1669
1670                 if (!hook_list)
1671                         hook_list = &basechain->hook_list;
1672
1673                 list_for_each_entry(hook, hook_list, list) {
1674                         if (!first)
1675                                 first = hook;
1676
1677                         if (nla_put_string(skb, NFTA_DEVICE_NAME,
1678                                            hook->ops.dev->name))
1679                                 goto nla_put_failure;
1680                         n++;
1681                 }
1682                 nla_nest_end(skb, nest_devs);
1683
1684                 if (n == 1 &&
1685                     nla_put_string(skb, NFTA_HOOK_DEV, first->ops.dev->name))
1686                         goto nla_put_failure;
1687         }
1688         nla_nest_end(skb, nest);
1689
1690         return 0;
1691 nla_put_failure:
1692         return -1;
1693 }
1694
1695 static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net,
1696                                      u32 portid, u32 seq, int event, u32 flags,
1697                                      int family, const struct nft_table *table,
1698                                      const struct nft_chain *chain,
1699                                      const struct list_head *hook_list)
1700 {
1701         struct nlmsghdr *nlh;
1702
1703         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
1704         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
1705                            NFNETLINK_V0, nft_base_seq(net));
1706         if (!nlh)
1707                 goto nla_put_failure;
1708
1709         if (nla_put_string(skb, NFTA_CHAIN_TABLE, table->name) ||
1710             nla_put_string(skb, NFTA_CHAIN_NAME, chain->name) ||
1711             nla_put_be64(skb, NFTA_CHAIN_HANDLE, cpu_to_be64(chain->handle),
1712                          NFTA_CHAIN_PAD))
1713                 goto nla_put_failure;
1714
1715         if (event == NFT_MSG_DELCHAIN && !hook_list) {
1716                 nlmsg_end(skb, nlh);
1717                 return 0;
1718         }
1719
1720         if (nft_is_base_chain(chain)) {
1721                 const struct nft_base_chain *basechain = nft_base_chain(chain);
1722                 struct nft_stats __percpu *stats;
1723
1724                 if (nft_dump_basechain_hook(skb, family, basechain, hook_list))
1725                         goto nla_put_failure;
1726
1727                 if (nla_put_be32(skb, NFTA_CHAIN_POLICY,
1728                                  htonl(basechain->policy)))
1729                         goto nla_put_failure;
1730
1731                 if (nla_put_string(skb, NFTA_CHAIN_TYPE, basechain->type->name))
1732                         goto nla_put_failure;
1733
1734                 stats = rcu_dereference_check(basechain->stats,
1735                                               lockdep_commit_lock_is_held(net));
1736                 if (nft_dump_stats(skb, stats))
1737                         goto nla_put_failure;
1738         }
1739
1740         if (chain->flags &&
1741             nla_put_be32(skb, NFTA_CHAIN_FLAGS, htonl(chain->flags)))
1742                 goto nla_put_failure;
1743
1744         if (nla_put_be32(skb, NFTA_CHAIN_USE, htonl(chain->use)))
1745                 goto nla_put_failure;
1746
1747         if (chain->udata &&
1748             nla_put(skb, NFTA_CHAIN_USERDATA, chain->udlen, chain->udata))
1749                 goto nla_put_failure;
1750
1751         nlmsg_end(skb, nlh);
1752         return 0;
1753
1754 nla_put_failure:
1755         nlmsg_trim(skb, nlh);
1756         return -1;
1757 }
1758
1759 static void nf_tables_chain_notify(const struct nft_ctx *ctx, int event,
1760                                    const struct list_head *hook_list)
1761 {
1762         struct nftables_pernet *nft_net;
1763         struct sk_buff *skb;
1764         u16 flags = 0;
1765         int err;
1766
1767         if (!ctx->report &&
1768             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
1769                 return;
1770
1771         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
1772         if (skb == NULL)
1773                 goto err;
1774
1775         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
1776                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
1777
1778         err = nf_tables_fill_chain_info(skb, ctx->net, ctx->portid, ctx->seq,
1779                                         event, flags, ctx->family, ctx->table,
1780                                         ctx->chain, hook_list);
1781         if (err < 0) {
1782                 kfree_skb(skb);
1783                 goto err;
1784         }
1785
1786         nft_net = nft_pernet(ctx->net);
1787         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
1788         return;
1789 err:
1790         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
1791 }
1792
1793 static int nf_tables_dump_chains(struct sk_buff *skb,
1794                                  struct netlink_callback *cb)
1795 {
1796         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
1797         unsigned int idx = 0, s_idx = cb->args[0];
1798         struct net *net = sock_net(skb->sk);
1799         int family = nfmsg->nfgen_family;
1800         struct nftables_pernet *nft_net;
1801         const struct nft_table *table;
1802         const struct nft_chain *chain;
1803
1804         rcu_read_lock();
1805         nft_net = nft_pernet(net);
1806         cb->seq = READ_ONCE(nft_net->base_seq);
1807
1808         list_for_each_entry_rcu(table, &nft_net->tables, list) {
1809                 if (family != NFPROTO_UNSPEC && family != table->family)
1810                         continue;
1811
1812                 list_for_each_entry_rcu(chain, &table->chains, list) {
1813                         if (idx < s_idx)
1814                                 goto cont;
1815                         if (idx > s_idx)
1816                                 memset(&cb->args[1], 0,
1817                                        sizeof(cb->args) - sizeof(cb->args[0]));
1818                         if (!nft_is_active(net, chain))
1819                                 continue;
1820                         if (nf_tables_fill_chain_info(skb, net,
1821                                                       NETLINK_CB(cb->skb).portid,
1822                                                       cb->nlh->nlmsg_seq,
1823                                                       NFT_MSG_NEWCHAIN,
1824                                                       NLM_F_MULTI,
1825                                                       table->family, table,
1826                                                       chain, NULL) < 0)
1827                                 goto done;
1828
1829                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
1830 cont:
1831                         idx++;
1832                 }
1833         }
1834 done:
1835         rcu_read_unlock();
1836         cb->args[0] = idx;
1837         return skb->len;
1838 }
1839
1840 /* called with rcu_read_lock held */
1841 static int nf_tables_getchain(struct sk_buff *skb, const struct nfnl_info *info,
1842                               const struct nlattr * const nla[])
1843 {
1844         struct netlink_ext_ack *extack = info->extack;
1845         u8 genmask = nft_genmask_cur(info->net);
1846         u8 family = info->nfmsg->nfgen_family;
1847         const struct nft_chain *chain;
1848         struct net *net = info->net;
1849         struct nft_table *table;
1850         struct sk_buff *skb2;
1851         int err;
1852
1853         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
1854                 struct netlink_dump_control c = {
1855                         .dump = nf_tables_dump_chains,
1856                         .module = THIS_MODULE,
1857                 };
1858
1859                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
1860         }
1861
1862         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask, 0);
1863         if (IS_ERR(table)) {
1864                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
1865                 return PTR_ERR(table);
1866         }
1867
1868         chain = nft_chain_lookup(net, table, nla[NFTA_CHAIN_NAME], genmask);
1869         if (IS_ERR(chain)) {
1870                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
1871                 return PTR_ERR(chain);
1872         }
1873
1874         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
1875         if (!skb2)
1876                 return -ENOMEM;
1877
1878         err = nf_tables_fill_chain_info(skb2, net, NETLINK_CB(skb).portid,
1879                                         info->nlh->nlmsg_seq, NFT_MSG_NEWCHAIN,
1880                                         0, family, table, chain, NULL);
1881         if (err < 0)
1882                 goto err_fill_chain_info;
1883
1884         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
1885
1886 err_fill_chain_info:
1887         kfree_skb(skb2);
1888         return err;
1889 }
1890
1891 static const struct nla_policy nft_counter_policy[NFTA_COUNTER_MAX + 1] = {
1892         [NFTA_COUNTER_PACKETS]  = { .type = NLA_U64 },
1893         [NFTA_COUNTER_BYTES]    = { .type = NLA_U64 },
1894 };
1895
1896 static struct nft_stats __percpu *nft_stats_alloc(const struct nlattr *attr)
1897 {
1898         struct nlattr *tb[NFTA_COUNTER_MAX+1];
1899         struct nft_stats __percpu *newstats;
1900         struct nft_stats *stats;
1901         int err;
1902
1903         err = nla_parse_nested_deprecated(tb, NFTA_COUNTER_MAX, attr,
1904                                           nft_counter_policy, NULL);
1905         if (err < 0)
1906                 return ERR_PTR(err);
1907
1908         if (!tb[NFTA_COUNTER_BYTES] || !tb[NFTA_COUNTER_PACKETS])
1909                 return ERR_PTR(-EINVAL);
1910
1911         newstats = netdev_alloc_pcpu_stats(struct nft_stats);
1912         if (newstats == NULL)
1913                 return ERR_PTR(-ENOMEM);
1914
1915         /* Restore old counters on this cpu, no problem. Per-cpu statistics
1916          * are not exposed to userspace.
1917          */
1918         preempt_disable();
1919         stats = this_cpu_ptr(newstats);
1920         stats->bytes = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_BYTES]));
1921         stats->pkts = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_PACKETS]));
1922         preempt_enable();
1923
1924         return newstats;
1925 }
1926
1927 static void nft_chain_stats_replace(struct nft_trans *trans)
1928 {
1929         struct nft_base_chain *chain = nft_base_chain(trans->ctx.chain);
1930
1931         if (!nft_trans_chain_stats(trans))
1932                 return;
1933
1934         nft_trans_chain_stats(trans) =
1935                 rcu_replace_pointer(chain->stats, nft_trans_chain_stats(trans),
1936                                     lockdep_commit_lock_is_held(trans->ctx.net));
1937
1938         if (!nft_trans_chain_stats(trans))
1939                 static_branch_inc(&nft_counters_enabled);
1940 }
1941
1942 static void nf_tables_chain_free_chain_rules(struct nft_chain *chain)
1943 {
1944         struct nft_rule_blob *g0 = rcu_dereference_raw(chain->blob_gen_0);
1945         struct nft_rule_blob *g1 = rcu_dereference_raw(chain->blob_gen_1);
1946
1947         if (g0 != g1)
1948                 kvfree(g1);
1949         kvfree(g0);
1950
1951         /* should be NULL either via abort or via successful commit */
1952         WARN_ON_ONCE(chain->blob_next);
1953         kvfree(chain->blob_next);
1954 }
1955
1956 void nf_tables_chain_destroy(struct nft_ctx *ctx)
1957 {
1958         struct nft_chain *chain = ctx->chain;
1959         struct nft_hook *hook, *next;
1960
1961         if (WARN_ON(chain->use > 0))
1962                 return;
1963
1964         /* no concurrent access possible anymore */
1965         nf_tables_chain_free_chain_rules(chain);
1966
1967         if (nft_is_base_chain(chain)) {
1968                 struct nft_base_chain *basechain = nft_base_chain(chain);
1969
1970                 if (nft_base_chain_netdev(ctx->family, basechain->ops.hooknum)) {
1971                         list_for_each_entry_safe(hook, next,
1972                                                  &basechain->hook_list, list) {
1973                                 list_del_rcu(&hook->list);
1974                                 kfree_rcu(hook, rcu);
1975                         }
1976                 }
1977                 module_put(basechain->type->owner);
1978                 if (rcu_access_pointer(basechain->stats)) {
1979                         static_branch_dec(&nft_counters_enabled);
1980                         free_percpu(rcu_dereference_raw(basechain->stats));
1981                 }
1982                 kfree(chain->name);
1983                 kfree(chain->udata);
1984                 kfree(basechain);
1985         } else {
1986                 kfree(chain->name);
1987                 kfree(chain->udata);
1988                 kfree(chain);
1989         }
1990 }
1991
1992 static struct nft_hook *nft_netdev_hook_alloc(struct net *net,
1993                                               const struct nlattr *attr)
1994 {
1995         struct net_device *dev;
1996         char ifname[IFNAMSIZ];
1997         struct nft_hook *hook;
1998         int err;
1999
2000         hook = kmalloc(sizeof(struct nft_hook), GFP_KERNEL_ACCOUNT);
2001         if (!hook) {
2002                 err = -ENOMEM;
2003                 goto err_hook_alloc;
2004         }
2005
2006         nla_strscpy(ifname, attr, IFNAMSIZ);
2007         /* nf_tables_netdev_event() is called under rtnl_mutex, this is
2008          * indirectly serializing all the other holders of the commit_mutex with
2009          * the rtnl_mutex.
2010          */
2011         dev = __dev_get_by_name(net, ifname);
2012         if (!dev) {
2013                 err = -ENOENT;
2014                 goto err_hook_dev;
2015         }
2016         hook->ops.dev = dev;
2017
2018         return hook;
2019
2020 err_hook_dev:
2021         kfree(hook);
2022 err_hook_alloc:
2023         return ERR_PTR(err);
2024 }
2025
2026 static struct nft_hook *nft_hook_list_find(struct list_head *hook_list,
2027                                            const struct nft_hook *this)
2028 {
2029         struct nft_hook *hook;
2030
2031         list_for_each_entry(hook, hook_list, list) {
2032                 if (this->ops.dev == hook->ops.dev)
2033                         return hook;
2034         }
2035
2036         return NULL;
2037 }
2038
2039 static int nf_tables_parse_netdev_hooks(struct net *net,
2040                                         const struct nlattr *attr,
2041                                         struct list_head *hook_list,
2042                                         struct netlink_ext_ack *extack)
2043 {
2044         struct nft_hook *hook, *next;
2045         const struct nlattr *tmp;
2046         int rem, n = 0, err;
2047
2048         nla_for_each_nested(tmp, attr, rem) {
2049                 if (nla_type(tmp) != NFTA_DEVICE_NAME) {
2050                         err = -EINVAL;
2051                         goto err_hook;
2052                 }
2053
2054                 hook = nft_netdev_hook_alloc(net, tmp);
2055                 if (IS_ERR(hook)) {
2056                         NL_SET_BAD_ATTR(extack, tmp);
2057                         err = PTR_ERR(hook);
2058                         goto err_hook;
2059                 }
2060                 if (nft_hook_list_find(hook_list, hook)) {
2061                         NL_SET_BAD_ATTR(extack, tmp);
2062                         kfree(hook);
2063                         err = -EEXIST;
2064                         goto err_hook;
2065                 }
2066                 list_add_tail(&hook->list, hook_list);
2067                 n++;
2068
2069                 if (n == NFT_NETDEVICE_MAX) {
2070                         err = -EFBIG;
2071                         goto err_hook;
2072                 }
2073         }
2074
2075         return 0;
2076
2077 err_hook:
2078         list_for_each_entry_safe(hook, next, hook_list, list) {
2079                 list_del(&hook->list);
2080                 kfree(hook);
2081         }
2082         return err;
2083 }
2084
2085 struct nft_chain_hook {
2086         u32                             num;
2087         s32                             priority;
2088         const struct nft_chain_type     *type;
2089         struct list_head                list;
2090 };
2091
2092 static int nft_chain_parse_netdev(struct net *net, struct nlattr *tb[],
2093                                   struct list_head *hook_list,
2094                                   struct netlink_ext_ack *extack, u32 flags)
2095 {
2096         struct nft_hook *hook;
2097         int err;
2098
2099         if (tb[NFTA_HOOK_DEV]) {
2100                 hook = nft_netdev_hook_alloc(net, tb[NFTA_HOOK_DEV]);
2101                 if (IS_ERR(hook)) {
2102                         NL_SET_BAD_ATTR(extack, tb[NFTA_HOOK_DEV]);
2103                         return PTR_ERR(hook);
2104                 }
2105
2106                 list_add_tail(&hook->list, hook_list);
2107         } else if (tb[NFTA_HOOK_DEVS]) {
2108                 err = nf_tables_parse_netdev_hooks(net, tb[NFTA_HOOK_DEVS],
2109                                                    hook_list, extack);
2110                 if (err < 0)
2111                         return err;
2112
2113         }
2114
2115         if (flags & NFT_CHAIN_HW_OFFLOAD &&
2116             list_empty(hook_list))
2117                 return -EINVAL;
2118
2119         return 0;
2120 }
2121
2122 static int nft_chain_parse_hook(struct net *net,
2123                                 struct nft_base_chain *basechain,
2124                                 const struct nlattr * const nla[],
2125                                 struct nft_chain_hook *hook, u8 family,
2126                                 u32 flags, struct netlink_ext_ack *extack)
2127 {
2128         struct nftables_pernet *nft_net = nft_pernet(net);
2129         struct nlattr *ha[NFTA_HOOK_MAX + 1];
2130         const struct nft_chain_type *type;
2131         int err;
2132
2133         lockdep_assert_held(&nft_net->commit_mutex);
2134         lockdep_nfnl_nft_mutex_not_held();
2135
2136         err = nla_parse_nested_deprecated(ha, NFTA_HOOK_MAX,
2137                                           nla[NFTA_CHAIN_HOOK],
2138                                           nft_hook_policy, NULL);
2139         if (err < 0)
2140                 return err;
2141
2142         if (!basechain) {
2143                 if (!ha[NFTA_HOOK_HOOKNUM] ||
2144                     !ha[NFTA_HOOK_PRIORITY]) {
2145                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2146                         return -ENOENT;
2147                 }
2148
2149                 hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
2150                 hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
2151
2152                 type = __nft_chain_type_get(family, NFT_CHAIN_T_DEFAULT);
2153                 if (!type)
2154                         return -EOPNOTSUPP;
2155
2156                 if (nla[NFTA_CHAIN_TYPE]) {
2157                         type = nf_tables_chain_type_lookup(net, nla[NFTA_CHAIN_TYPE],
2158                                                            family, true);
2159                         if (IS_ERR(type)) {
2160                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2161                                 return PTR_ERR(type);
2162                         }
2163                 }
2164                 if (hook->num >= NFT_MAX_HOOKS || !(type->hook_mask & (1 << hook->num)))
2165                         return -EOPNOTSUPP;
2166
2167                 if (type->type == NFT_CHAIN_T_NAT &&
2168                     hook->priority <= NF_IP_PRI_CONNTRACK)
2169                         return -EOPNOTSUPP;
2170         } else {
2171                 if (ha[NFTA_HOOK_HOOKNUM]) {
2172                         hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
2173                         if (hook->num != basechain->ops.hooknum)
2174                                 return -EOPNOTSUPP;
2175                 }
2176                 if (ha[NFTA_HOOK_PRIORITY]) {
2177                         hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
2178                         if (hook->priority != basechain->ops.priority)
2179                                 return -EOPNOTSUPP;
2180                 }
2181
2182                 type = basechain->type;
2183         }
2184
2185         if (!try_module_get(type->owner)) {
2186                 if (nla[NFTA_CHAIN_TYPE])
2187                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2188                 return -ENOENT;
2189         }
2190
2191         hook->type = type;
2192
2193         INIT_LIST_HEAD(&hook->list);
2194         if (nft_base_chain_netdev(family, hook->num)) {
2195                 err = nft_chain_parse_netdev(net, ha, &hook->list, extack, flags);
2196                 if (err < 0) {
2197                         module_put(type->owner);
2198                         return err;
2199                 }
2200         } else if (ha[NFTA_HOOK_DEV] || ha[NFTA_HOOK_DEVS]) {
2201                 module_put(type->owner);
2202                 return -EOPNOTSUPP;
2203         }
2204
2205         return 0;
2206 }
2207
2208 static void nft_chain_release_hook(struct nft_chain_hook *hook)
2209 {
2210         struct nft_hook *h, *next;
2211
2212         list_for_each_entry_safe(h, next, &hook->list, list) {
2213                 list_del(&h->list);
2214                 kfree(h);
2215         }
2216         module_put(hook->type->owner);
2217 }
2218
2219 static void nft_last_rule(const struct nft_chain *chain, const void *ptr)
2220 {
2221         struct nft_rule_dp_last *lrule;
2222
2223         BUILD_BUG_ON(offsetof(struct nft_rule_dp_last, end) != 0);
2224
2225         lrule = (struct nft_rule_dp_last *)ptr;
2226         lrule->end.is_last = 1;
2227         lrule->chain = chain;
2228         /* blob size does not include the trailer rule */
2229 }
2230
2231 static struct nft_rule_blob *nf_tables_chain_alloc_rules(const struct nft_chain *chain,
2232                                                          unsigned int size)
2233 {
2234         struct nft_rule_blob *blob;
2235
2236         if (size > INT_MAX)
2237                 return NULL;
2238
2239         size += sizeof(struct nft_rule_blob) + sizeof(struct nft_rule_dp_last);
2240
2241         blob = kvmalloc(size, GFP_KERNEL_ACCOUNT);
2242         if (!blob)
2243                 return NULL;
2244
2245         blob->size = 0;
2246         nft_last_rule(chain, blob->data);
2247
2248         return blob;
2249 }
2250
2251 static void nft_basechain_hook_init(struct nf_hook_ops *ops, u8 family,
2252                                     const struct nft_chain_hook *hook,
2253                                     struct nft_chain *chain)
2254 {
2255         ops->pf                 = family;
2256         ops->hooknum            = hook->num;
2257         ops->priority           = hook->priority;
2258         ops->priv               = chain;
2259         ops->hook               = hook->type->hooks[ops->hooknum];
2260         ops->hook_ops_type      = NF_HOOK_OP_NF_TABLES;
2261 }
2262
2263 static int nft_basechain_init(struct nft_base_chain *basechain, u8 family,
2264                               struct nft_chain_hook *hook, u32 flags)
2265 {
2266         struct nft_chain *chain;
2267         struct nft_hook *h;
2268
2269         basechain->type = hook->type;
2270         INIT_LIST_HEAD(&basechain->hook_list);
2271         chain = &basechain->chain;
2272
2273         if (nft_base_chain_netdev(family, hook->num)) {
2274                 list_splice_init(&hook->list, &basechain->hook_list);
2275                 list_for_each_entry(h, &basechain->hook_list, list)
2276                         nft_basechain_hook_init(&h->ops, family, hook, chain);
2277         }
2278         nft_basechain_hook_init(&basechain->ops, family, hook, chain);
2279
2280         chain->flags |= NFT_CHAIN_BASE | flags;
2281         basechain->policy = NF_ACCEPT;
2282         if (chain->flags & NFT_CHAIN_HW_OFFLOAD &&
2283             !nft_chain_offload_support(basechain)) {
2284                 list_splice_init(&basechain->hook_list, &hook->list);
2285                 return -EOPNOTSUPP;
2286         }
2287
2288         flow_block_init(&basechain->flow_block);
2289
2290         return 0;
2291 }
2292
2293 int nft_chain_add(struct nft_table *table, struct nft_chain *chain)
2294 {
2295         int err;
2296
2297         err = rhltable_insert_key(&table->chains_ht, chain->name,
2298                                   &chain->rhlhead, nft_chain_ht_params);
2299         if (err)
2300                 return err;
2301
2302         list_add_tail_rcu(&chain->list, &table->chains);
2303
2304         return 0;
2305 }
2306
2307 static u64 chain_id;
2308
2309 static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 genmask,
2310                               u8 policy, u32 flags,
2311                               struct netlink_ext_ack *extack)
2312 {
2313         const struct nlattr * const *nla = ctx->nla;
2314         struct nft_table *table = ctx->table;
2315         struct nft_base_chain *basechain;
2316         struct net *net = ctx->net;
2317         char name[NFT_NAME_MAXLEN];
2318         struct nft_rule_blob *blob;
2319         struct nft_trans *trans;
2320         struct nft_chain *chain;
2321         int err;
2322
2323         if (table->use == UINT_MAX)
2324                 return -EOVERFLOW;
2325
2326         if (nla[NFTA_CHAIN_HOOK]) {
2327                 struct nft_stats __percpu *stats = NULL;
2328                 struct nft_chain_hook hook = {};
2329
2330                 if (flags & NFT_CHAIN_BINDING)
2331                         return -EOPNOTSUPP;
2332
2333                 err = nft_chain_parse_hook(net, NULL, nla, &hook, family, flags,
2334                                            extack);
2335                 if (err < 0)
2336                         return err;
2337
2338                 basechain = kzalloc(sizeof(*basechain), GFP_KERNEL_ACCOUNT);
2339                 if (basechain == NULL) {
2340                         nft_chain_release_hook(&hook);
2341                         return -ENOMEM;
2342                 }
2343                 chain = &basechain->chain;
2344
2345                 if (nla[NFTA_CHAIN_COUNTERS]) {
2346                         stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2347                         if (IS_ERR(stats)) {
2348                                 nft_chain_release_hook(&hook);
2349                                 kfree(basechain);
2350                                 return PTR_ERR(stats);
2351                         }
2352                         rcu_assign_pointer(basechain->stats, stats);
2353                 }
2354
2355                 err = nft_basechain_init(basechain, family, &hook, flags);
2356                 if (err < 0) {
2357                         nft_chain_release_hook(&hook);
2358                         kfree(basechain);
2359                         free_percpu(stats);
2360                         return err;
2361                 }
2362                 if (stats)
2363                         static_branch_inc(&nft_counters_enabled);
2364         } else {
2365                 if (flags & NFT_CHAIN_BASE)
2366                         return -EINVAL;
2367                 if (flags & NFT_CHAIN_HW_OFFLOAD)
2368                         return -EOPNOTSUPP;
2369
2370                 chain = kzalloc(sizeof(*chain), GFP_KERNEL_ACCOUNT);
2371                 if (chain == NULL)
2372                         return -ENOMEM;
2373
2374                 chain->flags = flags;
2375         }
2376         ctx->chain = chain;
2377
2378         INIT_LIST_HEAD(&chain->rules);
2379         chain->handle = nf_tables_alloc_handle(table);
2380         chain->table = table;
2381
2382         if (nla[NFTA_CHAIN_NAME]) {
2383                 chain->name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
2384         } else {
2385                 if (!(flags & NFT_CHAIN_BINDING)) {
2386                         err = -EINVAL;
2387                         goto err_destroy_chain;
2388                 }
2389
2390                 snprintf(name, sizeof(name), "__chain%llu", ++chain_id);
2391                 chain->name = kstrdup(name, GFP_KERNEL_ACCOUNT);
2392         }
2393
2394         if (!chain->name) {
2395                 err = -ENOMEM;
2396                 goto err_destroy_chain;
2397         }
2398
2399         if (nla[NFTA_CHAIN_USERDATA]) {
2400                 chain->udata = nla_memdup(nla[NFTA_CHAIN_USERDATA], GFP_KERNEL_ACCOUNT);
2401                 if (chain->udata == NULL) {
2402                         err = -ENOMEM;
2403                         goto err_destroy_chain;
2404                 }
2405                 chain->udlen = nla_len(nla[NFTA_CHAIN_USERDATA]);
2406         }
2407
2408         blob = nf_tables_chain_alloc_rules(chain, 0);
2409         if (!blob) {
2410                 err = -ENOMEM;
2411                 goto err_destroy_chain;
2412         }
2413
2414         RCU_INIT_POINTER(chain->blob_gen_0, blob);
2415         RCU_INIT_POINTER(chain->blob_gen_1, blob);
2416
2417         err = nf_tables_register_hook(net, table, chain);
2418         if (err < 0)
2419                 goto err_destroy_chain;
2420
2421         trans = nft_trans_chain_add(ctx, NFT_MSG_NEWCHAIN);
2422         if (IS_ERR(trans)) {
2423                 err = PTR_ERR(trans);
2424                 goto err_unregister_hook;
2425         }
2426
2427         nft_trans_chain_policy(trans) = NFT_CHAIN_POLICY_UNSET;
2428         if (nft_is_base_chain(chain))
2429                 nft_trans_chain_policy(trans) = policy;
2430
2431         err = nft_chain_add(table, chain);
2432         if (err < 0) {
2433                 nft_trans_destroy(trans);
2434                 goto err_unregister_hook;
2435         }
2436
2437         table->use++;
2438
2439         return 0;
2440 err_unregister_hook:
2441         nf_tables_unregister_hook(net, table, chain);
2442 err_destroy_chain:
2443         nf_tables_chain_destroy(ctx);
2444
2445         return err;
2446 }
2447
2448 static int nf_tables_updchain(struct nft_ctx *ctx, u8 genmask, u8 policy,
2449                               u32 flags, const struct nlattr *attr,
2450                               struct netlink_ext_ack *extack)
2451 {
2452         const struct nlattr * const *nla = ctx->nla;
2453         struct nft_base_chain *basechain = NULL;
2454         struct nft_table *table = ctx->table;
2455         struct nft_chain *chain = ctx->chain;
2456         struct nft_chain_hook hook = {};
2457         struct nft_stats *stats = NULL;
2458         struct nft_hook *h, *next;
2459         struct nf_hook_ops *ops;
2460         struct nft_trans *trans;
2461         bool unregister = false;
2462         int err;
2463
2464         if (chain->flags ^ flags)
2465                 return -EOPNOTSUPP;
2466
2467         INIT_LIST_HEAD(&hook.list);
2468
2469         if (nla[NFTA_CHAIN_HOOK]) {
2470                 if (!nft_is_base_chain(chain)) {
2471                         NL_SET_BAD_ATTR(extack, attr);
2472                         return -EEXIST;
2473                 }
2474
2475                 basechain = nft_base_chain(chain);
2476                 err = nft_chain_parse_hook(ctx->net, basechain, nla, &hook,
2477                                            ctx->family, flags, extack);
2478                 if (err < 0)
2479                         return err;
2480
2481                 if (basechain->type != hook.type) {
2482                         nft_chain_release_hook(&hook);
2483                         NL_SET_BAD_ATTR(extack, attr);
2484                         return -EEXIST;
2485                 }
2486
2487                 if (nft_base_chain_netdev(ctx->family, basechain->ops.hooknum)) {
2488                         list_for_each_entry_safe(h, next, &hook.list, list) {
2489                                 h->ops.pf       = basechain->ops.pf;
2490                                 h->ops.hooknum  = basechain->ops.hooknum;
2491                                 h->ops.priority = basechain->ops.priority;
2492                                 h->ops.priv     = basechain->ops.priv;
2493                                 h->ops.hook     = basechain->ops.hook;
2494
2495                                 if (nft_hook_list_find(&basechain->hook_list, h)) {
2496                                         list_del(&h->list);
2497                                         kfree(h);
2498                                 }
2499                         }
2500                 } else {
2501                         ops = &basechain->ops;
2502                         if (ops->hooknum != hook.num ||
2503                             ops->priority != hook.priority) {
2504                                 nft_chain_release_hook(&hook);
2505                                 NL_SET_BAD_ATTR(extack, attr);
2506                                 return -EEXIST;
2507                         }
2508                 }
2509         }
2510
2511         if (nla[NFTA_CHAIN_HANDLE] &&
2512             nla[NFTA_CHAIN_NAME]) {
2513                 struct nft_chain *chain2;
2514
2515                 chain2 = nft_chain_lookup(ctx->net, table,
2516                                           nla[NFTA_CHAIN_NAME], genmask);
2517                 if (!IS_ERR(chain2)) {
2518                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2519                         err = -EEXIST;
2520                         goto err_hooks;
2521                 }
2522         }
2523
2524         if (nla[NFTA_CHAIN_COUNTERS]) {
2525                 if (!nft_is_base_chain(chain)) {
2526                         err = -EOPNOTSUPP;
2527                         goto err_hooks;
2528                 }
2529
2530                 stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2531                 if (IS_ERR(stats)) {
2532                         err = PTR_ERR(stats);
2533                         goto err_hooks;
2534                 }
2535         }
2536
2537         if (!(table->flags & NFT_TABLE_F_DORMANT) &&
2538             nft_is_base_chain(chain) &&
2539             !list_empty(&hook.list)) {
2540                 basechain = nft_base_chain(chain);
2541                 ops = &basechain->ops;
2542
2543                 if (nft_base_chain_netdev(table->family, basechain->ops.hooknum)) {
2544                         err = nft_netdev_register_hooks(ctx->net, &hook.list);
2545                         if (err < 0)
2546                                 goto err_hooks;
2547                 }
2548         }
2549
2550         unregister = true;
2551         err = -ENOMEM;
2552         trans = nft_trans_alloc(ctx, NFT_MSG_NEWCHAIN,
2553                                 sizeof(struct nft_trans_chain));
2554         if (trans == NULL)
2555                 goto err_trans;
2556
2557         nft_trans_chain_stats(trans) = stats;
2558         nft_trans_chain_update(trans) = true;
2559
2560         if (nla[NFTA_CHAIN_POLICY])
2561                 nft_trans_chain_policy(trans) = policy;
2562         else
2563                 nft_trans_chain_policy(trans) = -1;
2564
2565         if (nla[NFTA_CHAIN_HANDLE] &&
2566             nla[NFTA_CHAIN_NAME]) {
2567                 struct nftables_pernet *nft_net = nft_pernet(ctx->net);
2568                 struct nft_trans *tmp;
2569                 char *name;
2570
2571                 err = -ENOMEM;
2572                 name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
2573                 if (!name)
2574                         goto err_trans;
2575
2576                 err = -EEXIST;
2577                 list_for_each_entry(tmp, &nft_net->commit_list, list) {
2578                         if (tmp->msg_type == NFT_MSG_NEWCHAIN &&
2579                             tmp->ctx.table == table &&
2580                             nft_trans_chain_update(tmp) &&
2581                             nft_trans_chain_name(tmp) &&
2582                             strcmp(name, nft_trans_chain_name(tmp)) == 0) {
2583                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2584                                 kfree(name);
2585                                 goto err_trans;
2586                         }
2587                 }
2588
2589                 nft_trans_chain_name(trans) = name;
2590         }
2591
2592         nft_trans_basechain(trans) = basechain;
2593         INIT_LIST_HEAD(&nft_trans_chain_hooks(trans));
2594         list_splice(&hook.list, &nft_trans_chain_hooks(trans));
2595
2596         nft_trans_commit_list_add_tail(ctx->net, trans);
2597
2598         return 0;
2599
2600 err_trans:
2601         free_percpu(stats);
2602         kfree(trans);
2603 err_hooks:
2604         if (nla[NFTA_CHAIN_HOOK]) {
2605                 list_for_each_entry_safe(h, next, &hook.list, list) {
2606                         if (unregister)
2607                                 nf_unregister_net_hook(ctx->net, &h->ops);
2608                         list_del(&h->list);
2609                         kfree_rcu(h, rcu);
2610                 }
2611                 module_put(hook.type->owner);
2612         }
2613
2614         return err;
2615 }
2616
2617 static struct nft_chain *nft_chain_lookup_byid(const struct net *net,
2618                                                const struct nft_table *table,
2619                                                const struct nlattr *nla)
2620 {
2621         struct nftables_pernet *nft_net = nft_pernet(net);
2622         u32 id = ntohl(nla_get_be32(nla));
2623         struct nft_trans *trans;
2624
2625         list_for_each_entry(trans, &nft_net->commit_list, list) {
2626                 struct nft_chain *chain = trans->ctx.chain;
2627
2628                 if (trans->msg_type == NFT_MSG_NEWCHAIN &&
2629                     chain->table == table &&
2630                     id == nft_trans_chain_id(trans))
2631                         return chain;
2632         }
2633         return ERR_PTR(-ENOENT);
2634 }
2635
2636 static int nf_tables_newchain(struct sk_buff *skb, const struct nfnl_info *info,
2637                               const struct nlattr * const nla[])
2638 {
2639         struct nftables_pernet *nft_net = nft_pernet(info->net);
2640         struct netlink_ext_ack *extack = info->extack;
2641         u8 genmask = nft_genmask_next(info->net);
2642         u8 family = info->nfmsg->nfgen_family;
2643         struct nft_chain *chain = NULL;
2644         struct net *net = info->net;
2645         const struct nlattr *attr;
2646         struct nft_table *table;
2647         u8 policy = NF_ACCEPT;
2648         struct nft_ctx ctx;
2649         u64 handle = 0;
2650         u32 flags = 0;
2651
2652         lockdep_assert_held(&nft_net->commit_mutex);
2653
2654         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
2655                                  NETLINK_CB(skb).portid);
2656         if (IS_ERR(table)) {
2657                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2658                 return PTR_ERR(table);
2659         }
2660
2661         chain = NULL;
2662         attr = nla[NFTA_CHAIN_NAME];
2663
2664         if (nla[NFTA_CHAIN_HANDLE]) {
2665                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_CHAIN_HANDLE]));
2666                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2667                 if (IS_ERR(chain)) {
2668                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_HANDLE]);
2669                         return PTR_ERR(chain);
2670                 }
2671                 attr = nla[NFTA_CHAIN_HANDLE];
2672         } else if (nla[NFTA_CHAIN_NAME]) {
2673                 chain = nft_chain_lookup(net, table, attr, genmask);
2674                 if (IS_ERR(chain)) {
2675                         if (PTR_ERR(chain) != -ENOENT) {
2676                                 NL_SET_BAD_ATTR(extack, attr);
2677                                 return PTR_ERR(chain);
2678                         }
2679                         chain = NULL;
2680                 }
2681         } else if (!nla[NFTA_CHAIN_ID]) {
2682                 return -EINVAL;
2683         }
2684
2685         if (nla[NFTA_CHAIN_POLICY]) {
2686                 if (chain != NULL &&
2687                     !nft_is_base_chain(chain)) {
2688                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2689                         return -EOPNOTSUPP;
2690                 }
2691
2692                 if (chain == NULL &&
2693                     nla[NFTA_CHAIN_HOOK] == NULL) {
2694                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2695                         return -EOPNOTSUPP;
2696                 }
2697
2698                 policy = ntohl(nla_get_be32(nla[NFTA_CHAIN_POLICY]));
2699                 switch (policy) {
2700                 case NF_DROP:
2701                 case NF_ACCEPT:
2702                         break;
2703                 default:
2704                         return -EINVAL;
2705                 }
2706         }
2707
2708         if (nla[NFTA_CHAIN_FLAGS])
2709                 flags = ntohl(nla_get_be32(nla[NFTA_CHAIN_FLAGS]));
2710         else if (chain)
2711                 flags = chain->flags;
2712
2713         if (flags & ~NFT_CHAIN_FLAGS)
2714                 return -EOPNOTSUPP;
2715
2716         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
2717
2718         if (chain != NULL) {
2719                 if (chain->flags & NFT_CHAIN_BINDING)
2720                         return -EINVAL;
2721
2722                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
2723                         NL_SET_BAD_ATTR(extack, attr);
2724                         return -EEXIST;
2725                 }
2726                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
2727                         return -EOPNOTSUPP;
2728
2729                 flags |= chain->flags & NFT_CHAIN_BASE;
2730                 return nf_tables_updchain(&ctx, genmask, policy, flags, attr,
2731                                           extack);
2732         }
2733
2734         return nf_tables_addchain(&ctx, family, genmask, policy, flags, extack);
2735 }
2736
2737 static int nft_delchain_hook(struct nft_ctx *ctx, struct nft_chain *chain,
2738                              struct netlink_ext_ack *extack)
2739 {
2740         const struct nlattr * const *nla = ctx->nla;
2741         struct nft_chain_hook chain_hook = {};
2742         struct nft_base_chain *basechain;
2743         struct nft_hook *this, *hook;
2744         LIST_HEAD(chain_del_list);
2745         struct nft_trans *trans;
2746         int err;
2747
2748         if (!nft_is_base_chain(chain))
2749                 return -EOPNOTSUPP;
2750
2751         basechain = nft_base_chain(chain);
2752         err = nft_chain_parse_hook(ctx->net, basechain, nla, &chain_hook,
2753                                    ctx->family, chain->flags, extack);
2754         if (err < 0)
2755                 return err;
2756
2757         list_for_each_entry(this, &chain_hook.list, list) {
2758                 hook = nft_hook_list_find(&basechain->hook_list, this);
2759                 if (!hook) {
2760                         err = -ENOENT;
2761                         goto err_chain_del_hook;
2762                 }
2763                 list_move(&hook->list, &chain_del_list);
2764         }
2765
2766         trans = nft_trans_alloc(ctx, NFT_MSG_DELCHAIN,
2767                                 sizeof(struct nft_trans_chain));
2768         if (!trans) {
2769                 err = -ENOMEM;
2770                 goto err_chain_del_hook;
2771         }
2772
2773         nft_trans_basechain(trans) = basechain;
2774         nft_trans_chain_update(trans) = true;
2775         INIT_LIST_HEAD(&nft_trans_chain_hooks(trans));
2776         list_splice(&chain_del_list, &nft_trans_chain_hooks(trans));
2777         nft_chain_release_hook(&chain_hook);
2778
2779         nft_trans_commit_list_add_tail(ctx->net, trans);
2780
2781         return 0;
2782
2783 err_chain_del_hook:
2784         list_splice(&chain_del_list, &basechain->hook_list);
2785         nft_chain_release_hook(&chain_hook);
2786
2787         return err;
2788 }
2789
2790 static int nf_tables_delchain(struct sk_buff *skb, const struct nfnl_info *info,
2791                               const struct nlattr * const nla[])
2792 {
2793         struct netlink_ext_ack *extack = info->extack;
2794         u8 genmask = nft_genmask_next(info->net);
2795         u8 family = info->nfmsg->nfgen_family;
2796         struct net *net = info->net;
2797         const struct nlattr *attr;
2798         struct nft_table *table;
2799         struct nft_chain *chain;
2800         struct nft_rule *rule;
2801         struct nft_ctx ctx;
2802         u64 handle;
2803         u32 use;
2804         int err;
2805
2806         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
2807                                  NETLINK_CB(skb).portid);
2808         if (IS_ERR(table)) {
2809                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2810                 return PTR_ERR(table);
2811         }
2812
2813         if (nla[NFTA_CHAIN_HANDLE]) {
2814                 attr = nla[NFTA_CHAIN_HANDLE];
2815                 handle = be64_to_cpu(nla_get_be64(attr));
2816                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2817         } else {
2818                 attr = nla[NFTA_CHAIN_NAME];
2819                 chain = nft_chain_lookup(net, table, attr, genmask);
2820         }
2821         if (IS_ERR(chain)) {
2822                 if (PTR_ERR(chain) == -ENOENT &&
2823                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYCHAIN)
2824                         return 0;
2825
2826                 NL_SET_BAD_ATTR(extack, attr);
2827                 return PTR_ERR(chain);
2828         }
2829
2830         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
2831
2832         if (nla[NFTA_CHAIN_HOOK]) {
2833                 if (chain->flags & NFT_CHAIN_HW_OFFLOAD)
2834                         return -EOPNOTSUPP;
2835
2836                 return nft_delchain_hook(&ctx, chain, extack);
2837         }
2838
2839         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
2840             chain->use > 0)
2841                 return -EBUSY;
2842
2843         use = chain->use;
2844         list_for_each_entry(rule, &chain->rules, list) {
2845                 if (!nft_is_active_next(net, rule))
2846                         continue;
2847                 use--;
2848
2849                 err = nft_delrule(&ctx, rule);
2850                 if (err < 0)
2851                         return err;
2852         }
2853
2854         /* There are rules and elements that are still holding references to us,
2855          * we cannot do a recursive removal in this case.
2856          */
2857         if (use > 0) {
2858                 NL_SET_BAD_ATTR(extack, attr);
2859                 return -EBUSY;
2860         }
2861
2862         return nft_delchain(&ctx);
2863 }
2864
2865 /*
2866  * Expressions
2867  */
2868
2869 /**
2870  *      nft_register_expr - register nf_tables expr type
2871  *      @type: expr type
2872  *
2873  *      Registers the expr type for use with nf_tables. Returns zero on
2874  *      success or a negative errno code otherwise.
2875  */
2876 int nft_register_expr(struct nft_expr_type *type)
2877 {
2878         nfnl_lock(NFNL_SUBSYS_NFTABLES);
2879         if (type->family == NFPROTO_UNSPEC)
2880                 list_add_tail_rcu(&type->list, &nf_tables_expressions);
2881         else
2882                 list_add_rcu(&type->list, &nf_tables_expressions);
2883         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
2884         return 0;
2885 }
2886 EXPORT_SYMBOL_GPL(nft_register_expr);
2887
2888 /**
2889  *      nft_unregister_expr - unregister nf_tables expr type
2890  *      @type: expr type
2891  *
2892  *      Unregisters the expr typefor use with nf_tables.
2893  */
2894 void nft_unregister_expr(struct nft_expr_type *type)
2895 {
2896         nfnl_lock(NFNL_SUBSYS_NFTABLES);
2897         list_del_rcu(&type->list);
2898         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
2899 }
2900 EXPORT_SYMBOL_GPL(nft_unregister_expr);
2901
2902 static const struct nft_expr_type *__nft_expr_type_get(u8 family,
2903                                                        struct nlattr *nla)
2904 {
2905         const struct nft_expr_type *type, *candidate = NULL;
2906
2907         list_for_each_entry(type, &nf_tables_expressions, list) {
2908                 if (!nla_strcmp(nla, type->name)) {
2909                         if (!type->family && !candidate)
2910                                 candidate = type;
2911                         else if (type->family == family)
2912                                 candidate = type;
2913                 }
2914         }
2915         return candidate;
2916 }
2917
2918 #ifdef CONFIG_MODULES
2919 static int nft_expr_type_request_module(struct net *net, u8 family,
2920                                         struct nlattr *nla)
2921 {
2922         if (nft_request_module(net, "nft-expr-%u-%.*s", family,
2923                                nla_len(nla), (char *)nla_data(nla)) == -EAGAIN)
2924                 return -EAGAIN;
2925
2926         return 0;
2927 }
2928 #endif
2929
2930 static const struct nft_expr_type *nft_expr_type_get(struct net *net,
2931                                                      u8 family,
2932                                                      struct nlattr *nla)
2933 {
2934         const struct nft_expr_type *type;
2935
2936         if (nla == NULL)
2937                 return ERR_PTR(-EINVAL);
2938
2939         type = __nft_expr_type_get(family, nla);
2940         if (type != NULL && try_module_get(type->owner))
2941                 return type;
2942
2943         lockdep_nfnl_nft_mutex_not_held();
2944 #ifdef CONFIG_MODULES
2945         if (type == NULL) {
2946                 if (nft_expr_type_request_module(net, family, nla) == -EAGAIN)
2947                         return ERR_PTR(-EAGAIN);
2948
2949                 if (nft_request_module(net, "nft-expr-%.*s",
2950                                        nla_len(nla),
2951                                        (char *)nla_data(nla)) == -EAGAIN)
2952                         return ERR_PTR(-EAGAIN);
2953         }
2954 #endif
2955         return ERR_PTR(-ENOENT);
2956 }
2957
2958 static const struct nla_policy nft_expr_policy[NFTA_EXPR_MAX + 1] = {
2959         [NFTA_EXPR_NAME]        = { .type = NLA_STRING,
2960                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
2961         [NFTA_EXPR_DATA]        = { .type = NLA_NESTED },
2962 };
2963
2964 static int nf_tables_fill_expr_info(struct sk_buff *skb,
2965                                     const struct nft_expr *expr, bool reset)
2966 {
2967         if (nla_put_string(skb, NFTA_EXPR_NAME, expr->ops->type->name))
2968                 goto nla_put_failure;
2969
2970         if (expr->ops->dump) {
2971                 struct nlattr *data = nla_nest_start_noflag(skb,
2972                                                             NFTA_EXPR_DATA);
2973                 if (data == NULL)
2974                         goto nla_put_failure;
2975                 if (expr->ops->dump(skb, expr, reset) < 0)
2976                         goto nla_put_failure;
2977                 nla_nest_end(skb, data);
2978         }
2979
2980         return skb->len;
2981
2982 nla_put_failure:
2983         return -1;
2984 };
2985
2986 int nft_expr_dump(struct sk_buff *skb, unsigned int attr,
2987                   const struct nft_expr *expr, bool reset)
2988 {
2989         struct nlattr *nest;
2990
2991         nest = nla_nest_start_noflag(skb, attr);
2992         if (!nest)
2993                 goto nla_put_failure;
2994         if (nf_tables_fill_expr_info(skb, expr, reset) < 0)
2995                 goto nla_put_failure;
2996         nla_nest_end(skb, nest);
2997         return 0;
2998
2999 nla_put_failure:
3000         return -1;
3001 }
3002
3003 struct nft_expr_info {
3004         const struct nft_expr_ops       *ops;
3005         const struct nlattr             *attr;
3006         struct nlattr                   *tb[NFT_EXPR_MAXATTR + 1];
3007 };
3008
3009 static int nf_tables_expr_parse(const struct nft_ctx *ctx,
3010                                 const struct nlattr *nla,
3011                                 struct nft_expr_info *info)
3012 {
3013         const struct nft_expr_type *type;
3014         const struct nft_expr_ops *ops;
3015         struct nlattr *tb[NFTA_EXPR_MAX + 1];
3016         int err;
3017
3018         err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
3019                                           nft_expr_policy, NULL);
3020         if (err < 0)
3021                 return err;
3022
3023         type = nft_expr_type_get(ctx->net, ctx->family, tb[NFTA_EXPR_NAME]);
3024         if (IS_ERR(type))
3025                 return PTR_ERR(type);
3026
3027         if (tb[NFTA_EXPR_DATA]) {
3028                 err = nla_parse_nested_deprecated(info->tb, type->maxattr,
3029                                                   tb[NFTA_EXPR_DATA],
3030                                                   type->policy, NULL);
3031                 if (err < 0)
3032                         goto err1;
3033         } else
3034                 memset(info->tb, 0, sizeof(info->tb[0]) * (type->maxattr + 1));
3035
3036         if (type->select_ops != NULL) {
3037                 ops = type->select_ops(ctx,
3038                                        (const struct nlattr * const *)info->tb);
3039                 if (IS_ERR(ops)) {
3040                         err = PTR_ERR(ops);
3041 #ifdef CONFIG_MODULES
3042                         if (err == -EAGAIN)
3043                                 if (nft_expr_type_request_module(ctx->net,
3044                                                                  ctx->family,
3045                                                                  tb[NFTA_EXPR_NAME]) != -EAGAIN)
3046                                         err = -ENOENT;
3047 #endif
3048                         goto err1;
3049                 }
3050         } else
3051                 ops = type->ops;
3052
3053         info->attr = nla;
3054         info->ops = ops;
3055
3056         return 0;
3057
3058 err1:
3059         module_put(type->owner);
3060         return err;
3061 }
3062
3063 int nft_expr_inner_parse(const struct nft_ctx *ctx, const struct nlattr *nla,
3064                          struct nft_expr_info *info)
3065 {
3066         struct nlattr *tb[NFTA_EXPR_MAX + 1];
3067         const struct nft_expr_type *type;
3068         int err;
3069
3070         err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
3071                                           nft_expr_policy, NULL);
3072         if (err < 0)
3073                 return err;
3074
3075         if (!tb[NFTA_EXPR_DATA])
3076                 return -EINVAL;
3077
3078         type = __nft_expr_type_get(ctx->family, tb[NFTA_EXPR_NAME]);
3079         if (!type)
3080                 return -ENOENT;
3081
3082         if (!type->inner_ops)
3083                 return -EOPNOTSUPP;
3084
3085         err = nla_parse_nested_deprecated(info->tb, type->maxattr,
3086                                           tb[NFTA_EXPR_DATA],
3087                                           type->policy, NULL);
3088         if (err < 0)
3089                 goto err_nla_parse;
3090
3091         info->attr = nla;
3092         info->ops = type->inner_ops;
3093
3094         return 0;
3095
3096 err_nla_parse:
3097         return err;
3098 }
3099
3100 static int nf_tables_newexpr(const struct nft_ctx *ctx,
3101                              const struct nft_expr_info *expr_info,
3102                              struct nft_expr *expr)
3103 {
3104         const struct nft_expr_ops *ops = expr_info->ops;
3105         int err;
3106
3107         expr->ops = ops;
3108         if (ops->init) {
3109                 err = ops->init(ctx, expr, (const struct nlattr **)expr_info->tb);
3110                 if (err < 0)
3111                         goto err1;
3112         }
3113
3114         return 0;
3115 err1:
3116         expr->ops = NULL;
3117         return err;
3118 }
3119
3120 static void nf_tables_expr_destroy(const struct nft_ctx *ctx,
3121                                    struct nft_expr *expr)
3122 {
3123         const struct nft_expr_type *type = expr->ops->type;
3124
3125         if (expr->ops->destroy)
3126                 expr->ops->destroy(ctx, expr);
3127         module_put(type->owner);
3128 }
3129
3130 static struct nft_expr *nft_expr_init(const struct nft_ctx *ctx,
3131                                       const struct nlattr *nla)
3132 {
3133         struct nft_expr_info expr_info;
3134         struct nft_expr *expr;
3135         struct module *owner;
3136         int err;
3137
3138         err = nf_tables_expr_parse(ctx, nla, &expr_info);
3139         if (err < 0)
3140                 goto err_expr_parse;
3141
3142         err = -EOPNOTSUPP;
3143         if (!(expr_info.ops->type->flags & NFT_EXPR_STATEFUL))
3144                 goto err_expr_stateful;
3145
3146         err = -ENOMEM;
3147         expr = kzalloc(expr_info.ops->size, GFP_KERNEL_ACCOUNT);
3148         if (expr == NULL)
3149                 goto err_expr_stateful;
3150
3151         err = nf_tables_newexpr(ctx, &expr_info, expr);
3152         if (err < 0)
3153                 goto err_expr_new;
3154
3155         return expr;
3156 err_expr_new:
3157         kfree(expr);
3158 err_expr_stateful:
3159         owner = expr_info.ops->type->owner;
3160         if (expr_info.ops->type->release_ops)
3161                 expr_info.ops->type->release_ops(expr_info.ops);
3162
3163         module_put(owner);
3164 err_expr_parse:
3165         return ERR_PTR(err);
3166 }
3167
3168 int nft_expr_clone(struct nft_expr *dst, struct nft_expr *src)
3169 {
3170         int err;
3171
3172         if (src->ops->clone) {
3173                 dst->ops = src->ops;
3174                 err = src->ops->clone(dst, src);
3175                 if (err < 0)
3176                         return err;
3177         } else {
3178                 memcpy(dst, src, src->ops->size);
3179         }
3180
3181         __module_get(src->ops->type->owner);
3182
3183         return 0;
3184 }
3185
3186 void nft_expr_destroy(const struct nft_ctx *ctx, struct nft_expr *expr)
3187 {
3188         nf_tables_expr_destroy(ctx, expr);
3189         kfree(expr);
3190 }
3191
3192 /*
3193  * Rules
3194  */
3195
3196 static struct nft_rule *__nft_rule_lookup(const struct nft_chain *chain,
3197                                           u64 handle)
3198 {
3199         struct nft_rule *rule;
3200
3201         // FIXME: this sucks
3202         list_for_each_entry_rcu(rule, &chain->rules, list) {
3203                 if (handle == rule->handle)
3204                         return rule;
3205         }
3206
3207         return ERR_PTR(-ENOENT);
3208 }
3209
3210 static struct nft_rule *nft_rule_lookup(const struct nft_chain *chain,
3211                                         const struct nlattr *nla)
3212 {
3213         if (nla == NULL)
3214                 return ERR_PTR(-EINVAL);
3215
3216         return __nft_rule_lookup(chain, be64_to_cpu(nla_get_be64(nla)));
3217 }
3218
3219 static const struct nla_policy nft_rule_policy[NFTA_RULE_MAX + 1] = {
3220         [NFTA_RULE_TABLE]       = { .type = NLA_STRING,
3221                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
3222         [NFTA_RULE_CHAIN]       = { .type = NLA_STRING,
3223                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
3224         [NFTA_RULE_HANDLE]      = { .type = NLA_U64 },
3225         [NFTA_RULE_EXPRESSIONS] = { .type = NLA_NESTED },
3226         [NFTA_RULE_COMPAT]      = { .type = NLA_NESTED },
3227         [NFTA_RULE_POSITION]    = { .type = NLA_U64 },
3228         [NFTA_RULE_USERDATA]    = { .type = NLA_BINARY,
3229                                     .len = NFT_USERDATA_MAXLEN },
3230         [NFTA_RULE_ID]          = { .type = NLA_U32 },
3231         [NFTA_RULE_POSITION_ID] = { .type = NLA_U32 },
3232         [NFTA_RULE_CHAIN_ID]    = { .type = NLA_U32 },
3233 };
3234
3235 static int nf_tables_fill_rule_info(struct sk_buff *skb, struct net *net,
3236                                     u32 portid, u32 seq, int event,
3237                                     u32 flags, int family,
3238                                     const struct nft_table *table,
3239                                     const struct nft_chain *chain,
3240                                     const struct nft_rule *rule, u64 handle,
3241                                     bool reset)
3242 {
3243         struct nlmsghdr *nlh;
3244         const struct nft_expr *expr, *next;
3245         struct nlattr *list;
3246         u16 type = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
3247
3248         nlh = nfnl_msg_put(skb, portid, seq, type, flags, family, NFNETLINK_V0,
3249                            nft_base_seq(net));
3250         if (!nlh)
3251                 goto nla_put_failure;
3252
3253         if (nla_put_string(skb, NFTA_RULE_TABLE, table->name))
3254                 goto nla_put_failure;
3255         if (nla_put_string(skb, NFTA_RULE_CHAIN, chain->name))
3256                 goto nla_put_failure;
3257         if (nla_put_be64(skb, NFTA_RULE_HANDLE, cpu_to_be64(rule->handle),
3258                          NFTA_RULE_PAD))
3259                 goto nla_put_failure;
3260
3261         if (event != NFT_MSG_DELRULE && handle) {
3262                 if (nla_put_be64(skb, NFTA_RULE_POSITION, cpu_to_be64(handle),
3263                                  NFTA_RULE_PAD))
3264                         goto nla_put_failure;
3265         }
3266
3267         if (chain->flags & NFT_CHAIN_HW_OFFLOAD)
3268                 nft_flow_rule_stats(chain, rule);
3269
3270         list = nla_nest_start_noflag(skb, NFTA_RULE_EXPRESSIONS);
3271         if (list == NULL)
3272                 goto nla_put_failure;
3273         nft_rule_for_each_expr(expr, next, rule) {
3274                 if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr, reset) < 0)
3275                         goto nla_put_failure;
3276         }
3277         nla_nest_end(skb, list);
3278
3279         if (rule->udata) {
3280                 struct nft_userdata *udata = nft_userdata(rule);
3281                 if (nla_put(skb, NFTA_RULE_USERDATA, udata->len + 1,
3282                             udata->data) < 0)
3283                         goto nla_put_failure;
3284         }
3285
3286         nlmsg_end(skb, nlh);
3287         return 0;
3288
3289 nla_put_failure:
3290         nlmsg_trim(skb, nlh);
3291         return -1;
3292 }
3293
3294 static void nf_tables_rule_notify(const struct nft_ctx *ctx,
3295                                   const struct nft_rule *rule, int event)
3296 {
3297         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
3298         const struct nft_rule *prule;
3299         struct sk_buff *skb;
3300         u64 handle = 0;
3301         u16 flags = 0;
3302         int err;
3303
3304         if (!ctx->report &&
3305             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
3306                 return;
3307
3308         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
3309         if (skb == NULL)
3310                 goto err;
3311
3312         if (event == NFT_MSG_NEWRULE &&
3313             !list_is_first(&rule->list, &ctx->chain->rules) &&
3314             !list_is_last(&rule->list, &ctx->chain->rules)) {
3315                 prule = list_prev_entry(rule, list);
3316                 handle = prule->handle;
3317         }
3318         if (ctx->flags & (NLM_F_APPEND | NLM_F_REPLACE))
3319                 flags |= NLM_F_APPEND;
3320         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
3321                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
3322
3323         err = nf_tables_fill_rule_info(skb, ctx->net, ctx->portid, ctx->seq,
3324                                        event, flags, ctx->family, ctx->table,
3325                                        ctx->chain, rule, handle, false);
3326         if (err < 0) {
3327                 kfree_skb(skb);
3328                 goto err;
3329         }
3330
3331         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
3332         return;
3333 err:
3334         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
3335 }
3336
3337 struct nft_rule_dump_ctx {
3338         char *table;
3339         char *chain;
3340 };
3341
3342 static int __nf_tables_dump_rules(struct sk_buff *skb,
3343                                   unsigned int *idx,
3344                                   struct netlink_callback *cb,
3345                                   const struct nft_table *table,
3346                                   const struct nft_chain *chain,
3347                                   bool reset)
3348 {
3349         struct net *net = sock_net(skb->sk);
3350         const struct nft_rule *rule, *prule;
3351         unsigned int s_idx = cb->args[0];
3352         u64 handle;
3353
3354         prule = NULL;
3355         list_for_each_entry_rcu(rule, &chain->rules, list) {
3356                 if (!nft_is_active(net, rule))
3357                         goto cont_skip;
3358                 if (*idx < s_idx)
3359                         goto cont;
3360                 if (*idx > s_idx) {
3361                         memset(&cb->args[1], 0,
3362                                         sizeof(cb->args) - sizeof(cb->args[0]));
3363                 }
3364                 if (prule)
3365                         handle = prule->handle;
3366                 else
3367                         handle = 0;
3368
3369                 if (nf_tables_fill_rule_info(skb, net, NETLINK_CB(cb->skb).portid,
3370                                         cb->nlh->nlmsg_seq,
3371                                         NFT_MSG_NEWRULE,
3372                                         NLM_F_MULTI | NLM_F_APPEND,
3373                                         table->family,
3374                                         table, chain, rule, handle, reset) < 0)
3375                         return 1;
3376
3377                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
3378 cont:
3379                 prule = rule;
3380 cont_skip:
3381                 (*idx)++;
3382         }
3383         return 0;
3384 }
3385
3386 static int nf_tables_dump_rules(struct sk_buff *skb,
3387                                 struct netlink_callback *cb)
3388 {
3389         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
3390         const struct nft_rule_dump_ctx *ctx = cb->data;
3391         struct nft_table *table;
3392         const struct nft_chain *chain;
3393         unsigned int idx = 0;
3394         struct net *net = sock_net(skb->sk);
3395         int family = nfmsg->nfgen_family;
3396         struct nftables_pernet *nft_net;
3397         bool reset = false;
3398
3399         if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETRULE_RESET)
3400                 reset = true;
3401
3402         rcu_read_lock();
3403         nft_net = nft_pernet(net);
3404         cb->seq = READ_ONCE(nft_net->base_seq);
3405
3406         list_for_each_entry_rcu(table, &nft_net->tables, list) {
3407                 if (family != NFPROTO_UNSPEC && family != table->family)
3408                         continue;
3409
3410                 if (ctx && ctx->table && strcmp(ctx->table, table->name) != 0)
3411                         continue;
3412
3413                 if (ctx && ctx->table && ctx->chain) {
3414                         struct rhlist_head *list, *tmp;
3415
3416                         list = rhltable_lookup(&table->chains_ht, ctx->chain,
3417                                                nft_chain_ht_params);
3418                         if (!list)
3419                                 goto done;
3420
3421                         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
3422                                 if (!nft_is_active(net, chain))
3423                                         continue;
3424                                 __nf_tables_dump_rules(skb, &idx,
3425                                                        cb, table, chain, reset);
3426                                 break;
3427                         }
3428                         goto done;
3429                 }
3430
3431                 list_for_each_entry_rcu(chain, &table->chains, list) {
3432                         if (__nf_tables_dump_rules(skb, &idx,
3433                                                    cb, table, chain, reset))
3434                                 goto done;
3435                 }
3436
3437                 if (ctx && ctx->table)
3438                         break;
3439         }
3440 done:
3441         rcu_read_unlock();
3442
3443         cb->args[0] = idx;
3444         return skb->len;
3445 }
3446
3447 static int nf_tables_dump_rules_start(struct netlink_callback *cb)
3448 {
3449         const struct nlattr * const *nla = cb->data;
3450         struct nft_rule_dump_ctx *ctx = NULL;
3451
3452         if (nla[NFTA_RULE_TABLE] || nla[NFTA_RULE_CHAIN]) {
3453                 ctx = kzalloc(sizeof(*ctx), GFP_ATOMIC);
3454                 if (!ctx)
3455                         return -ENOMEM;
3456
3457                 if (nla[NFTA_RULE_TABLE]) {
3458                         ctx->table = nla_strdup(nla[NFTA_RULE_TABLE],
3459                                                         GFP_ATOMIC);
3460                         if (!ctx->table) {
3461                                 kfree(ctx);
3462                                 return -ENOMEM;
3463                         }
3464                 }
3465                 if (nla[NFTA_RULE_CHAIN]) {
3466                         ctx->chain = nla_strdup(nla[NFTA_RULE_CHAIN],
3467                                                 GFP_ATOMIC);
3468                         if (!ctx->chain) {
3469                                 kfree(ctx->table);
3470                                 kfree(ctx);
3471                                 return -ENOMEM;
3472                         }
3473                 }
3474         }
3475
3476         cb->data = ctx;
3477         return 0;
3478 }
3479
3480 static int nf_tables_dump_rules_done(struct netlink_callback *cb)
3481 {
3482         struct nft_rule_dump_ctx *ctx = cb->data;
3483
3484         if (ctx) {
3485                 kfree(ctx->table);
3486                 kfree(ctx->chain);
3487                 kfree(ctx);
3488         }
3489         return 0;
3490 }
3491
3492 /* called with rcu_read_lock held */
3493 static int nf_tables_getrule(struct sk_buff *skb, const struct nfnl_info *info,
3494                              const struct nlattr * const nla[])
3495 {
3496         struct netlink_ext_ack *extack = info->extack;
3497         u8 genmask = nft_genmask_cur(info->net);
3498         u8 family = info->nfmsg->nfgen_family;
3499         const struct nft_chain *chain;
3500         const struct nft_rule *rule;
3501         struct net *net = info->net;
3502         struct nft_table *table;
3503         struct sk_buff *skb2;
3504         bool reset = false;
3505         int err;
3506
3507         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
3508                 struct netlink_dump_control c = {
3509                         .start= nf_tables_dump_rules_start,
3510                         .dump = nf_tables_dump_rules,
3511                         .done = nf_tables_dump_rules_done,
3512                         .module = THIS_MODULE,
3513                         .data = (void *)nla,
3514                 };
3515
3516                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
3517         }
3518
3519         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask, 0);
3520         if (IS_ERR(table)) {
3521                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3522                 return PTR_ERR(table);
3523         }
3524
3525         chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN], genmask);
3526         if (IS_ERR(chain)) {
3527                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3528                 return PTR_ERR(chain);
3529         }
3530
3531         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
3532         if (IS_ERR(rule)) {
3533                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3534                 return PTR_ERR(rule);
3535         }
3536
3537         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
3538         if (!skb2)
3539                 return -ENOMEM;
3540
3541         if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETRULE_RESET)
3542                 reset = true;
3543
3544         err = nf_tables_fill_rule_info(skb2, net, NETLINK_CB(skb).portid,
3545                                        info->nlh->nlmsg_seq, NFT_MSG_NEWRULE, 0,
3546                                        family, table, chain, rule, 0, reset);
3547         if (err < 0)
3548                 goto err_fill_rule_info;
3549
3550         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
3551
3552 err_fill_rule_info:
3553         kfree_skb(skb2);
3554         return err;
3555 }
3556
3557 void nf_tables_rule_destroy(const struct nft_ctx *ctx, struct nft_rule *rule)
3558 {
3559         struct nft_expr *expr, *next;
3560
3561         /*
3562          * Careful: some expressions might not be initialized in case this
3563          * is called on error from nf_tables_newrule().
3564          */
3565         expr = nft_expr_first(rule);
3566         while (nft_expr_more(rule, expr)) {
3567                 next = nft_expr_next(expr);
3568                 nf_tables_expr_destroy(ctx, expr);
3569                 expr = next;
3570         }
3571         kfree(rule);
3572 }
3573
3574 static void nf_tables_rule_release(const struct nft_ctx *ctx, struct nft_rule *rule)
3575 {
3576         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_RELEASE);
3577         nf_tables_rule_destroy(ctx, rule);
3578 }
3579
3580 int nft_chain_validate(const struct nft_ctx *ctx, const struct nft_chain *chain)
3581 {
3582         struct nft_expr *expr, *last;
3583         const struct nft_data *data;
3584         struct nft_rule *rule;
3585         int err;
3586
3587         if (ctx->level == NFT_JUMP_STACK_SIZE)
3588                 return -EMLINK;
3589
3590         list_for_each_entry(rule, &chain->rules, list) {
3591                 if (!nft_is_active_next(ctx->net, rule))
3592                         continue;
3593
3594                 nft_rule_for_each_expr(expr, last, rule) {
3595                         if (!expr->ops->validate)
3596                                 continue;
3597
3598                         err = expr->ops->validate(ctx, expr, &data);
3599                         if (err < 0)
3600                                 return err;
3601                 }
3602
3603                 cond_resched();
3604         }
3605
3606         return 0;
3607 }
3608 EXPORT_SYMBOL_GPL(nft_chain_validate);
3609
3610 static int nft_table_validate(struct net *net, const struct nft_table *table)
3611 {
3612         struct nft_chain *chain;
3613         struct nft_ctx ctx = {
3614                 .net    = net,
3615                 .family = table->family,
3616         };
3617         int err;
3618
3619         list_for_each_entry(chain, &table->chains, list) {
3620                 if (!nft_is_base_chain(chain))
3621                         continue;
3622
3623                 ctx.chain = chain;
3624                 err = nft_chain_validate(&ctx, chain);
3625                 if (err < 0)
3626                         return err;
3627         }
3628
3629         return 0;
3630 }
3631
3632 int nft_setelem_validate(const struct nft_ctx *ctx, struct nft_set *set,
3633                          const struct nft_set_iter *iter,
3634                          struct nft_set_elem *elem)
3635 {
3636         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
3637         struct nft_ctx *pctx = (struct nft_ctx *)ctx;
3638         const struct nft_data *data;
3639         int err;
3640
3641         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
3642             *nft_set_ext_flags(ext) & NFT_SET_ELEM_INTERVAL_END)
3643                 return 0;
3644
3645         data = nft_set_ext_data(ext);
3646         switch (data->verdict.code) {
3647         case NFT_JUMP:
3648         case NFT_GOTO:
3649                 pctx->level++;
3650                 err = nft_chain_validate(ctx, data->verdict.chain);
3651                 if (err < 0)
3652                         return err;
3653                 pctx->level--;
3654                 break;
3655         default:
3656                 break;
3657         }
3658
3659         return 0;
3660 }
3661
3662 struct nft_set_elem_catchall {
3663         struct list_head        list;
3664         struct rcu_head         rcu;
3665         void                    *elem;
3666 };
3667
3668 int nft_set_catchall_validate(const struct nft_ctx *ctx, struct nft_set *set)
3669 {
3670         u8 genmask = nft_genmask_next(ctx->net);
3671         struct nft_set_elem_catchall *catchall;
3672         struct nft_set_elem elem;
3673         struct nft_set_ext *ext;
3674         int ret = 0;
3675
3676         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
3677                 ext = nft_set_elem_ext(set, catchall->elem);
3678                 if (!nft_set_elem_active(ext, genmask))
3679                         continue;
3680
3681                 elem.priv = catchall->elem;
3682                 ret = nft_setelem_validate(ctx, set, NULL, &elem);
3683                 if (ret < 0)
3684                         return ret;
3685         }
3686
3687         return ret;
3688 }
3689
3690 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
3691                                              const struct nft_chain *chain,
3692                                              const struct nlattr *nla);
3693
3694 #define NFT_RULE_MAXEXPRS       128
3695
3696 static int nf_tables_newrule(struct sk_buff *skb, const struct nfnl_info *info,
3697                              const struct nlattr * const nla[])
3698 {
3699         struct nftables_pernet *nft_net = nft_pernet(info->net);
3700         struct netlink_ext_ack *extack = info->extack;
3701         unsigned int size, i, n, ulen = 0, usize = 0;
3702         u8 genmask = nft_genmask_next(info->net);
3703         struct nft_rule *rule, *old_rule = NULL;
3704         struct nft_expr_info *expr_info = NULL;
3705         u8 family = info->nfmsg->nfgen_family;
3706         struct nft_flow_rule *flow = NULL;
3707         struct net *net = info->net;
3708         struct nft_userdata *udata;
3709         struct nft_table *table;
3710         struct nft_chain *chain;
3711         struct nft_trans *trans;
3712         u64 handle, pos_handle;
3713         struct nft_expr *expr;
3714         struct nft_ctx ctx;
3715         struct nlattr *tmp;
3716         int err, rem;
3717
3718         lockdep_assert_held(&nft_net->commit_mutex);
3719
3720         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
3721                                  NETLINK_CB(skb).portid);
3722         if (IS_ERR(table)) {
3723                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3724                 return PTR_ERR(table);
3725         }
3726
3727         if (nla[NFTA_RULE_CHAIN]) {
3728                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
3729                                          genmask);
3730                 if (IS_ERR(chain)) {
3731                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3732                         return PTR_ERR(chain);
3733                 }
3734                 if (nft_chain_is_bound(chain))
3735                         return -EOPNOTSUPP;
3736
3737         } else if (nla[NFTA_RULE_CHAIN_ID]) {
3738                 chain = nft_chain_lookup_byid(net, table, nla[NFTA_RULE_CHAIN_ID]);
3739                 if (IS_ERR(chain)) {
3740                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN_ID]);
3741                         return PTR_ERR(chain);
3742                 }
3743         } else {
3744                 return -EINVAL;
3745         }
3746
3747         if (nla[NFTA_RULE_HANDLE]) {
3748                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_HANDLE]));
3749                 rule = __nft_rule_lookup(chain, handle);
3750                 if (IS_ERR(rule)) {
3751                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3752                         return PTR_ERR(rule);
3753                 }
3754
3755                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
3756                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3757                         return -EEXIST;
3758                 }
3759                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
3760                         old_rule = rule;
3761                 else
3762                         return -EOPNOTSUPP;
3763         } else {
3764                 if (!(info->nlh->nlmsg_flags & NLM_F_CREATE) ||
3765                     info->nlh->nlmsg_flags & NLM_F_REPLACE)
3766                         return -EINVAL;
3767                 handle = nf_tables_alloc_handle(table);
3768
3769                 if (chain->use == UINT_MAX)
3770                         return -EOVERFLOW;
3771
3772                 if (nla[NFTA_RULE_POSITION]) {
3773                         pos_handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_POSITION]));
3774                         old_rule = __nft_rule_lookup(chain, pos_handle);
3775                         if (IS_ERR(old_rule)) {
3776                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION]);
3777                                 return PTR_ERR(old_rule);
3778                         }
3779                 } else if (nla[NFTA_RULE_POSITION_ID]) {
3780                         old_rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_POSITION_ID]);
3781                         if (IS_ERR(old_rule)) {
3782                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION_ID]);
3783                                 return PTR_ERR(old_rule);
3784                         }
3785                 }
3786         }
3787
3788         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
3789
3790         n = 0;
3791         size = 0;
3792         if (nla[NFTA_RULE_EXPRESSIONS]) {
3793                 expr_info = kvmalloc_array(NFT_RULE_MAXEXPRS,
3794                                            sizeof(struct nft_expr_info),
3795                                            GFP_KERNEL);
3796                 if (!expr_info)
3797                         return -ENOMEM;
3798
3799                 nla_for_each_nested(tmp, nla[NFTA_RULE_EXPRESSIONS], rem) {
3800                         err = -EINVAL;
3801                         if (nla_type(tmp) != NFTA_LIST_ELEM)
3802                                 goto err_release_expr;
3803                         if (n == NFT_RULE_MAXEXPRS)
3804                                 goto err_release_expr;
3805                         err = nf_tables_expr_parse(&ctx, tmp, &expr_info[n]);
3806                         if (err < 0) {
3807                                 NL_SET_BAD_ATTR(extack, tmp);
3808                                 goto err_release_expr;
3809                         }
3810                         size += expr_info[n].ops->size;
3811                         n++;
3812                 }
3813         }
3814         /* Check for overflow of dlen field */
3815         err = -EFBIG;
3816         if (size >= 1 << 12)
3817                 goto err_release_expr;
3818
3819         if (nla[NFTA_RULE_USERDATA]) {
3820                 ulen = nla_len(nla[NFTA_RULE_USERDATA]);
3821                 if (ulen > 0)
3822                         usize = sizeof(struct nft_userdata) + ulen;
3823         }
3824
3825         err = -ENOMEM;
3826         rule = kzalloc(sizeof(*rule) + size + usize, GFP_KERNEL_ACCOUNT);
3827         if (rule == NULL)
3828                 goto err_release_expr;
3829
3830         nft_activate_next(net, rule);
3831
3832         rule->handle = handle;
3833         rule->dlen   = size;
3834         rule->udata  = ulen ? 1 : 0;
3835
3836         if (ulen) {
3837                 udata = nft_userdata(rule);
3838                 udata->len = ulen - 1;
3839                 nla_memcpy(udata->data, nla[NFTA_RULE_USERDATA], ulen);
3840         }
3841
3842         expr = nft_expr_first(rule);
3843         for (i = 0; i < n; i++) {
3844                 err = nf_tables_newexpr(&ctx, &expr_info[i], expr);
3845                 if (err < 0) {
3846                         NL_SET_BAD_ATTR(extack, expr_info[i].attr);
3847                         goto err_release_rule;
3848                 }
3849
3850                 if (expr_info[i].ops->validate)
3851                         nft_validate_state_update(table, NFT_VALIDATE_NEED);
3852
3853                 expr_info[i].ops = NULL;
3854                 expr = nft_expr_next(expr);
3855         }
3856
3857         if (chain->flags & NFT_CHAIN_HW_OFFLOAD) {
3858                 flow = nft_flow_rule_create(net, rule);
3859                 if (IS_ERR(flow)) {
3860                         err = PTR_ERR(flow);
3861                         goto err_release_rule;
3862                 }
3863         }
3864
3865         if (info->nlh->nlmsg_flags & NLM_F_REPLACE) {
3866                 err = nft_delrule(&ctx, old_rule);
3867                 if (err < 0)
3868                         goto err_destroy_flow_rule;
3869
3870                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
3871                 if (trans == NULL) {
3872                         err = -ENOMEM;
3873                         goto err_destroy_flow_rule;
3874                 }
3875                 list_add_tail_rcu(&rule->list, &old_rule->list);
3876         } else {
3877                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
3878                 if (!trans) {
3879                         err = -ENOMEM;
3880                         goto err_destroy_flow_rule;
3881                 }
3882
3883                 if (info->nlh->nlmsg_flags & NLM_F_APPEND) {
3884                         if (old_rule)
3885                                 list_add_rcu(&rule->list, &old_rule->list);
3886                         else
3887                                 list_add_tail_rcu(&rule->list, &chain->rules);
3888                  } else {
3889                         if (old_rule)
3890                                 list_add_tail_rcu(&rule->list, &old_rule->list);
3891                         else
3892                                 list_add_rcu(&rule->list, &chain->rules);
3893                 }
3894         }
3895         kvfree(expr_info);
3896         chain->use++;
3897
3898         if (flow)
3899                 nft_trans_flow_rule(trans) = flow;
3900
3901         if (table->validate_state == NFT_VALIDATE_DO)
3902                 return nft_table_validate(net, table);
3903
3904         return 0;
3905
3906 err_destroy_flow_rule:
3907         if (flow)
3908                 nft_flow_rule_destroy(flow);
3909 err_release_rule:
3910         nft_rule_expr_deactivate(&ctx, rule, NFT_TRANS_PREPARE_ERROR);
3911         nf_tables_rule_destroy(&ctx, rule);
3912 err_release_expr:
3913         for (i = 0; i < n; i++) {
3914                 if (expr_info[i].ops) {
3915                         module_put(expr_info[i].ops->type->owner);
3916                         if (expr_info[i].ops->type->release_ops)
3917                                 expr_info[i].ops->type->release_ops(expr_info[i].ops);
3918                 }
3919         }
3920         kvfree(expr_info);
3921
3922         return err;
3923 }
3924
3925 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
3926                                              const struct nft_chain *chain,
3927                                              const struct nlattr *nla)
3928 {
3929         struct nftables_pernet *nft_net = nft_pernet(net);
3930         u32 id = ntohl(nla_get_be32(nla));
3931         struct nft_trans *trans;
3932
3933         list_for_each_entry(trans, &nft_net->commit_list, list) {
3934                 if (trans->msg_type == NFT_MSG_NEWRULE &&
3935                     trans->ctx.chain == chain &&
3936                     id == nft_trans_rule_id(trans))
3937                         return nft_trans_rule(trans);
3938         }
3939         return ERR_PTR(-ENOENT);
3940 }
3941
3942 static int nf_tables_delrule(struct sk_buff *skb, const struct nfnl_info *info,
3943                              const struct nlattr * const nla[])
3944 {
3945         struct netlink_ext_ack *extack = info->extack;
3946         u8 genmask = nft_genmask_next(info->net);
3947         u8 family = info->nfmsg->nfgen_family;
3948         struct nft_chain *chain = NULL;
3949         struct net *net = info->net;
3950         struct nft_table *table;
3951         struct nft_rule *rule;
3952         struct nft_ctx ctx;
3953         int err = 0;
3954
3955         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
3956                                  NETLINK_CB(skb).portid);
3957         if (IS_ERR(table)) {
3958                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3959                 return PTR_ERR(table);
3960         }
3961
3962         if (nla[NFTA_RULE_CHAIN]) {
3963                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
3964                                          genmask);
3965                 if (IS_ERR(chain)) {
3966                         if (PTR_ERR(chain) == -ENOENT &&
3967                             NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYRULE)
3968                                 return 0;
3969
3970                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3971                         return PTR_ERR(chain);
3972                 }
3973                 if (nft_chain_is_bound(chain))
3974                         return -EOPNOTSUPP;
3975         }
3976
3977         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
3978
3979         if (chain) {
3980                 if (nla[NFTA_RULE_HANDLE]) {
3981                         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
3982                         if (IS_ERR(rule)) {
3983                                 if (PTR_ERR(rule) == -ENOENT &&
3984                                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYRULE)
3985                                         return 0;
3986
3987                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3988                                 return PTR_ERR(rule);
3989                         }
3990
3991                         err = nft_delrule(&ctx, rule);
3992                 } else if (nla[NFTA_RULE_ID]) {
3993                         rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_ID]);
3994                         if (IS_ERR(rule)) {
3995                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_ID]);
3996                                 return PTR_ERR(rule);
3997                         }
3998
3999                         err = nft_delrule(&ctx, rule);
4000                 } else {
4001                         err = nft_delrule_by_chain(&ctx);
4002                 }
4003         } else {
4004                 list_for_each_entry(chain, &table->chains, list) {
4005                         if (!nft_is_active_next(net, chain))
4006                                 continue;
4007
4008                         ctx.chain = chain;
4009                         err = nft_delrule_by_chain(&ctx);
4010                         if (err < 0)
4011                                 break;
4012                 }
4013         }
4014
4015         return err;
4016 }
4017
4018 /*
4019  * Sets
4020  */
4021 static const struct nft_set_type *nft_set_types[] = {
4022         &nft_set_hash_fast_type,
4023         &nft_set_hash_type,
4024         &nft_set_rhash_type,
4025         &nft_set_bitmap_type,
4026         &nft_set_rbtree_type,
4027 #if defined(CONFIG_X86_64) && !defined(CONFIG_UML)
4028         &nft_set_pipapo_avx2_type,
4029 #endif
4030         &nft_set_pipapo_type,
4031 };
4032
4033 #define NFT_SET_FEATURES        (NFT_SET_INTERVAL | NFT_SET_MAP | \
4034                                  NFT_SET_TIMEOUT | NFT_SET_OBJECT | \
4035                                  NFT_SET_EVAL)
4036
4037 static bool nft_set_ops_candidate(const struct nft_set_type *type, u32 flags)
4038 {
4039         return (flags & type->features) == (flags & NFT_SET_FEATURES);
4040 }
4041
4042 /*
4043  * Select a set implementation based on the data characteristics and the
4044  * given policy. The total memory use might not be known if no size is
4045  * given, in that case the amount of memory per element is used.
4046  */
4047 static const struct nft_set_ops *
4048 nft_select_set_ops(const struct nft_ctx *ctx,
4049                    const struct nlattr * const nla[],
4050                    const struct nft_set_desc *desc)
4051 {
4052         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
4053         const struct nft_set_ops *ops, *bops;
4054         struct nft_set_estimate est, best;
4055         const struct nft_set_type *type;
4056         u32 flags = 0;
4057         int i;
4058
4059         lockdep_assert_held(&nft_net->commit_mutex);
4060         lockdep_nfnl_nft_mutex_not_held();
4061
4062         if (nla[NFTA_SET_FLAGS] != NULL)
4063                 flags = ntohl(nla_get_be32(nla[NFTA_SET_FLAGS]));
4064
4065         bops        = NULL;
4066         best.size   = ~0;
4067         best.lookup = ~0;
4068         best.space  = ~0;
4069
4070         for (i = 0; i < ARRAY_SIZE(nft_set_types); i++) {
4071                 type = nft_set_types[i];
4072                 ops = &type->ops;
4073
4074                 if (!nft_set_ops_candidate(type, flags))
4075                         continue;
4076                 if (!ops->estimate(desc, flags, &est))
4077                         continue;
4078
4079                 switch (desc->policy) {
4080                 case NFT_SET_POL_PERFORMANCE:
4081                         if (est.lookup < best.lookup)
4082                                 break;
4083                         if (est.lookup == best.lookup &&
4084                             est.space < best.space)
4085                                 break;
4086                         continue;
4087                 case NFT_SET_POL_MEMORY:
4088                         if (!desc->size) {
4089                                 if (est.space < best.space)
4090                                         break;
4091                                 if (est.space == best.space &&
4092                                     est.lookup < best.lookup)
4093                                         break;
4094                         } else if (est.size < best.size || !bops) {
4095                                 break;
4096                         }
4097                         continue;
4098                 default:
4099                         break;
4100                 }
4101
4102                 bops = ops;
4103                 best = est;
4104         }
4105
4106         if (bops != NULL)
4107                 return bops;
4108
4109         return ERR_PTR(-EOPNOTSUPP);
4110 }
4111
4112 static const struct nla_policy nft_set_policy[NFTA_SET_MAX + 1] = {
4113         [NFTA_SET_TABLE]                = { .type = NLA_STRING,
4114                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
4115         [NFTA_SET_NAME]                 = { .type = NLA_STRING,
4116                                             .len = NFT_SET_MAXNAMELEN - 1 },
4117         [NFTA_SET_FLAGS]                = { .type = NLA_U32 },
4118         [NFTA_SET_KEY_TYPE]             = { .type = NLA_U32 },
4119         [NFTA_SET_KEY_LEN]              = { .type = NLA_U32 },
4120         [NFTA_SET_DATA_TYPE]            = { .type = NLA_U32 },
4121         [NFTA_SET_DATA_LEN]             = { .type = NLA_U32 },
4122         [NFTA_SET_POLICY]               = { .type = NLA_U32 },
4123         [NFTA_SET_DESC]                 = { .type = NLA_NESTED },
4124         [NFTA_SET_ID]                   = { .type = NLA_U32 },
4125         [NFTA_SET_TIMEOUT]              = { .type = NLA_U64 },
4126         [NFTA_SET_GC_INTERVAL]          = { .type = NLA_U32 },
4127         [NFTA_SET_USERDATA]             = { .type = NLA_BINARY,
4128                                             .len  = NFT_USERDATA_MAXLEN },
4129         [NFTA_SET_OBJ_TYPE]             = { .type = NLA_U32 },
4130         [NFTA_SET_HANDLE]               = { .type = NLA_U64 },
4131         [NFTA_SET_EXPR]                 = { .type = NLA_NESTED },
4132         [NFTA_SET_EXPRESSIONS]          = { .type = NLA_NESTED },
4133 };
4134
4135 static const struct nla_policy nft_set_desc_policy[NFTA_SET_DESC_MAX + 1] = {
4136         [NFTA_SET_DESC_SIZE]            = { .type = NLA_U32 },
4137         [NFTA_SET_DESC_CONCAT]          = { .type = NLA_NESTED },
4138 };
4139
4140 static struct nft_set *nft_set_lookup(const struct nft_table *table,
4141                                       const struct nlattr *nla, u8 genmask)
4142 {
4143         struct nft_set *set;
4144
4145         if (nla == NULL)
4146                 return ERR_PTR(-EINVAL);
4147
4148         list_for_each_entry_rcu(set, &table->sets, list) {
4149                 if (!nla_strcmp(nla, set->name) &&
4150                     nft_active_genmask(set, genmask))
4151                         return set;
4152         }
4153         return ERR_PTR(-ENOENT);
4154 }
4155
4156 static struct nft_set *nft_set_lookup_byhandle(const struct nft_table *table,
4157                                                const struct nlattr *nla,
4158                                                u8 genmask)
4159 {
4160         struct nft_set *set;
4161
4162         list_for_each_entry(set, &table->sets, list) {
4163                 if (be64_to_cpu(nla_get_be64(nla)) == set->handle &&
4164                     nft_active_genmask(set, genmask))
4165                         return set;
4166         }
4167         return ERR_PTR(-ENOENT);
4168 }
4169
4170 static struct nft_set *nft_set_lookup_byid(const struct net *net,
4171                                            const struct nft_table *table,
4172                                            const struct nlattr *nla, u8 genmask)
4173 {
4174         struct nftables_pernet *nft_net = nft_pernet(net);
4175         u32 id = ntohl(nla_get_be32(nla));
4176         struct nft_trans *trans;
4177
4178         list_for_each_entry(trans, &nft_net->commit_list, list) {
4179                 if (trans->msg_type == NFT_MSG_NEWSET) {
4180                         struct nft_set *set = nft_trans_set(trans);
4181
4182                         if (id == nft_trans_set_id(trans) &&
4183                             set->table == table &&
4184                             nft_active_genmask(set, genmask))
4185                                 return set;
4186                 }
4187         }
4188         return ERR_PTR(-ENOENT);
4189 }
4190
4191 struct nft_set *nft_set_lookup_global(const struct net *net,
4192                                       const struct nft_table *table,
4193                                       const struct nlattr *nla_set_name,
4194                                       const struct nlattr *nla_set_id,
4195                                       u8 genmask)
4196 {
4197         struct nft_set *set;
4198
4199         set = nft_set_lookup(table, nla_set_name, genmask);
4200         if (IS_ERR(set)) {
4201                 if (!nla_set_id)
4202                         return set;
4203
4204                 set = nft_set_lookup_byid(net, table, nla_set_id, genmask);
4205         }
4206         return set;
4207 }
4208 EXPORT_SYMBOL_GPL(nft_set_lookup_global);
4209
4210 static int nf_tables_set_alloc_name(struct nft_ctx *ctx, struct nft_set *set,
4211                                     const char *name)
4212 {
4213         const struct nft_set *i;
4214         const char *p;
4215         unsigned long *inuse;
4216         unsigned int n = 0, min = 0;
4217
4218         p = strchr(name, '%');
4219         if (p != NULL) {
4220                 if (p[1] != 'd' || strchr(p + 2, '%'))
4221                         return -EINVAL;
4222
4223                 inuse = (unsigned long *)get_zeroed_page(GFP_KERNEL);
4224                 if (inuse == NULL)
4225                         return -ENOMEM;
4226 cont:
4227                 list_for_each_entry(i, &ctx->table->sets, list) {
4228                         int tmp;
4229
4230                         if (!nft_is_active_next(ctx->net, i))
4231                                 continue;
4232                         if (!sscanf(i->name, name, &tmp))
4233                                 continue;
4234                         if (tmp < min || tmp >= min + BITS_PER_BYTE * PAGE_SIZE)
4235                                 continue;
4236
4237                         set_bit(tmp - min, inuse);
4238                 }
4239
4240                 n = find_first_zero_bit(inuse, BITS_PER_BYTE * PAGE_SIZE);
4241                 if (n >= BITS_PER_BYTE * PAGE_SIZE) {
4242                         min += BITS_PER_BYTE * PAGE_SIZE;
4243                         memset(inuse, 0, PAGE_SIZE);
4244                         goto cont;
4245                 }
4246                 free_page((unsigned long)inuse);
4247         }
4248
4249         set->name = kasprintf(GFP_KERNEL_ACCOUNT, name, min + n);
4250         if (!set->name)
4251                 return -ENOMEM;
4252
4253         list_for_each_entry(i, &ctx->table->sets, list) {
4254                 if (!nft_is_active_next(ctx->net, i))
4255                         continue;
4256                 if (!strcmp(set->name, i->name)) {
4257                         kfree(set->name);
4258                         set->name = NULL;
4259                         return -ENFILE;
4260                 }
4261         }
4262         return 0;
4263 }
4264
4265 int nf_msecs_to_jiffies64(const struct nlattr *nla, u64 *result)
4266 {
4267         u64 ms = be64_to_cpu(nla_get_be64(nla));
4268         u64 max = (u64)(~((u64)0));
4269
4270         max = div_u64(max, NSEC_PER_MSEC);
4271         if (ms >= max)
4272                 return -ERANGE;
4273
4274         ms *= NSEC_PER_MSEC;
4275         *result = nsecs_to_jiffies64(ms);
4276         return 0;
4277 }
4278
4279 __be64 nf_jiffies64_to_msecs(u64 input)
4280 {
4281         return cpu_to_be64(jiffies64_to_msecs(input));
4282 }
4283
4284 static int nf_tables_fill_set_concat(struct sk_buff *skb,
4285                                      const struct nft_set *set)
4286 {
4287         struct nlattr *concat, *field;
4288         int i;
4289
4290         concat = nla_nest_start_noflag(skb, NFTA_SET_DESC_CONCAT);
4291         if (!concat)
4292                 return -ENOMEM;
4293
4294         for (i = 0; i < set->field_count; i++) {
4295                 field = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
4296                 if (!field)
4297                         return -ENOMEM;
4298
4299                 if (nla_put_be32(skb, NFTA_SET_FIELD_LEN,
4300                                  htonl(set->field_len[i])))
4301                         return -ENOMEM;
4302
4303                 nla_nest_end(skb, field);
4304         }
4305
4306         nla_nest_end(skb, concat);
4307
4308         return 0;
4309 }
4310
4311 static int nf_tables_fill_set(struct sk_buff *skb, const struct nft_ctx *ctx,
4312                               const struct nft_set *set, u16 event, u16 flags)
4313 {
4314         u64 timeout = READ_ONCE(set->timeout);
4315         u32 gc_int = READ_ONCE(set->gc_int);
4316         u32 portid = ctx->portid;
4317         struct nlmsghdr *nlh;
4318         struct nlattr *nest;
4319         u32 seq = ctx->seq;
4320         int i;
4321
4322         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
4323         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
4324                            NFNETLINK_V0, nft_base_seq(ctx->net));
4325         if (!nlh)
4326                 goto nla_put_failure;
4327
4328         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
4329                 goto nla_put_failure;
4330         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
4331                 goto nla_put_failure;
4332         if (nla_put_be64(skb, NFTA_SET_HANDLE, cpu_to_be64(set->handle),
4333                          NFTA_SET_PAD))
4334                 goto nla_put_failure;
4335
4336         if (event == NFT_MSG_DELSET) {
4337                 nlmsg_end(skb, nlh);
4338                 return 0;
4339         }
4340
4341         if (set->flags != 0)
4342                 if (nla_put_be32(skb, NFTA_SET_FLAGS, htonl(set->flags)))
4343                         goto nla_put_failure;
4344
4345         if (nla_put_be32(skb, NFTA_SET_KEY_TYPE, htonl(set->ktype)))
4346                 goto nla_put_failure;
4347         if (nla_put_be32(skb, NFTA_SET_KEY_LEN, htonl(set->klen)))
4348                 goto nla_put_failure;
4349         if (set->flags & NFT_SET_MAP) {
4350                 if (nla_put_be32(skb, NFTA_SET_DATA_TYPE, htonl(set->dtype)))
4351                         goto nla_put_failure;
4352                 if (nla_put_be32(skb, NFTA_SET_DATA_LEN, htonl(set->dlen)))
4353                         goto nla_put_failure;
4354         }
4355         if (set->flags & NFT_SET_OBJECT &&
4356             nla_put_be32(skb, NFTA_SET_OBJ_TYPE, htonl(set->objtype)))
4357                 goto nla_put_failure;
4358
4359         if (timeout &&
4360             nla_put_be64(skb, NFTA_SET_TIMEOUT,
4361                          nf_jiffies64_to_msecs(timeout),
4362                          NFTA_SET_PAD))
4363                 goto nla_put_failure;
4364         if (gc_int &&
4365             nla_put_be32(skb, NFTA_SET_GC_INTERVAL, htonl(gc_int)))
4366                 goto nla_put_failure;
4367
4368         if (set->policy != NFT_SET_POL_PERFORMANCE) {
4369                 if (nla_put_be32(skb, NFTA_SET_POLICY, htonl(set->policy)))
4370                         goto nla_put_failure;
4371         }
4372
4373         if (set->udata &&
4374             nla_put(skb, NFTA_SET_USERDATA, set->udlen, set->udata))
4375                 goto nla_put_failure;
4376
4377         nest = nla_nest_start_noflag(skb, NFTA_SET_DESC);
4378         if (!nest)
4379                 goto nla_put_failure;
4380         if (set->size &&
4381             nla_put_be32(skb, NFTA_SET_DESC_SIZE, htonl(set->size)))
4382                 goto nla_put_failure;
4383
4384         if (set->field_count > 1 &&
4385             nf_tables_fill_set_concat(skb, set))
4386                 goto nla_put_failure;
4387
4388         nla_nest_end(skb, nest);
4389
4390         if (set->num_exprs == 1) {
4391                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPR);
4392                 if (nf_tables_fill_expr_info(skb, set->exprs[0], false) < 0)
4393                         goto nla_put_failure;
4394
4395                 nla_nest_end(skb, nest);
4396         } else if (set->num_exprs > 1) {
4397                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPRESSIONS);
4398                 if (nest == NULL)
4399                         goto nla_put_failure;
4400
4401                 for (i = 0; i < set->num_exprs; i++) {
4402                         if (nft_expr_dump(skb, NFTA_LIST_ELEM,
4403                                           set->exprs[i], false) < 0)
4404                                 goto nla_put_failure;
4405                 }
4406                 nla_nest_end(skb, nest);
4407         }
4408
4409         nlmsg_end(skb, nlh);
4410         return 0;
4411
4412 nla_put_failure:
4413         nlmsg_trim(skb, nlh);
4414         return -1;
4415 }
4416
4417 static void nf_tables_set_notify(const struct nft_ctx *ctx,
4418                                  const struct nft_set *set, int event,
4419                                  gfp_t gfp_flags)
4420 {
4421         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
4422         u32 portid = ctx->portid;
4423         struct sk_buff *skb;
4424         u16 flags = 0;
4425         int err;
4426
4427         if (!ctx->report &&
4428             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
4429                 return;
4430
4431         skb = nlmsg_new(NLMSG_GOODSIZE, gfp_flags);
4432         if (skb == NULL)
4433                 goto err;
4434
4435         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
4436                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
4437
4438         err = nf_tables_fill_set(skb, ctx, set, event, flags);
4439         if (err < 0) {
4440                 kfree_skb(skb);
4441                 goto err;
4442         }
4443
4444         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
4445         return;
4446 err:
4447         nfnetlink_set_err(ctx->net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
4448 }
4449
4450 static int nf_tables_dump_sets(struct sk_buff *skb, struct netlink_callback *cb)
4451 {
4452         const struct nft_set *set;
4453         unsigned int idx, s_idx = cb->args[0];
4454         struct nft_table *table, *cur_table = (struct nft_table *)cb->args[2];
4455         struct net *net = sock_net(skb->sk);
4456         struct nft_ctx *ctx = cb->data, ctx_set;
4457         struct nftables_pernet *nft_net;
4458
4459         if (cb->args[1])
4460                 return skb->len;
4461
4462         rcu_read_lock();
4463         nft_net = nft_pernet(net);
4464         cb->seq = READ_ONCE(nft_net->base_seq);
4465
4466         list_for_each_entry_rcu(table, &nft_net->tables, list) {
4467                 if (ctx->family != NFPROTO_UNSPEC &&
4468                     ctx->family != table->family)
4469                         continue;
4470
4471                 if (ctx->table && ctx->table != table)
4472                         continue;
4473
4474                 if (cur_table) {
4475                         if (cur_table != table)
4476                                 continue;
4477
4478                         cur_table = NULL;
4479                 }
4480                 idx = 0;
4481                 list_for_each_entry_rcu(set, &table->sets, list) {
4482                         if (idx < s_idx)
4483                                 goto cont;
4484                         if (!nft_is_active(net, set))
4485                                 goto cont;
4486
4487                         ctx_set = *ctx;
4488                         ctx_set.table = table;
4489                         ctx_set.family = table->family;
4490
4491                         if (nf_tables_fill_set(skb, &ctx_set, set,
4492                                                NFT_MSG_NEWSET,
4493                                                NLM_F_MULTI) < 0) {
4494                                 cb->args[0] = idx;
4495                                 cb->args[2] = (unsigned long) table;
4496                                 goto done;
4497                         }
4498                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
4499 cont:
4500                         idx++;
4501                 }
4502                 if (s_idx)
4503                         s_idx = 0;
4504         }
4505         cb->args[1] = 1;
4506 done:
4507         rcu_read_unlock();
4508         return skb->len;
4509 }
4510
4511 static int nf_tables_dump_sets_start(struct netlink_callback *cb)
4512 {
4513         struct nft_ctx *ctx_dump = NULL;
4514
4515         ctx_dump = kmemdup(cb->data, sizeof(*ctx_dump), GFP_ATOMIC);
4516         if (ctx_dump == NULL)
4517                 return -ENOMEM;
4518
4519         cb->data = ctx_dump;
4520         return 0;
4521 }
4522
4523 static int nf_tables_dump_sets_done(struct netlink_callback *cb)
4524 {
4525         kfree(cb->data);
4526         return 0;
4527 }
4528
4529 /* called with rcu_read_lock held */
4530 static int nf_tables_getset(struct sk_buff *skb, const struct nfnl_info *info,
4531                             const struct nlattr * const nla[])
4532 {
4533         struct netlink_ext_ack *extack = info->extack;
4534         u8 genmask = nft_genmask_cur(info->net);
4535         u8 family = info->nfmsg->nfgen_family;
4536         struct nft_table *table = NULL;
4537         struct net *net = info->net;
4538         const struct nft_set *set;
4539         struct sk_buff *skb2;
4540         struct nft_ctx ctx;
4541         int err;
4542
4543         if (nla[NFTA_SET_TABLE]) {
4544                 table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
4545                                          genmask, 0);
4546                 if (IS_ERR(table)) {
4547                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
4548                         return PTR_ERR(table);
4549                 }
4550         }
4551
4552         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
4553
4554         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
4555                 struct netlink_dump_control c = {
4556                         .start = nf_tables_dump_sets_start,
4557                         .dump = nf_tables_dump_sets,
4558                         .done = nf_tables_dump_sets_done,
4559                         .data = &ctx,
4560                         .module = THIS_MODULE,
4561                 };
4562
4563                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
4564         }
4565
4566         /* Only accept unspec with dump */
4567         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
4568                 return -EAFNOSUPPORT;
4569         if (!nla[NFTA_SET_TABLE])
4570                 return -EINVAL;
4571
4572         set = nft_set_lookup(table, nla[NFTA_SET_NAME], genmask);
4573         if (IS_ERR(set))
4574                 return PTR_ERR(set);
4575
4576         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
4577         if (skb2 == NULL)
4578                 return -ENOMEM;
4579
4580         err = nf_tables_fill_set(skb2, &ctx, set, NFT_MSG_NEWSET, 0);
4581         if (err < 0)
4582                 goto err_fill_set_info;
4583
4584         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
4585
4586 err_fill_set_info:
4587         kfree_skb(skb2);
4588         return err;
4589 }
4590
4591 static const struct nla_policy nft_concat_policy[NFTA_SET_FIELD_MAX + 1] = {
4592         [NFTA_SET_FIELD_LEN]    = { .type = NLA_U32 },
4593 };
4594
4595 static int nft_set_desc_concat_parse(const struct nlattr *attr,
4596                                      struct nft_set_desc *desc)
4597 {
4598         struct nlattr *tb[NFTA_SET_FIELD_MAX + 1];
4599         u32 len;
4600         int err;
4601
4602         if (desc->field_count >= ARRAY_SIZE(desc->field_len))
4603                 return -E2BIG;
4604
4605         err = nla_parse_nested_deprecated(tb, NFTA_SET_FIELD_MAX, attr,
4606                                           nft_concat_policy, NULL);
4607         if (err < 0)
4608                 return err;
4609
4610         if (!tb[NFTA_SET_FIELD_LEN])
4611                 return -EINVAL;
4612
4613         len = ntohl(nla_get_be32(tb[NFTA_SET_FIELD_LEN]));
4614         if (!len || len > U8_MAX)
4615                 return -EINVAL;
4616
4617         desc->field_len[desc->field_count++] = len;
4618
4619         return 0;
4620 }
4621
4622 static int nft_set_desc_concat(struct nft_set_desc *desc,
4623                                const struct nlattr *nla)
4624 {
4625         struct nlattr *attr;
4626         u32 num_regs = 0;
4627         int rem, err, i;
4628
4629         nla_for_each_nested(attr, nla, rem) {
4630                 if (nla_type(attr) != NFTA_LIST_ELEM)
4631                         return -EINVAL;
4632
4633                 err = nft_set_desc_concat_parse(attr, desc);
4634                 if (err < 0)
4635                         return err;
4636         }
4637
4638         for (i = 0; i < desc->field_count; i++)
4639                 num_regs += DIV_ROUND_UP(desc->field_len[i], sizeof(u32));
4640
4641         if (num_regs > NFT_REG32_COUNT)
4642                 return -E2BIG;
4643
4644         return 0;
4645 }
4646
4647 static int nf_tables_set_desc_parse(struct nft_set_desc *desc,
4648                                     const struct nlattr *nla)
4649 {
4650         struct nlattr *da[NFTA_SET_DESC_MAX + 1];
4651         int err;
4652
4653         err = nla_parse_nested_deprecated(da, NFTA_SET_DESC_MAX, nla,
4654                                           nft_set_desc_policy, NULL);
4655         if (err < 0)
4656                 return err;
4657
4658         if (da[NFTA_SET_DESC_SIZE] != NULL)
4659                 desc->size = ntohl(nla_get_be32(da[NFTA_SET_DESC_SIZE]));
4660         if (da[NFTA_SET_DESC_CONCAT])
4661                 err = nft_set_desc_concat(desc, da[NFTA_SET_DESC_CONCAT]);
4662
4663         return err;
4664 }
4665
4666 static int nft_set_expr_alloc(struct nft_ctx *ctx, struct nft_set *set,
4667                               const struct nlattr * const *nla,
4668                               struct nft_expr **exprs, int *num_exprs,
4669                               u32 flags)
4670 {
4671         struct nft_expr *expr;
4672         int err, i;
4673
4674         if (nla[NFTA_SET_EXPR]) {
4675                 expr = nft_set_elem_expr_alloc(ctx, set, nla[NFTA_SET_EXPR]);
4676                 if (IS_ERR(expr)) {
4677                         err = PTR_ERR(expr);
4678                         goto err_set_expr_alloc;
4679                 }
4680                 exprs[0] = expr;
4681                 (*num_exprs)++;
4682         } else if (nla[NFTA_SET_EXPRESSIONS]) {
4683                 struct nlattr *tmp;
4684                 int left;
4685
4686                 if (!(flags & NFT_SET_EXPR)) {
4687                         err = -EINVAL;
4688                         goto err_set_expr_alloc;
4689                 }
4690                 i = 0;
4691                 nla_for_each_nested(tmp, nla[NFTA_SET_EXPRESSIONS], left) {
4692                         if (i == NFT_SET_EXPR_MAX) {
4693                                 err = -E2BIG;
4694                                 goto err_set_expr_alloc;
4695                         }
4696                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
4697                                 err = -EINVAL;
4698                                 goto err_set_expr_alloc;
4699                         }
4700                         expr = nft_set_elem_expr_alloc(ctx, set, tmp);
4701                         if (IS_ERR(expr)) {
4702                                 err = PTR_ERR(expr);
4703                                 goto err_set_expr_alloc;
4704                         }
4705                         exprs[i++] = expr;
4706                         (*num_exprs)++;
4707                 }
4708         }
4709
4710         return 0;
4711
4712 err_set_expr_alloc:
4713         for (i = 0; i < *num_exprs; i++)
4714                 nft_expr_destroy(ctx, exprs[i]);
4715
4716         return err;
4717 }
4718
4719 static bool nft_set_is_same(const struct nft_set *set,
4720                             const struct nft_set_desc *desc,
4721                             struct nft_expr *exprs[], u32 num_exprs, u32 flags)
4722 {
4723         int i;
4724
4725         if (set->ktype != desc->ktype ||
4726             set->dtype != desc->dtype ||
4727             set->flags != flags ||
4728             set->klen != desc->klen ||
4729             set->dlen != desc->dlen ||
4730             set->field_count != desc->field_count ||
4731             set->num_exprs != num_exprs)
4732                 return false;
4733
4734         for (i = 0; i < desc->field_count; i++) {
4735                 if (set->field_len[i] != desc->field_len[i])
4736                         return false;
4737         }
4738
4739         for (i = 0; i < num_exprs; i++) {
4740                 if (set->exprs[i]->ops != exprs[i]->ops)
4741                         return false;
4742         }
4743
4744         return true;
4745 }
4746
4747 static int nf_tables_newset(struct sk_buff *skb, const struct nfnl_info *info,
4748                             const struct nlattr * const nla[])
4749 {
4750         struct netlink_ext_ack *extack = info->extack;
4751         u8 genmask = nft_genmask_next(info->net);
4752         u8 family = info->nfmsg->nfgen_family;
4753         const struct nft_set_ops *ops;
4754         struct net *net = info->net;
4755         struct nft_set_desc desc;
4756         struct nft_table *table;
4757         unsigned char *udata;
4758         struct nft_set *set;
4759         struct nft_ctx ctx;
4760         size_t alloc_size;
4761         int num_exprs = 0;
4762         char *name;
4763         int err, i;
4764         u16 udlen;
4765         u32 flags;
4766         u64 size;
4767
4768         if (nla[NFTA_SET_TABLE] == NULL ||
4769             nla[NFTA_SET_NAME] == NULL ||
4770             nla[NFTA_SET_KEY_LEN] == NULL ||
4771             nla[NFTA_SET_ID] == NULL)
4772                 return -EINVAL;
4773
4774         memset(&desc, 0, sizeof(desc));
4775
4776         desc.ktype = NFT_DATA_VALUE;
4777         if (nla[NFTA_SET_KEY_TYPE] != NULL) {
4778                 desc.ktype = ntohl(nla_get_be32(nla[NFTA_SET_KEY_TYPE]));
4779                 if ((desc.ktype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK)
4780                         return -EINVAL;
4781         }
4782
4783         desc.klen = ntohl(nla_get_be32(nla[NFTA_SET_KEY_LEN]));
4784         if (desc.klen == 0 || desc.klen > NFT_DATA_VALUE_MAXLEN)
4785                 return -EINVAL;
4786
4787         flags = 0;
4788         if (nla[NFTA_SET_FLAGS] != NULL) {
4789                 flags = ntohl(nla_get_be32(nla[NFTA_SET_FLAGS]));
4790                 if (flags & ~(NFT_SET_ANONYMOUS | NFT_SET_CONSTANT |
4791                               NFT_SET_INTERVAL | NFT_SET_TIMEOUT |
4792                               NFT_SET_MAP | NFT_SET_EVAL |
4793                               NFT_SET_OBJECT | NFT_SET_CONCAT | NFT_SET_EXPR))
4794                         return -EOPNOTSUPP;
4795                 /* Only one of these operations is supported */
4796                 if ((flags & (NFT_SET_MAP | NFT_SET_OBJECT)) ==
4797                              (NFT_SET_MAP | NFT_SET_OBJECT))
4798                         return -EOPNOTSUPP;
4799                 if ((flags & (NFT_SET_EVAL | NFT_SET_OBJECT)) ==
4800                              (NFT_SET_EVAL | NFT_SET_OBJECT))
4801                         return -EOPNOTSUPP;
4802         }
4803
4804         desc.dtype = 0;
4805         if (nla[NFTA_SET_DATA_TYPE] != NULL) {
4806                 if (!(flags & NFT_SET_MAP))
4807                         return -EINVAL;
4808
4809                 desc.dtype = ntohl(nla_get_be32(nla[NFTA_SET_DATA_TYPE]));
4810                 if ((desc.dtype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK &&
4811                     desc.dtype != NFT_DATA_VERDICT)
4812                         return -EINVAL;
4813
4814                 if (desc.dtype != NFT_DATA_VERDICT) {
4815                         if (nla[NFTA_SET_DATA_LEN] == NULL)
4816                                 return -EINVAL;
4817                         desc.dlen = ntohl(nla_get_be32(nla[NFTA_SET_DATA_LEN]));
4818                         if (desc.dlen == 0 || desc.dlen > NFT_DATA_VALUE_MAXLEN)
4819                                 return -EINVAL;
4820                 } else
4821                         desc.dlen = sizeof(struct nft_verdict);
4822         } else if (flags & NFT_SET_MAP)
4823                 return -EINVAL;
4824
4825         if (nla[NFTA_SET_OBJ_TYPE] != NULL) {
4826                 if (!(flags & NFT_SET_OBJECT))
4827                         return -EINVAL;
4828
4829                 desc.objtype = ntohl(nla_get_be32(nla[NFTA_SET_OBJ_TYPE]));
4830                 if (desc.objtype == NFT_OBJECT_UNSPEC ||
4831                     desc.objtype > NFT_OBJECT_MAX)
4832                         return -EOPNOTSUPP;
4833         } else if (flags & NFT_SET_OBJECT)
4834                 return -EINVAL;
4835         else
4836                 desc.objtype = NFT_OBJECT_UNSPEC;
4837
4838         desc.timeout = 0;
4839         if (nla[NFTA_SET_TIMEOUT] != NULL) {
4840                 if (!(flags & NFT_SET_TIMEOUT))
4841                         return -EINVAL;
4842
4843                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_TIMEOUT], &desc.timeout);
4844                 if (err)
4845                         return err;
4846         }
4847         desc.gc_int = 0;
4848         if (nla[NFTA_SET_GC_INTERVAL] != NULL) {
4849                 if (!(flags & NFT_SET_TIMEOUT))
4850                         return -EINVAL;
4851                 desc.gc_int = ntohl(nla_get_be32(nla[NFTA_SET_GC_INTERVAL]));
4852         }
4853
4854         desc.policy = NFT_SET_POL_PERFORMANCE;
4855         if (nla[NFTA_SET_POLICY] != NULL)
4856                 desc.policy = ntohl(nla_get_be32(nla[NFTA_SET_POLICY]));
4857
4858         if (nla[NFTA_SET_DESC] != NULL) {
4859                 err = nf_tables_set_desc_parse(&desc, nla[NFTA_SET_DESC]);
4860                 if (err < 0)
4861                         return err;
4862
4863                 if (desc.field_count > 1 && !(flags & NFT_SET_CONCAT))
4864                         return -EINVAL;
4865         } else if (flags & NFT_SET_CONCAT) {
4866                 return -EINVAL;
4867         }
4868
4869         if (nla[NFTA_SET_EXPR] || nla[NFTA_SET_EXPRESSIONS])
4870                 desc.expr = true;
4871
4872         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family, genmask,
4873                                  NETLINK_CB(skb).portid);
4874         if (IS_ERR(table)) {
4875                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
4876                 return PTR_ERR(table);
4877         }
4878
4879         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
4880
4881         set = nft_set_lookup(table, nla[NFTA_SET_NAME], genmask);
4882         if (IS_ERR(set)) {
4883                 if (PTR_ERR(set) != -ENOENT) {
4884                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
4885                         return PTR_ERR(set);
4886                 }
4887         } else {
4888                 struct nft_expr *exprs[NFT_SET_EXPR_MAX] = {};
4889
4890                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
4891                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
4892                         return -EEXIST;
4893                 }
4894                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
4895                         return -EOPNOTSUPP;
4896
4897                 err = nft_set_expr_alloc(&ctx, set, nla, exprs, &num_exprs, flags);
4898                 if (err < 0)
4899                         return err;
4900
4901                 err = 0;
4902                 if (!nft_set_is_same(set, &desc, exprs, num_exprs, flags)) {
4903                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
4904                         err = -EEXIST;
4905                 }
4906
4907                 for (i = 0; i < num_exprs; i++)
4908                         nft_expr_destroy(&ctx, exprs[i]);
4909
4910                 if (err < 0)
4911                         return err;
4912
4913                 return __nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set, &desc);
4914         }
4915
4916         if (!(info->nlh->nlmsg_flags & NLM_F_CREATE))
4917                 return -ENOENT;
4918
4919         ops = nft_select_set_ops(&ctx, nla, &desc);
4920         if (IS_ERR(ops))
4921                 return PTR_ERR(ops);
4922
4923         udlen = 0;
4924         if (nla[NFTA_SET_USERDATA])
4925                 udlen = nla_len(nla[NFTA_SET_USERDATA]);
4926
4927         size = 0;
4928         if (ops->privsize != NULL)
4929                 size = ops->privsize(nla, &desc);
4930         alloc_size = sizeof(*set) + size + udlen;
4931         if (alloc_size < size || alloc_size > INT_MAX)
4932                 return -ENOMEM;
4933         set = kvzalloc(alloc_size, GFP_KERNEL_ACCOUNT);
4934         if (!set)
4935                 return -ENOMEM;
4936
4937         name = nla_strdup(nla[NFTA_SET_NAME], GFP_KERNEL_ACCOUNT);
4938         if (!name) {
4939                 err = -ENOMEM;
4940                 goto err_set_name;
4941         }
4942
4943         err = nf_tables_set_alloc_name(&ctx, set, name);
4944         kfree(name);
4945         if (err < 0)
4946                 goto err_set_name;
4947
4948         udata = NULL;
4949         if (udlen) {
4950                 udata = set->data + size;
4951                 nla_memcpy(udata, nla[NFTA_SET_USERDATA], udlen);
4952         }
4953
4954         INIT_LIST_HEAD(&set->bindings);
4955         INIT_LIST_HEAD(&set->catchall_list);
4956         set->table = table;
4957         write_pnet(&set->net, net);
4958         set->ops = ops;
4959         set->ktype = desc.ktype;
4960         set->klen = desc.klen;
4961         set->dtype = desc.dtype;
4962         set->objtype = desc.objtype;
4963         set->dlen = desc.dlen;
4964         set->flags = flags;
4965         set->size = desc.size;
4966         set->policy = desc.policy;
4967         set->udlen = udlen;
4968         set->udata = udata;
4969         set->timeout = desc.timeout;
4970         set->gc_int = desc.gc_int;
4971
4972         set->field_count = desc.field_count;
4973         for (i = 0; i < desc.field_count; i++)
4974                 set->field_len[i] = desc.field_len[i];
4975
4976         err = ops->init(set, &desc, nla);
4977         if (err < 0)
4978                 goto err_set_init;
4979
4980         err = nft_set_expr_alloc(&ctx, set, nla, set->exprs, &num_exprs, flags);
4981         if (err < 0)
4982                 goto err_set_destroy;
4983
4984         set->num_exprs = num_exprs;
4985         set->handle = nf_tables_alloc_handle(table);
4986         INIT_LIST_HEAD(&set->pending_update);
4987
4988         err = nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set);
4989         if (err < 0)
4990                 goto err_set_expr_alloc;
4991
4992         list_add_tail_rcu(&set->list, &table->sets);
4993         table->use++;
4994         return 0;
4995
4996 err_set_expr_alloc:
4997         for (i = 0; i < set->num_exprs; i++)
4998                 nft_expr_destroy(&ctx, set->exprs[i]);
4999 err_set_destroy:
5000         ops->destroy(set);
5001 err_set_init:
5002         kfree(set->name);
5003 err_set_name:
5004         kvfree(set);
5005         return err;
5006 }
5007
5008 static void nft_set_catchall_destroy(const struct nft_ctx *ctx,
5009                                      struct nft_set *set)
5010 {
5011         struct nft_set_elem_catchall *next, *catchall;
5012
5013         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
5014                 list_del_rcu(&catchall->list);
5015                 nft_set_elem_destroy(set, catchall->elem, true);
5016                 kfree_rcu(catchall, rcu);
5017         }
5018 }
5019
5020 static void nft_set_destroy(const struct nft_ctx *ctx, struct nft_set *set)
5021 {
5022         int i;
5023
5024         if (WARN_ON(set->use > 0))
5025                 return;
5026
5027         for (i = 0; i < set->num_exprs; i++)
5028                 nft_expr_destroy(ctx, set->exprs[i]);
5029
5030         set->ops->destroy(set);
5031         nft_set_catchall_destroy(ctx, set);
5032         kfree(set->name);
5033         kvfree(set);
5034 }
5035
5036 static int nf_tables_delset(struct sk_buff *skb, const struct nfnl_info *info,
5037                             const struct nlattr * const nla[])
5038 {
5039         struct netlink_ext_ack *extack = info->extack;
5040         u8 genmask = nft_genmask_next(info->net);
5041         u8 family = info->nfmsg->nfgen_family;
5042         struct net *net = info->net;
5043         const struct nlattr *attr;
5044         struct nft_table *table;
5045         struct nft_set *set;
5046         struct nft_ctx ctx;
5047
5048         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
5049                 return -EAFNOSUPPORT;
5050
5051         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
5052                                  genmask, NETLINK_CB(skb).portid);
5053         if (IS_ERR(table)) {
5054                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
5055                 return PTR_ERR(table);
5056         }
5057
5058         if (nla[NFTA_SET_HANDLE]) {
5059                 attr = nla[NFTA_SET_HANDLE];
5060                 set = nft_set_lookup_byhandle(table, attr, genmask);
5061         } else {
5062                 attr = nla[NFTA_SET_NAME];
5063                 set = nft_set_lookup(table, attr, genmask);
5064         }
5065
5066         if (IS_ERR(set)) {
5067                 if (PTR_ERR(set) == -ENOENT &&
5068                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYSET)
5069                         return 0;
5070
5071                 NL_SET_BAD_ATTR(extack, attr);
5072                 return PTR_ERR(set);
5073         }
5074         if (set->use ||
5075             (info->nlh->nlmsg_flags & NLM_F_NONREC &&
5076              atomic_read(&set->nelems) > 0)) {
5077                 NL_SET_BAD_ATTR(extack, attr);
5078                 return -EBUSY;
5079         }
5080
5081         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
5082
5083         return nft_delset(&ctx, set);
5084 }
5085
5086 static int nft_validate_register_store(const struct nft_ctx *ctx,
5087                                        enum nft_registers reg,
5088                                        const struct nft_data *data,
5089                                        enum nft_data_types type,
5090                                        unsigned int len);
5091
5092 static int nft_setelem_data_validate(const struct nft_ctx *ctx,
5093                                      struct nft_set *set,
5094                                      struct nft_set_elem *elem)
5095 {
5096         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
5097         enum nft_registers dreg;
5098
5099         dreg = nft_type_to_reg(set->dtype);
5100         return nft_validate_register_store(ctx, dreg, nft_set_ext_data(ext),
5101                                            set->dtype == NFT_DATA_VERDICT ?
5102                                            NFT_DATA_VERDICT : NFT_DATA_VALUE,
5103                                            set->dlen);
5104 }
5105
5106 static int nf_tables_bind_check_setelem(const struct nft_ctx *ctx,
5107                                         struct nft_set *set,
5108                                         const struct nft_set_iter *iter,
5109                                         struct nft_set_elem *elem)
5110 {
5111         return nft_setelem_data_validate(ctx, set, elem);
5112 }
5113
5114 static int nft_set_catchall_bind_check(const struct nft_ctx *ctx,
5115                                        struct nft_set *set)
5116 {
5117         u8 genmask = nft_genmask_next(ctx->net);
5118         struct nft_set_elem_catchall *catchall;
5119         struct nft_set_elem elem;
5120         struct nft_set_ext *ext;
5121         int ret = 0;
5122
5123         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5124                 ext = nft_set_elem_ext(set, catchall->elem);
5125                 if (!nft_set_elem_active(ext, genmask))
5126                         continue;
5127
5128                 elem.priv = catchall->elem;
5129                 ret = nft_setelem_data_validate(ctx, set, &elem);
5130                 if (ret < 0)
5131                         break;
5132         }
5133
5134         return ret;
5135 }
5136
5137 int nf_tables_bind_set(const struct nft_ctx *ctx, struct nft_set *set,
5138                        struct nft_set_binding *binding)
5139 {
5140         struct nft_set_binding *i;
5141         struct nft_set_iter iter;
5142
5143         if (set->use == UINT_MAX)
5144                 return -EOVERFLOW;
5145
5146         if (!list_empty(&set->bindings) && nft_set_is_anonymous(set))
5147                 return -EBUSY;
5148
5149         if (binding->flags & NFT_SET_MAP) {
5150                 /* If the set is already bound to the same chain all
5151                  * jumps are already validated for that chain.
5152                  */
5153                 list_for_each_entry(i, &set->bindings, list) {
5154                         if (i->flags & NFT_SET_MAP &&
5155                             i->chain == binding->chain)
5156                                 goto bind;
5157                 }
5158
5159                 iter.genmask    = nft_genmask_next(ctx->net);
5160                 iter.skip       = 0;
5161                 iter.count      = 0;
5162                 iter.err        = 0;
5163                 iter.fn         = nf_tables_bind_check_setelem;
5164
5165                 set->ops->walk(ctx, set, &iter);
5166                 if (!iter.err)
5167                         iter.err = nft_set_catchall_bind_check(ctx, set);
5168
5169                 if (iter.err < 0)
5170                         return iter.err;
5171         }
5172 bind:
5173         binding->chain = ctx->chain;
5174         list_add_tail_rcu(&binding->list, &set->bindings);
5175         nft_set_trans_bind(ctx, set);
5176         set->use++;
5177
5178         return 0;
5179 }
5180 EXPORT_SYMBOL_GPL(nf_tables_bind_set);
5181
5182 static void nf_tables_unbind_set(const struct nft_ctx *ctx, struct nft_set *set,
5183                                  struct nft_set_binding *binding, bool event)
5184 {
5185         list_del_rcu(&binding->list);
5186
5187         if (list_empty(&set->bindings) && nft_set_is_anonymous(set)) {
5188                 list_del_rcu(&set->list);
5189                 if (event)
5190                         nf_tables_set_notify(ctx, set, NFT_MSG_DELSET,
5191                                              GFP_KERNEL);
5192         }
5193 }
5194
5195 void nf_tables_activate_set(const struct nft_ctx *ctx, struct nft_set *set)
5196 {
5197         if (nft_set_is_anonymous(set))
5198                 nft_clear(ctx->net, set);
5199
5200         set->use++;
5201 }
5202 EXPORT_SYMBOL_GPL(nf_tables_activate_set);
5203
5204 void nf_tables_deactivate_set(const struct nft_ctx *ctx, struct nft_set *set,
5205                               struct nft_set_binding *binding,
5206                               enum nft_trans_phase phase)
5207 {
5208         switch (phase) {
5209         case NFT_TRANS_PREPARE_ERROR:
5210                 nft_set_trans_unbind(ctx, set);
5211                 if (nft_set_is_anonymous(set))
5212                         nft_deactivate_next(ctx->net, set);
5213
5214                 set->use--;
5215                 break;
5216         case NFT_TRANS_PREPARE:
5217                 if (nft_set_is_anonymous(set))
5218                         nft_deactivate_next(ctx->net, set);
5219
5220                 set->use--;
5221                 return;
5222         case NFT_TRANS_ABORT:
5223         case NFT_TRANS_RELEASE:
5224                 set->use--;
5225                 fallthrough;
5226         default:
5227                 nf_tables_unbind_set(ctx, set, binding,
5228                                      phase == NFT_TRANS_COMMIT);
5229         }
5230 }
5231 EXPORT_SYMBOL_GPL(nf_tables_deactivate_set);
5232
5233 void nf_tables_destroy_set(const struct nft_ctx *ctx, struct nft_set *set)
5234 {
5235         if (list_empty(&set->bindings) && nft_set_is_anonymous(set))
5236                 nft_set_destroy(ctx, set);
5237 }
5238 EXPORT_SYMBOL_GPL(nf_tables_destroy_set);
5239
5240 const struct nft_set_ext_type nft_set_ext_types[] = {
5241         [NFT_SET_EXT_KEY]               = {
5242                 .align  = __alignof__(u32),
5243         },
5244         [NFT_SET_EXT_DATA]              = {
5245                 .align  = __alignof__(u32),
5246         },
5247         [NFT_SET_EXT_EXPRESSIONS]       = {
5248                 .align  = __alignof__(struct nft_set_elem_expr),
5249         },
5250         [NFT_SET_EXT_OBJREF]            = {
5251                 .len    = sizeof(struct nft_object *),
5252                 .align  = __alignof__(struct nft_object *),
5253         },
5254         [NFT_SET_EXT_FLAGS]             = {
5255                 .len    = sizeof(u8),
5256                 .align  = __alignof__(u8),
5257         },
5258         [NFT_SET_EXT_TIMEOUT]           = {
5259                 .len    = sizeof(u64),
5260                 .align  = __alignof__(u64),
5261         },
5262         [NFT_SET_EXT_EXPIRATION]        = {
5263                 .len    = sizeof(u64),
5264                 .align  = __alignof__(u64),
5265         },
5266         [NFT_SET_EXT_USERDATA]          = {
5267                 .len    = sizeof(struct nft_userdata),
5268                 .align  = __alignof__(struct nft_userdata),
5269         },
5270         [NFT_SET_EXT_KEY_END]           = {
5271                 .align  = __alignof__(u32),
5272         },
5273 };
5274
5275 /*
5276  * Set elements
5277  */
5278
5279 static const struct nla_policy nft_set_elem_policy[NFTA_SET_ELEM_MAX + 1] = {
5280         [NFTA_SET_ELEM_KEY]             = { .type = NLA_NESTED },
5281         [NFTA_SET_ELEM_DATA]            = { .type = NLA_NESTED },
5282         [NFTA_SET_ELEM_FLAGS]           = { .type = NLA_U32 },
5283         [NFTA_SET_ELEM_TIMEOUT]         = { .type = NLA_U64 },
5284         [NFTA_SET_ELEM_EXPIRATION]      = { .type = NLA_U64 },
5285         [NFTA_SET_ELEM_USERDATA]        = { .type = NLA_BINARY,
5286                                             .len = NFT_USERDATA_MAXLEN },
5287         [NFTA_SET_ELEM_EXPR]            = { .type = NLA_NESTED },
5288         [NFTA_SET_ELEM_OBJREF]          = { .type = NLA_STRING,
5289                                             .len = NFT_OBJ_MAXNAMELEN - 1 },
5290         [NFTA_SET_ELEM_KEY_END]         = { .type = NLA_NESTED },
5291         [NFTA_SET_ELEM_EXPRESSIONS]     = { .type = NLA_NESTED },
5292 };
5293
5294 static const struct nla_policy nft_set_elem_list_policy[NFTA_SET_ELEM_LIST_MAX + 1] = {
5295         [NFTA_SET_ELEM_LIST_TABLE]      = { .type = NLA_STRING,
5296                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
5297         [NFTA_SET_ELEM_LIST_SET]        = { .type = NLA_STRING,
5298                                             .len = NFT_SET_MAXNAMELEN - 1 },
5299         [NFTA_SET_ELEM_LIST_ELEMENTS]   = { .type = NLA_NESTED },
5300         [NFTA_SET_ELEM_LIST_SET_ID]     = { .type = NLA_U32 },
5301 };
5302
5303 static int nft_set_elem_expr_dump(struct sk_buff *skb,
5304                                   const struct nft_set *set,
5305                                   const struct nft_set_ext *ext)
5306 {
5307         struct nft_set_elem_expr *elem_expr;
5308         u32 size, num_exprs = 0;
5309         struct nft_expr *expr;
5310         struct nlattr *nest;
5311
5312         elem_expr = nft_set_ext_expr(ext);
5313         nft_setelem_expr_foreach(expr, elem_expr, size)
5314                 num_exprs++;
5315
5316         if (num_exprs == 1) {
5317                 expr = nft_setelem_expr_at(elem_expr, 0);
5318                 if (nft_expr_dump(skb, NFTA_SET_ELEM_EXPR, expr, false) < 0)
5319                         return -1;
5320
5321                 return 0;
5322         } else if (num_exprs > 1) {
5323                 nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_EXPRESSIONS);
5324                 if (nest == NULL)
5325                         goto nla_put_failure;
5326
5327                 nft_setelem_expr_foreach(expr, elem_expr, size) {
5328                         expr = nft_setelem_expr_at(elem_expr, size);
5329                         if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr, false) < 0)
5330                                 goto nla_put_failure;
5331                 }
5332                 nla_nest_end(skb, nest);
5333         }
5334         return 0;
5335
5336 nla_put_failure:
5337         return -1;
5338 }
5339
5340 static int nf_tables_fill_setelem(struct sk_buff *skb,
5341                                   const struct nft_set *set,
5342                                   const struct nft_set_elem *elem)
5343 {
5344         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
5345         unsigned char *b = skb_tail_pointer(skb);
5346         struct nlattr *nest;
5347
5348         nest = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
5349         if (nest == NULL)
5350                 goto nla_put_failure;
5351
5352         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
5353             nft_data_dump(skb, NFTA_SET_ELEM_KEY, nft_set_ext_key(ext),
5354                           NFT_DATA_VALUE, set->klen) < 0)
5355                 goto nla_put_failure;
5356
5357         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
5358             nft_data_dump(skb, NFTA_SET_ELEM_KEY_END, nft_set_ext_key_end(ext),
5359                           NFT_DATA_VALUE, set->klen) < 0)
5360                 goto nla_put_failure;
5361
5362         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
5363             nft_data_dump(skb, NFTA_SET_ELEM_DATA, nft_set_ext_data(ext),
5364                           set->dtype == NFT_DATA_VERDICT ? NFT_DATA_VERDICT : NFT_DATA_VALUE,
5365                           set->dlen) < 0)
5366                 goto nla_put_failure;
5367
5368         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS) &&
5369             nft_set_elem_expr_dump(skb, set, ext))
5370                 goto nla_put_failure;
5371
5372         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
5373             nla_put_string(skb, NFTA_SET_ELEM_OBJREF,
5374                            (*nft_set_ext_obj(ext))->key.name) < 0)
5375                 goto nla_put_failure;
5376
5377         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
5378             nla_put_be32(skb, NFTA_SET_ELEM_FLAGS,
5379                          htonl(*nft_set_ext_flags(ext))))
5380                 goto nla_put_failure;
5381
5382         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT) &&
5383             nla_put_be64(skb, NFTA_SET_ELEM_TIMEOUT,
5384                          nf_jiffies64_to_msecs(*nft_set_ext_timeout(ext)),
5385                          NFTA_SET_ELEM_PAD))
5386                 goto nla_put_failure;
5387
5388         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
5389                 u64 expires, now = get_jiffies_64();
5390
5391                 expires = *nft_set_ext_expiration(ext);
5392                 if (time_before64(now, expires))
5393                         expires -= now;
5394                 else
5395                         expires = 0;
5396
5397                 if (nla_put_be64(skb, NFTA_SET_ELEM_EXPIRATION,
5398                                  nf_jiffies64_to_msecs(expires),
5399                                  NFTA_SET_ELEM_PAD))
5400                         goto nla_put_failure;
5401         }
5402
5403         if (nft_set_ext_exists(ext, NFT_SET_EXT_USERDATA)) {
5404                 struct nft_userdata *udata;
5405
5406                 udata = nft_set_ext_userdata(ext);
5407                 if (nla_put(skb, NFTA_SET_ELEM_USERDATA,
5408                             udata->len + 1, udata->data))
5409                         goto nla_put_failure;
5410         }
5411
5412         nla_nest_end(skb, nest);
5413         return 0;
5414
5415 nla_put_failure:
5416         nlmsg_trim(skb, b);
5417         return -EMSGSIZE;
5418 }
5419
5420 struct nft_set_dump_args {
5421         const struct netlink_callback   *cb;
5422         struct nft_set_iter             iter;
5423         struct sk_buff                  *skb;
5424 };
5425
5426 static int nf_tables_dump_setelem(const struct nft_ctx *ctx,
5427                                   struct nft_set *set,
5428                                   const struct nft_set_iter *iter,
5429                                   struct nft_set_elem *elem)
5430 {
5431         struct nft_set_dump_args *args;
5432
5433         args = container_of(iter, struct nft_set_dump_args, iter);
5434         return nf_tables_fill_setelem(args->skb, set, elem);
5435 }
5436
5437 struct nft_set_dump_ctx {
5438         const struct nft_set    *set;
5439         struct nft_ctx          ctx;
5440 };
5441
5442 static int nft_set_catchall_dump(struct net *net, struct sk_buff *skb,
5443                                  const struct nft_set *set)
5444 {
5445         struct nft_set_elem_catchall *catchall;
5446         u8 genmask = nft_genmask_cur(net);
5447         struct nft_set_elem elem;
5448         struct nft_set_ext *ext;
5449         int ret = 0;
5450
5451         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5452                 ext = nft_set_elem_ext(set, catchall->elem);
5453                 if (!nft_set_elem_active(ext, genmask) ||
5454                     nft_set_elem_expired(ext))
5455                         continue;
5456
5457                 elem.priv = catchall->elem;
5458                 ret = nf_tables_fill_setelem(skb, set, &elem);
5459                 break;
5460         }
5461
5462         return ret;
5463 }
5464
5465 static int nf_tables_dump_set(struct sk_buff *skb, struct netlink_callback *cb)
5466 {
5467         struct nft_set_dump_ctx *dump_ctx = cb->data;
5468         struct net *net = sock_net(skb->sk);
5469         struct nftables_pernet *nft_net;
5470         struct nft_table *table;
5471         struct nft_set *set;
5472         struct nft_set_dump_args args;
5473         bool set_found = false;
5474         struct nlmsghdr *nlh;
5475         struct nlattr *nest;
5476         u32 portid, seq;
5477         int event;
5478
5479         rcu_read_lock();
5480         nft_net = nft_pernet(net);
5481         cb->seq = READ_ONCE(nft_net->base_seq);
5482
5483         list_for_each_entry_rcu(table, &nft_net->tables, list) {
5484                 if (dump_ctx->ctx.family != NFPROTO_UNSPEC &&
5485                     dump_ctx->ctx.family != table->family)
5486                         continue;
5487
5488                 if (table != dump_ctx->ctx.table)
5489                         continue;
5490
5491                 list_for_each_entry_rcu(set, &table->sets, list) {
5492                         if (set == dump_ctx->set) {
5493                                 set_found = true;
5494                                 break;
5495                         }
5496                 }
5497                 break;
5498         }
5499
5500         if (!set_found) {
5501                 rcu_read_unlock();
5502                 return -ENOENT;
5503         }
5504
5505         event  = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWSETELEM);
5506         portid = NETLINK_CB(cb->skb).portid;
5507         seq    = cb->nlh->nlmsg_seq;
5508
5509         nlh = nfnl_msg_put(skb, portid, seq, event, NLM_F_MULTI,
5510                            table->family, NFNETLINK_V0, nft_base_seq(net));
5511         if (!nlh)
5512                 goto nla_put_failure;
5513
5514         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_TABLE, table->name))
5515                 goto nla_put_failure;
5516         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_SET, set->name))
5517                 goto nla_put_failure;
5518
5519         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
5520         if (nest == NULL)
5521                 goto nla_put_failure;
5522
5523         args.cb                 = cb;
5524         args.skb                = skb;
5525         args.iter.genmask       = nft_genmask_cur(net);
5526         args.iter.skip          = cb->args[0];
5527         args.iter.count         = 0;
5528         args.iter.err           = 0;
5529         args.iter.fn            = nf_tables_dump_setelem;
5530         set->ops->walk(&dump_ctx->ctx, set, &args.iter);
5531
5532         if (!args.iter.err && args.iter.count == cb->args[0])
5533                 args.iter.err = nft_set_catchall_dump(net, skb, set);
5534         rcu_read_unlock();
5535
5536         nla_nest_end(skb, nest);
5537         nlmsg_end(skb, nlh);
5538
5539         if (args.iter.err && args.iter.err != -EMSGSIZE)
5540                 return args.iter.err;
5541         if (args.iter.count == cb->args[0])
5542                 return 0;
5543
5544         cb->args[0] = args.iter.count;
5545         return skb->len;
5546
5547 nla_put_failure:
5548         rcu_read_unlock();
5549         return -ENOSPC;
5550 }
5551
5552 static int nf_tables_dump_set_start(struct netlink_callback *cb)
5553 {
5554         struct nft_set_dump_ctx *dump_ctx = cb->data;
5555
5556         cb->data = kmemdup(dump_ctx, sizeof(*dump_ctx), GFP_ATOMIC);
5557
5558         return cb->data ? 0 : -ENOMEM;
5559 }
5560
5561 static int nf_tables_dump_set_done(struct netlink_callback *cb)
5562 {
5563         kfree(cb->data);
5564         return 0;
5565 }
5566
5567 static int nf_tables_fill_setelem_info(struct sk_buff *skb,
5568                                        const struct nft_ctx *ctx, u32 seq,
5569                                        u32 portid, int event, u16 flags,
5570                                        const struct nft_set *set,
5571                                        const struct nft_set_elem *elem)
5572 {
5573         struct nlmsghdr *nlh;
5574         struct nlattr *nest;
5575         int err;
5576
5577         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
5578         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
5579                            NFNETLINK_V0, nft_base_seq(ctx->net));
5580         if (!nlh)
5581                 goto nla_put_failure;
5582
5583         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
5584                 goto nla_put_failure;
5585         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
5586                 goto nla_put_failure;
5587
5588         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
5589         if (nest == NULL)
5590                 goto nla_put_failure;
5591
5592         err = nf_tables_fill_setelem(skb, set, elem);
5593         if (err < 0)
5594                 goto nla_put_failure;
5595
5596         nla_nest_end(skb, nest);
5597
5598         nlmsg_end(skb, nlh);
5599         return 0;
5600
5601 nla_put_failure:
5602         nlmsg_trim(skb, nlh);
5603         return -1;
5604 }
5605
5606 static int nft_setelem_parse_flags(const struct nft_set *set,
5607                                    const struct nlattr *attr, u32 *flags)
5608 {
5609         if (attr == NULL)
5610                 return 0;
5611
5612         *flags = ntohl(nla_get_be32(attr));
5613         if (*flags & ~(NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
5614                 return -EOPNOTSUPP;
5615         if (!(set->flags & NFT_SET_INTERVAL) &&
5616             *flags & NFT_SET_ELEM_INTERVAL_END)
5617                 return -EINVAL;
5618         if ((*flags & (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL)) ==
5619             (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
5620                 return -EINVAL;
5621
5622         return 0;
5623 }
5624
5625 static int nft_setelem_parse_key(struct nft_ctx *ctx, struct nft_set *set,
5626                                  struct nft_data *key, struct nlattr *attr)
5627 {
5628         struct nft_data_desc desc = {
5629                 .type   = NFT_DATA_VALUE,
5630                 .size   = NFT_DATA_VALUE_MAXLEN,
5631                 .len    = set->klen,
5632         };
5633
5634         return nft_data_init(ctx, key, &desc, attr);
5635 }
5636
5637 static int nft_setelem_parse_data(struct nft_ctx *ctx, struct nft_set *set,
5638                                   struct nft_data_desc *desc,
5639                                   struct nft_data *data,
5640                                   struct nlattr *attr)
5641 {
5642         u32 dtype;
5643
5644         if (set->dtype == NFT_DATA_VERDICT)
5645                 dtype = NFT_DATA_VERDICT;
5646         else
5647                 dtype = NFT_DATA_VALUE;
5648
5649         desc->type = dtype;
5650         desc->size = NFT_DATA_VALUE_MAXLEN;
5651         desc->len = set->dlen;
5652         desc->flags = NFT_DATA_DESC_SETELEM;
5653
5654         return nft_data_init(ctx, data, desc, attr);
5655 }
5656
5657 static void *nft_setelem_catchall_get(const struct net *net,
5658                                       const struct nft_set *set)
5659 {
5660         struct nft_set_elem_catchall *catchall;
5661         u8 genmask = nft_genmask_cur(net);
5662         struct nft_set_ext *ext;
5663         void *priv = NULL;
5664
5665         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5666                 ext = nft_set_elem_ext(set, catchall->elem);
5667                 if (!nft_set_elem_active(ext, genmask) ||
5668                     nft_set_elem_expired(ext))
5669                         continue;
5670
5671                 priv = catchall->elem;
5672                 break;
5673         }
5674
5675         return priv;
5676 }
5677
5678 static int nft_setelem_get(struct nft_ctx *ctx, struct nft_set *set,
5679                            struct nft_set_elem *elem, u32 flags)
5680 {
5681         void *priv;
5682
5683         if (!(flags & NFT_SET_ELEM_CATCHALL)) {
5684                 priv = set->ops->get(ctx->net, set, elem, flags);
5685                 if (IS_ERR(priv))
5686                         return PTR_ERR(priv);
5687         } else {
5688                 priv = nft_setelem_catchall_get(ctx->net, set);
5689                 if (!priv)
5690                         return -ENOENT;
5691         }
5692         elem->priv = priv;
5693
5694         return 0;
5695 }
5696
5697 static int nft_get_set_elem(struct nft_ctx *ctx, struct nft_set *set,
5698                             const struct nlattr *attr)
5699 {
5700         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
5701         struct nft_set_elem elem;
5702         struct sk_buff *skb;
5703         uint32_t flags = 0;
5704         int err;
5705
5706         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
5707                                           nft_set_elem_policy, NULL);
5708         if (err < 0)
5709                 return err;
5710
5711         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
5712         if (err < 0)
5713                 return err;
5714
5715         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
5716                 return -EINVAL;
5717
5718         if (nla[NFTA_SET_ELEM_KEY]) {
5719                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
5720                                             nla[NFTA_SET_ELEM_KEY]);
5721                 if (err < 0)
5722                         return err;
5723         }
5724
5725         if (nla[NFTA_SET_ELEM_KEY_END]) {
5726                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
5727                                             nla[NFTA_SET_ELEM_KEY_END]);
5728                 if (err < 0)
5729                         return err;
5730         }
5731
5732         err = nft_setelem_get(ctx, set, &elem, flags);
5733         if (err < 0)
5734                 return err;
5735
5736         err = -ENOMEM;
5737         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_ATOMIC);
5738         if (skb == NULL)
5739                 return err;
5740
5741         err = nf_tables_fill_setelem_info(skb, ctx, ctx->seq, ctx->portid,
5742                                           NFT_MSG_NEWSETELEM, 0, set, &elem);
5743         if (err < 0)
5744                 goto err_fill_setelem;
5745
5746         return nfnetlink_unicast(skb, ctx->net, ctx->portid);
5747
5748 err_fill_setelem:
5749         kfree_skb(skb);
5750         return err;
5751 }
5752
5753 /* called with rcu_read_lock held */
5754 static int nf_tables_getsetelem(struct sk_buff *skb,
5755                                 const struct nfnl_info *info,
5756                                 const struct nlattr * const nla[])
5757 {
5758         struct netlink_ext_ack *extack = info->extack;
5759         u8 genmask = nft_genmask_cur(info->net);
5760         u8 family = info->nfmsg->nfgen_family;
5761         struct net *net = info->net;
5762         struct nft_table *table;
5763         struct nft_set *set;
5764         struct nlattr *attr;
5765         struct nft_ctx ctx;
5766         int rem, err = 0;
5767
5768         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
5769                                  genmask, 0);
5770         if (IS_ERR(table)) {
5771                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
5772                 return PTR_ERR(table);
5773         }
5774
5775         set = nft_set_lookup(table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
5776         if (IS_ERR(set))
5777                 return PTR_ERR(set);
5778
5779         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
5780
5781         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
5782                 struct netlink_dump_control c = {
5783                         .start = nf_tables_dump_set_start,
5784                         .dump = nf_tables_dump_set,
5785                         .done = nf_tables_dump_set_done,
5786                         .module = THIS_MODULE,
5787                 };
5788                 struct nft_set_dump_ctx dump_ctx = {
5789                         .set = set,
5790                         .ctx = ctx,
5791                 };
5792
5793                 c.data = &dump_ctx;
5794                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
5795         }
5796
5797         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
5798                 return -EINVAL;
5799
5800         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
5801                 err = nft_get_set_elem(&ctx, set, attr);
5802                 if (err < 0) {
5803                         NL_SET_BAD_ATTR(extack, attr);
5804                         break;
5805                 }
5806         }
5807
5808         return err;
5809 }
5810
5811 static void nf_tables_setelem_notify(const struct nft_ctx *ctx,
5812                                      const struct nft_set *set,
5813                                      const struct nft_set_elem *elem,
5814                                      int event)
5815 {
5816         struct nftables_pernet *nft_net;
5817         struct net *net = ctx->net;
5818         u32 portid = ctx->portid;
5819         struct sk_buff *skb;
5820         u16 flags = 0;
5821         int err;
5822
5823         if (!ctx->report && !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
5824                 return;
5825
5826         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
5827         if (skb == NULL)
5828                 goto err;
5829
5830         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
5831                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
5832
5833         err = nf_tables_fill_setelem_info(skb, ctx, 0, portid, event, flags,
5834                                           set, elem);
5835         if (err < 0) {
5836                 kfree_skb(skb);
5837                 goto err;
5838         }
5839
5840         nft_net = nft_pernet(net);
5841         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
5842         return;
5843 err:
5844         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
5845 }
5846
5847 static struct nft_trans *nft_trans_elem_alloc(struct nft_ctx *ctx,
5848                                               int msg_type,
5849                                               struct nft_set *set)
5850 {
5851         struct nft_trans *trans;
5852
5853         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_elem));
5854         if (trans == NULL)
5855                 return NULL;
5856
5857         nft_trans_elem_set(trans) = set;
5858         return trans;
5859 }
5860
5861 struct nft_expr *nft_set_elem_expr_alloc(const struct nft_ctx *ctx,
5862                                          const struct nft_set *set,
5863                                          const struct nlattr *attr)
5864 {
5865         struct nft_expr *expr;
5866         int err;
5867
5868         expr = nft_expr_init(ctx, attr);
5869         if (IS_ERR(expr))
5870                 return expr;
5871
5872         err = -EOPNOTSUPP;
5873         if (expr->ops->type->flags & NFT_EXPR_GC) {
5874                 if (set->flags & NFT_SET_TIMEOUT)
5875                         goto err_set_elem_expr;
5876                 if (!set->ops->gc_init)
5877                         goto err_set_elem_expr;
5878                 set->ops->gc_init(set);
5879         }
5880
5881         return expr;
5882
5883 err_set_elem_expr:
5884         nft_expr_destroy(ctx, expr);
5885         return ERR_PTR(err);
5886 }
5887
5888 static int nft_set_ext_check(const struct nft_set_ext_tmpl *tmpl, u8 id, u32 len)
5889 {
5890         len += nft_set_ext_types[id].len;
5891         if (len > tmpl->ext_len[id] ||
5892             len > U8_MAX)
5893                 return -1;
5894
5895         return 0;
5896 }
5897
5898 static int nft_set_ext_memcpy(const struct nft_set_ext_tmpl *tmpl, u8 id,
5899                               void *to, const void *from, u32 len)
5900 {
5901         if (nft_set_ext_check(tmpl, id, len) < 0)
5902                 return -1;
5903
5904         memcpy(to, from, len);
5905
5906         return 0;
5907 }
5908
5909 void *nft_set_elem_init(const struct nft_set *set,
5910                         const struct nft_set_ext_tmpl *tmpl,
5911                         const u32 *key, const u32 *key_end,
5912                         const u32 *data, u64 timeout, u64 expiration, gfp_t gfp)
5913 {
5914         struct nft_set_ext *ext;
5915         void *elem;
5916
5917         elem = kzalloc(set->ops->elemsize + tmpl->len, gfp);
5918         if (elem == NULL)
5919                 return ERR_PTR(-ENOMEM);
5920
5921         ext = nft_set_elem_ext(set, elem);
5922         nft_set_ext_init(ext, tmpl);
5923
5924         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
5925             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY,
5926                                nft_set_ext_key(ext), key, set->klen) < 0)
5927                 goto err_ext_check;
5928
5929         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
5930             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY_END,
5931                                nft_set_ext_key_end(ext), key_end, set->klen) < 0)
5932                 goto err_ext_check;
5933
5934         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
5935             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_DATA,
5936                                nft_set_ext_data(ext), data, set->dlen) < 0)
5937                 goto err_ext_check;
5938
5939         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
5940                 *nft_set_ext_expiration(ext) = get_jiffies_64() + expiration;
5941                 if (expiration == 0)
5942                         *nft_set_ext_expiration(ext) += timeout;
5943         }
5944         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT))
5945                 *nft_set_ext_timeout(ext) = timeout;
5946
5947         return elem;
5948
5949 err_ext_check:
5950         kfree(elem);
5951
5952         return ERR_PTR(-EINVAL);
5953 }
5954
5955 static void __nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
5956                                         struct nft_expr *expr)
5957 {
5958         if (expr->ops->destroy_clone) {
5959                 expr->ops->destroy_clone(ctx, expr);
5960                 module_put(expr->ops->type->owner);
5961         } else {
5962                 nf_tables_expr_destroy(ctx, expr);
5963         }
5964 }
5965
5966 static void nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
5967                                       struct nft_set_elem_expr *elem_expr)
5968 {
5969         struct nft_expr *expr;
5970         u32 size;
5971
5972         nft_setelem_expr_foreach(expr, elem_expr, size)
5973                 __nft_set_elem_expr_destroy(ctx, expr);
5974 }
5975
5976 void nft_set_elem_destroy(const struct nft_set *set, void *elem,
5977                           bool destroy_expr)
5978 {
5979         struct nft_set_ext *ext = nft_set_elem_ext(set, elem);
5980         struct nft_ctx ctx = {
5981                 .net    = read_pnet(&set->net),
5982                 .family = set->table->family,
5983         };
5984
5985         nft_data_release(nft_set_ext_key(ext), NFT_DATA_VALUE);
5986         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
5987                 nft_data_release(nft_set_ext_data(ext), set->dtype);
5988         if (destroy_expr && nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
5989                 nft_set_elem_expr_destroy(&ctx, nft_set_ext_expr(ext));
5990
5991         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
5992                 (*nft_set_ext_obj(ext))->use--;
5993         kfree(elem);
5994 }
5995 EXPORT_SYMBOL_GPL(nft_set_elem_destroy);
5996
5997 /* Only called from commit path, nft_setelem_data_deactivate() already deals
5998  * with the refcounting from the preparation phase.
5999  */
6000 static void nf_tables_set_elem_destroy(const struct nft_ctx *ctx,
6001                                        const struct nft_set *set, void *elem)
6002 {
6003         struct nft_set_ext *ext = nft_set_elem_ext(set, elem);
6004
6005         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
6006                 nft_set_elem_expr_destroy(ctx, nft_set_ext_expr(ext));
6007
6008         kfree(elem);
6009 }
6010
6011 int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set,
6012                             struct nft_expr *expr_array[])
6013 {
6014         struct nft_expr *expr;
6015         int err, i, k;
6016
6017         for (i = 0; i < set->num_exprs; i++) {
6018                 expr = kzalloc(set->exprs[i]->ops->size, GFP_KERNEL_ACCOUNT);
6019                 if (!expr)
6020                         goto err_expr;
6021
6022                 err = nft_expr_clone(expr, set->exprs[i]);
6023                 if (err < 0) {
6024                         kfree(expr);
6025                         goto err_expr;
6026                 }
6027                 expr_array[i] = expr;
6028         }
6029
6030         return 0;
6031
6032 err_expr:
6033         for (k = i - 1; k >= 0; k--)
6034                 nft_expr_destroy(ctx, expr_array[k]);
6035
6036         return -ENOMEM;
6037 }
6038
6039 static int nft_set_elem_expr_setup(struct nft_ctx *ctx,
6040                                    const struct nft_set_ext_tmpl *tmpl,
6041                                    const struct nft_set_ext *ext,
6042                                    struct nft_expr *expr_array[],
6043                                    u32 num_exprs)
6044 {
6045         struct nft_set_elem_expr *elem_expr = nft_set_ext_expr(ext);
6046         u32 len = sizeof(struct nft_set_elem_expr);
6047         struct nft_expr *expr;
6048         int i, err;
6049
6050         if (num_exprs == 0)
6051                 return 0;
6052
6053         for (i = 0; i < num_exprs; i++)
6054                 len += expr_array[i]->ops->size;
6055
6056         if (nft_set_ext_check(tmpl, NFT_SET_EXT_EXPRESSIONS, len) < 0)
6057                 return -EINVAL;
6058
6059         for (i = 0; i < num_exprs; i++) {
6060                 expr = nft_setelem_expr_at(elem_expr, elem_expr->size);
6061                 err = nft_expr_clone(expr, expr_array[i]);
6062                 if (err < 0)
6063                         goto err_elem_expr_setup;
6064
6065                 elem_expr->size += expr_array[i]->ops->size;
6066                 nft_expr_destroy(ctx, expr_array[i]);
6067                 expr_array[i] = NULL;
6068         }
6069
6070         return 0;
6071
6072 err_elem_expr_setup:
6073         for (; i < num_exprs; i++) {
6074                 nft_expr_destroy(ctx, expr_array[i]);
6075                 expr_array[i] = NULL;
6076         }
6077
6078         return -ENOMEM;
6079 }
6080
6081 struct nft_set_ext *nft_set_catchall_lookup(const struct net *net,
6082                                             const struct nft_set *set)
6083 {
6084         struct nft_set_elem_catchall *catchall;
6085         u8 genmask = nft_genmask_cur(net);
6086         struct nft_set_ext *ext;
6087
6088         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
6089                 ext = nft_set_elem_ext(set, catchall->elem);
6090                 if (nft_set_elem_active(ext, genmask) &&
6091                     !nft_set_elem_expired(ext))
6092                         return ext;
6093         }
6094
6095         return NULL;
6096 }
6097 EXPORT_SYMBOL_GPL(nft_set_catchall_lookup);
6098
6099 void *nft_set_catchall_gc(const struct nft_set *set)
6100 {
6101         struct nft_set_elem_catchall *catchall, *next;
6102         struct nft_set_ext *ext;
6103         void *elem = NULL;
6104
6105         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
6106                 ext = nft_set_elem_ext(set, catchall->elem);
6107
6108                 if (!nft_set_elem_expired(ext) ||
6109                     nft_set_elem_mark_busy(ext))
6110                         continue;
6111
6112                 elem = catchall->elem;
6113                 list_del_rcu(&catchall->list);
6114                 kfree_rcu(catchall, rcu);
6115                 break;
6116         }
6117
6118         return elem;
6119 }
6120 EXPORT_SYMBOL_GPL(nft_set_catchall_gc);
6121
6122 static int nft_setelem_catchall_insert(const struct net *net,
6123                                        struct nft_set *set,
6124                                        const struct nft_set_elem *elem,
6125                                        struct nft_set_ext **pext)
6126 {
6127         struct nft_set_elem_catchall *catchall;
6128         u8 genmask = nft_genmask_next(net);
6129         struct nft_set_ext *ext;
6130
6131         list_for_each_entry(catchall, &set->catchall_list, list) {
6132                 ext = nft_set_elem_ext(set, catchall->elem);
6133                 if (nft_set_elem_active(ext, genmask)) {
6134                         *pext = ext;
6135                         return -EEXIST;
6136                 }
6137         }
6138
6139         catchall = kmalloc(sizeof(*catchall), GFP_KERNEL);
6140         if (!catchall)
6141                 return -ENOMEM;
6142
6143         catchall->elem = elem->priv;
6144         list_add_tail_rcu(&catchall->list, &set->catchall_list);
6145
6146         return 0;
6147 }
6148
6149 static int nft_setelem_insert(const struct net *net,
6150                               struct nft_set *set,
6151                               const struct nft_set_elem *elem,
6152                               struct nft_set_ext **ext, unsigned int flags)
6153 {
6154         int ret;
6155
6156         if (flags & NFT_SET_ELEM_CATCHALL)
6157                 ret = nft_setelem_catchall_insert(net, set, elem, ext);
6158         else
6159                 ret = set->ops->insert(net, set, elem, ext);
6160
6161         return ret;
6162 }
6163
6164 static bool nft_setelem_is_catchall(const struct nft_set *set,
6165                                     const struct nft_set_elem *elem)
6166 {
6167         struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6168
6169         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
6170             *nft_set_ext_flags(ext) & NFT_SET_ELEM_CATCHALL)
6171                 return true;
6172
6173         return false;
6174 }
6175
6176 static void nft_setelem_activate(struct net *net, struct nft_set *set,
6177                                  struct nft_set_elem *elem)
6178 {
6179         struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6180
6181         if (nft_setelem_is_catchall(set, elem)) {
6182                 nft_set_elem_change_active(net, set, ext);
6183                 nft_set_elem_clear_busy(ext);
6184         } else {
6185                 set->ops->activate(net, set, elem);
6186         }
6187 }
6188
6189 static int nft_setelem_catchall_deactivate(const struct net *net,
6190                                            struct nft_set *set,
6191                                            struct nft_set_elem *elem)
6192 {
6193         struct nft_set_elem_catchall *catchall;
6194         struct nft_set_ext *ext;
6195
6196         list_for_each_entry(catchall, &set->catchall_list, list) {
6197                 ext = nft_set_elem_ext(set, catchall->elem);
6198                 if (!nft_is_active(net, ext) ||
6199                     nft_set_elem_mark_busy(ext))
6200                         continue;
6201
6202                 kfree(elem->priv);
6203                 elem->priv = catchall->elem;
6204                 nft_set_elem_change_active(net, set, ext);
6205                 return 0;
6206         }
6207
6208         return -ENOENT;
6209 }
6210
6211 static int __nft_setelem_deactivate(const struct net *net,
6212                                     struct nft_set *set,
6213                                     struct nft_set_elem *elem)
6214 {
6215         void *priv;
6216
6217         priv = set->ops->deactivate(net, set, elem);
6218         if (!priv)
6219                 return -ENOENT;
6220
6221         kfree(elem->priv);
6222         elem->priv = priv;
6223         set->ndeact++;
6224
6225         return 0;
6226 }
6227
6228 static int nft_setelem_deactivate(const struct net *net,
6229                                   struct nft_set *set,
6230                                   struct nft_set_elem *elem, u32 flags)
6231 {
6232         int ret;
6233
6234         if (flags & NFT_SET_ELEM_CATCHALL)
6235                 ret = nft_setelem_catchall_deactivate(net, set, elem);
6236         else
6237                 ret = __nft_setelem_deactivate(net, set, elem);
6238
6239         return ret;
6240 }
6241
6242 static void nft_setelem_catchall_remove(const struct net *net,
6243                                         const struct nft_set *set,
6244                                         const struct nft_set_elem *elem)
6245 {
6246         struct nft_set_elem_catchall *catchall, *next;
6247
6248         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
6249                 if (catchall->elem == elem->priv) {
6250                         list_del_rcu(&catchall->list);
6251                         kfree_rcu(catchall, rcu);
6252                         break;
6253                 }
6254         }
6255 }
6256
6257 static void nft_setelem_remove(const struct net *net,
6258                                const struct nft_set *set,
6259                                const struct nft_set_elem *elem)
6260 {
6261         if (nft_setelem_is_catchall(set, elem))
6262                 nft_setelem_catchall_remove(net, set, elem);
6263         else
6264                 set->ops->remove(net, set, elem);
6265 }
6266
6267 static bool nft_setelem_valid_key_end(const struct nft_set *set,
6268                                       struct nlattr **nla, u32 flags)
6269 {
6270         if ((set->flags & (NFT_SET_CONCAT | NFT_SET_INTERVAL)) ==
6271                           (NFT_SET_CONCAT | NFT_SET_INTERVAL)) {
6272                 if (flags & NFT_SET_ELEM_INTERVAL_END)
6273                         return false;
6274
6275                 if (nla[NFTA_SET_ELEM_KEY_END] &&
6276                     flags & NFT_SET_ELEM_CATCHALL)
6277                         return false;
6278         } else {
6279                 if (nla[NFTA_SET_ELEM_KEY_END])
6280                         return false;
6281         }
6282
6283         return true;
6284 }
6285
6286 static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set,
6287                             const struct nlattr *attr, u32 nlmsg_flags)
6288 {
6289         struct nft_expr *expr_array[NFT_SET_EXPR_MAX] = {};
6290         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
6291         u8 genmask = nft_genmask_next(ctx->net);
6292         u32 flags = 0, size = 0, num_exprs = 0;
6293         struct nft_set_ext_tmpl tmpl;
6294         struct nft_set_ext *ext, *ext2;
6295         struct nft_set_elem elem;
6296         struct nft_set_binding *binding;
6297         struct nft_object *obj = NULL;
6298         struct nft_userdata *udata;
6299         struct nft_data_desc desc;
6300         enum nft_registers dreg;
6301         struct nft_trans *trans;
6302         u64 timeout;
6303         u64 expiration;
6304         int err, i;
6305         u8 ulen;
6306
6307         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
6308                                           nft_set_elem_policy, NULL);
6309         if (err < 0)
6310                 return err;
6311
6312         nft_set_ext_prepare(&tmpl);
6313
6314         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
6315         if (err < 0)
6316                 return err;
6317
6318         if (((flags & NFT_SET_ELEM_CATCHALL) && nla[NFTA_SET_ELEM_KEY]) ||
6319             (!(flags & NFT_SET_ELEM_CATCHALL) && !nla[NFTA_SET_ELEM_KEY]))
6320                 return -EINVAL;
6321
6322         if (flags != 0) {
6323                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
6324                 if (err < 0)
6325                         return err;
6326         }
6327
6328         if (set->flags & NFT_SET_MAP) {
6329                 if (nla[NFTA_SET_ELEM_DATA] == NULL &&
6330                     !(flags & NFT_SET_ELEM_INTERVAL_END))
6331                         return -EINVAL;
6332         } else {
6333                 if (nla[NFTA_SET_ELEM_DATA] != NULL)
6334                         return -EINVAL;
6335         }
6336
6337         if (set->flags & NFT_SET_OBJECT) {
6338                 if (!nla[NFTA_SET_ELEM_OBJREF] &&
6339                     !(flags & NFT_SET_ELEM_INTERVAL_END))
6340                         return -EINVAL;
6341         } else {
6342                 if (nla[NFTA_SET_ELEM_OBJREF])
6343                         return -EINVAL;
6344         }
6345
6346         if (!nft_setelem_valid_key_end(set, nla, flags))
6347                 return -EINVAL;
6348
6349         if ((flags & NFT_SET_ELEM_INTERVAL_END) &&
6350              (nla[NFTA_SET_ELEM_DATA] ||
6351               nla[NFTA_SET_ELEM_OBJREF] ||
6352               nla[NFTA_SET_ELEM_TIMEOUT] ||
6353               nla[NFTA_SET_ELEM_EXPIRATION] ||
6354               nla[NFTA_SET_ELEM_USERDATA] ||
6355               nla[NFTA_SET_ELEM_EXPR] ||
6356               nla[NFTA_SET_ELEM_KEY_END] ||
6357               nla[NFTA_SET_ELEM_EXPRESSIONS]))
6358                 return -EINVAL;
6359
6360         timeout = 0;
6361         if (nla[NFTA_SET_ELEM_TIMEOUT] != NULL) {
6362                 if (!(set->flags & NFT_SET_TIMEOUT))
6363                         return -EINVAL;
6364                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_TIMEOUT],
6365                                             &timeout);
6366                 if (err)
6367                         return err;
6368         } else if (set->flags & NFT_SET_TIMEOUT &&
6369                    !(flags & NFT_SET_ELEM_INTERVAL_END)) {
6370                 timeout = READ_ONCE(set->timeout);
6371         }
6372
6373         expiration = 0;
6374         if (nla[NFTA_SET_ELEM_EXPIRATION] != NULL) {
6375                 if (!(set->flags & NFT_SET_TIMEOUT))
6376                         return -EINVAL;
6377                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_EXPIRATION],
6378                                             &expiration);
6379                 if (err)
6380                         return err;
6381         }
6382
6383         if (nla[NFTA_SET_ELEM_EXPR]) {
6384                 struct nft_expr *expr;
6385
6386                 if (set->num_exprs && set->num_exprs != 1)
6387                         return -EOPNOTSUPP;
6388
6389                 expr = nft_set_elem_expr_alloc(ctx, set,
6390                                                nla[NFTA_SET_ELEM_EXPR]);
6391                 if (IS_ERR(expr))
6392                         return PTR_ERR(expr);
6393
6394                 expr_array[0] = expr;
6395                 num_exprs = 1;
6396
6397                 if (set->num_exprs && set->exprs[0]->ops != expr->ops) {
6398                         err = -EOPNOTSUPP;
6399                         goto err_set_elem_expr;
6400                 }
6401         } else if (nla[NFTA_SET_ELEM_EXPRESSIONS]) {
6402                 struct nft_expr *expr;
6403                 struct nlattr *tmp;
6404                 int left;
6405
6406                 i = 0;
6407                 nla_for_each_nested(tmp, nla[NFTA_SET_ELEM_EXPRESSIONS], left) {
6408                         if (i == NFT_SET_EXPR_MAX ||
6409                             (set->num_exprs && set->num_exprs == i)) {
6410                                 err = -E2BIG;
6411                                 goto err_set_elem_expr;
6412                         }
6413                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
6414                                 err = -EINVAL;
6415                                 goto err_set_elem_expr;
6416                         }
6417                         expr = nft_set_elem_expr_alloc(ctx, set, tmp);
6418                         if (IS_ERR(expr)) {
6419                                 err = PTR_ERR(expr);
6420                                 goto err_set_elem_expr;
6421                         }
6422                         expr_array[i] = expr;
6423                         num_exprs++;
6424
6425                         if (set->num_exprs && expr->ops != set->exprs[i]->ops) {
6426                                 err = -EOPNOTSUPP;
6427                                 goto err_set_elem_expr;
6428                         }
6429                         i++;
6430                 }
6431                 if (set->num_exprs && set->num_exprs != i) {
6432                         err = -EOPNOTSUPP;
6433                         goto err_set_elem_expr;
6434                 }
6435         } else if (set->num_exprs > 0 &&
6436                    !(flags & NFT_SET_ELEM_INTERVAL_END)) {
6437                 err = nft_set_elem_expr_clone(ctx, set, expr_array);
6438                 if (err < 0)
6439                         goto err_set_elem_expr_clone;
6440
6441                 num_exprs = set->num_exprs;
6442         }
6443
6444         if (nla[NFTA_SET_ELEM_KEY]) {
6445                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
6446                                             nla[NFTA_SET_ELEM_KEY]);
6447                 if (err < 0)
6448                         goto err_set_elem_expr;
6449
6450                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
6451                 if (err < 0)
6452                         goto err_parse_key;
6453         }
6454
6455         if (nla[NFTA_SET_ELEM_KEY_END]) {
6456                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
6457                                             nla[NFTA_SET_ELEM_KEY_END]);
6458                 if (err < 0)
6459                         goto err_parse_key;
6460
6461                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
6462                 if (err < 0)
6463                         goto err_parse_key_end;
6464         }
6465
6466         if (timeout > 0) {
6467                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_EXPIRATION);
6468                 if (err < 0)
6469                         goto err_parse_key_end;
6470
6471                 if (timeout != READ_ONCE(set->timeout)) {
6472                         err = nft_set_ext_add(&tmpl, NFT_SET_EXT_TIMEOUT);
6473                         if (err < 0)
6474                                 goto err_parse_key_end;
6475                 }
6476         }
6477
6478         if (num_exprs) {
6479                 for (i = 0; i < num_exprs; i++)
6480                         size += expr_array[i]->ops->size;
6481
6482                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_EXPRESSIONS,
6483                                              sizeof(struct nft_set_elem_expr) + size);
6484                 if (err < 0)
6485                         goto err_parse_key_end;
6486         }
6487
6488         if (nla[NFTA_SET_ELEM_OBJREF] != NULL) {
6489                 obj = nft_obj_lookup(ctx->net, ctx->table,
6490                                      nla[NFTA_SET_ELEM_OBJREF],
6491                                      set->objtype, genmask);
6492                 if (IS_ERR(obj)) {
6493                         err = PTR_ERR(obj);
6494                         goto err_parse_key_end;
6495                 }
6496                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_OBJREF);
6497                 if (err < 0)
6498                         goto err_parse_key_end;
6499         }
6500
6501         if (nla[NFTA_SET_ELEM_DATA] != NULL) {
6502                 err = nft_setelem_parse_data(ctx, set, &desc, &elem.data.val,
6503                                              nla[NFTA_SET_ELEM_DATA]);
6504                 if (err < 0)
6505                         goto err_parse_key_end;
6506
6507                 dreg = nft_type_to_reg(set->dtype);
6508                 list_for_each_entry(binding, &set->bindings, list) {
6509                         struct nft_ctx bind_ctx = {
6510                                 .net    = ctx->net,
6511                                 .family = ctx->family,
6512                                 .table  = ctx->table,
6513                                 .chain  = (struct nft_chain *)binding->chain,
6514                         };
6515
6516                         if (!(binding->flags & NFT_SET_MAP))
6517                                 continue;
6518
6519                         err = nft_validate_register_store(&bind_ctx, dreg,
6520                                                           &elem.data.val,
6521                                                           desc.type, desc.len);
6522                         if (err < 0)
6523                                 goto err_parse_data;
6524
6525                         if (desc.type == NFT_DATA_VERDICT &&
6526                             (elem.data.val.verdict.code == NFT_GOTO ||
6527                              elem.data.val.verdict.code == NFT_JUMP))
6528                                 nft_validate_state_update(ctx->table,
6529                                                           NFT_VALIDATE_NEED);
6530                 }
6531
6532                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_DATA, desc.len);
6533                 if (err < 0)
6534                         goto err_parse_data;
6535         }
6536
6537         /* The full maximum length of userdata can exceed the maximum
6538          * offset value (U8_MAX) for following extensions, therefor it
6539          * must be the last extension added.
6540          */
6541         ulen = 0;
6542         if (nla[NFTA_SET_ELEM_USERDATA] != NULL) {
6543                 ulen = nla_len(nla[NFTA_SET_ELEM_USERDATA]);
6544                 if (ulen > 0) {
6545                         err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_USERDATA,
6546                                                      ulen);
6547                         if (err < 0)
6548                                 goto err_parse_data;
6549                 }
6550         }
6551
6552         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
6553                                       elem.key_end.val.data, elem.data.val.data,
6554                                       timeout, expiration, GFP_KERNEL_ACCOUNT);
6555         if (IS_ERR(elem.priv)) {
6556                 err = PTR_ERR(elem.priv);
6557                 goto err_parse_data;
6558         }
6559
6560         ext = nft_set_elem_ext(set, elem.priv);
6561         if (flags)
6562                 *nft_set_ext_flags(ext) = flags;
6563
6564         if (ulen > 0) {
6565                 if (nft_set_ext_check(&tmpl, NFT_SET_EXT_USERDATA, ulen) < 0) {
6566                         err = -EINVAL;
6567                         goto err_elem_userdata;
6568                 }
6569                 udata = nft_set_ext_userdata(ext);
6570                 udata->len = ulen - 1;
6571                 nla_memcpy(&udata->data, nla[NFTA_SET_ELEM_USERDATA], ulen);
6572         }
6573         if (obj) {
6574                 *nft_set_ext_obj(ext) = obj;
6575                 obj->use++;
6576         }
6577         err = nft_set_elem_expr_setup(ctx, &tmpl, ext, expr_array, num_exprs);
6578         if (err < 0)
6579                 goto err_elem_free;
6580
6581         trans = nft_trans_elem_alloc(ctx, NFT_MSG_NEWSETELEM, set);
6582         if (trans == NULL) {
6583                 err = -ENOMEM;
6584                 goto err_elem_free;
6585         }
6586
6587         ext->genmask = nft_genmask_cur(ctx->net) | NFT_SET_ELEM_BUSY_MASK;
6588
6589         err = nft_setelem_insert(ctx->net, set, &elem, &ext2, flags);
6590         if (err) {
6591                 if (err == -EEXIST) {
6592                         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) ^
6593                             nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) ||
6594                             nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) ^
6595                             nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF))
6596                                 goto err_element_clash;
6597                         if ((nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
6598                              nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) &&
6599                              memcmp(nft_set_ext_data(ext),
6600                                     nft_set_ext_data(ext2), set->dlen) != 0) ||
6601                             (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
6602                              nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF) &&
6603                              *nft_set_ext_obj(ext) != *nft_set_ext_obj(ext2)))
6604                                 goto err_element_clash;
6605                         else if (!(nlmsg_flags & NLM_F_EXCL))
6606                                 err = 0;
6607                 } else if (err == -ENOTEMPTY) {
6608                         /* ENOTEMPTY reports overlapping between this element
6609                          * and an existing one.
6610                          */
6611                         err = -EEXIST;
6612                 }
6613                 goto err_element_clash;
6614         }
6615
6616         if (!(flags & NFT_SET_ELEM_CATCHALL) && set->size &&
6617             !atomic_add_unless(&set->nelems, 1, set->size + set->ndeact)) {
6618                 err = -ENFILE;
6619                 goto err_set_full;
6620         }
6621
6622         nft_trans_elem(trans) = elem;
6623         nft_trans_commit_list_add_tail(ctx->net, trans);
6624         return 0;
6625
6626 err_set_full:
6627         nft_setelem_remove(ctx->net, set, &elem);
6628 err_element_clash:
6629         kfree(trans);
6630 err_elem_free:
6631         if (obj)
6632                 obj->use--;
6633 err_elem_userdata:
6634         nf_tables_set_elem_destroy(ctx, set, elem.priv);
6635 err_parse_data:
6636         if (nla[NFTA_SET_ELEM_DATA] != NULL)
6637                 nft_data_release(&elem.data.val, desc.type);
6638 err_parse_key_end:
6639         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
6640 err_parse_key:
6641         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
6642 err_set_elem_expr:
6643         for (i = 0; i < num_exprs && expr_array[i]; i++)
6644                 nft_expr_destroy(ctx, expr_array[i]);
6645 err_set_elem_expr_clone:
6646         return err;
6647 }
6648
6649 static int nf_tables_newsetelem(struct sk_buff *skb,
6650                                 const struct nfnl_info *info,
6651                                 const struct nlattr * const nla[])
6652 {
6653         struct netlink_ext_ack *extack = info->extack;
6654         u8 genmask = nft_genmask_next(info->net);
6655         u8 family = info->nfmsg->nfgen_family;
6656         struct net *net = info->net;
6657         const struct nlattr *attr;
6658         struct nft_table *table;
6659         struct nft_set *set;
6660         struct nft_ctx ctx;
6661         int rem, err;
6662
6663         if (nla[NFTA_SET_ELEM_LIST_ELEMENTS] == NULL)
6664                 return -EINVAL;
6665
6666         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
6667                                  genmask, NETLINK_CB(skb).portid);
6668         if (IS_ERR(table)) {
6669                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
6670                 return PTR_ERR(table);
6671         }
6672
6673         set = nft_set_lookup_global(net, table, nla[NFTA_SET_ELEM_LIST_SET],
6674                                     nla[NFTA_SET_ELEM_LIST_SET_ID], genmask);
6675         if (IS_ERR(set))
6676                 return PTR_ERR(set);
6677
6678         if (!list_empty(&set->bindings) && set->flags & NFT_SET_CONSTANT)
6679                 return -EBUSY;
6680
6681         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
6682
6683         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
6684                 err = nft_add_set_elem(&ctx, set, attr, info->nlh->nlmsg_flags);
6685                 if (err < 0) {
6686                         NL_SET_BAD_ATTR(extack, attr);
6687                         return err;
6688                 }
6689         }
6690
6691         if (table->validate_state == NFT_VALIDATE_DO)
6692                 return nft_table_validate(net, table);
6693
6694         return 0;
6695 }
6696
6697 /**
6698  *      nft_data_hold - hold a nft_data item
6699  *
6700  *      @data: struct nft_data to release
6701  *      @type: type of data
6702  *
6703  *      Hold a nft_data item. NFT_DATA_VALUE types can be silently discarded,
6704  *      NFT_DATA_VERDICT bumps the reference to chains in case of NFT_JUMP and
6705  *      NFT_GOTO verdicts. This function must be called on active data objects
6706  *      from the second phase of the commit protocol.
6707  */
6708 void nft_data_hold(const struct nft_data *data, enum nft_data_types type)
6709 {
6710         struct nft_chain *chain;
6711
6712         if (type == NFT_DATA_VERDICT) {
6713                 switch (data->verdict.code) {
6714                 case NFT_JUMP:
6715                 case NFT_GOTO:
6716                         chain = data->verdict.chain;
6717                         chain->use++;
6718                         break;
6719                 }
6720         }
6721 }
6722
6723 static void nft_setelem_data_activate(const struct net *net,
6724                                       const struct nft_set *set,
6725                                       struct nft_set_elem *elem)
6726 {
6727         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6728
6729         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
6730                 nft_data_hold(nft_set_ext_data(ext), set->dtype);
6731         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
6732                 (*nft_set_ext_obj(ext))->use++;
6733 }
6734
6735 static void nft_setelem_data_deactivate(const struct net *net,
6736                                         const struct nft_set *set,
6737                                         struct nft_set_elem *elem)
6738 {
6739         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6740
6741         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
6742                 nft_data_release(nft_set_ext_data(ext), set->dtype);
6743         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
6744                 (*nft_set_ext_obj(ext))->use--;
6745 }
6746
6747 static int nft_del_setelem(struct nft_ctx *ctx, struct nft_set *set,
6748                            const struct nlattr *attr)
6749 {
6750         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
6751         struct nft_set_ext_tmpl tmpl;
6752         struct nft_set_elem elem;
6753         struct nft_set_ext *ext;
6754         struct nft_trans *trans;
6755         u32 flags = 0;
6756         int err;
6757
6758         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
6759                                           nft_set_elem_policy, NULL);
6760         if (err < 0)
6761                 return err;
6762
6763         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
6764         if (err < 0)
6765                 return err;
6766
6767         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
6768                 return -EINVAL;
6769
6770         if (!nft_setelem_valid_key_end(set, nla, flags))
6771                 return -EINVAL;
6772
6773         nft_set_ext_prepare(&tmpl);
6774
6775         if (flags != 0) {
6776                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
6777                 if (err < 0)
6778                         return err;
6779         }
6780
6781         if (nla[NFTA_SET_ELEM_KEY]) {
6782                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
6783                                             nla[NFTA_SET_ELEM_KEY]);
6784                 if (err < 0)
6785                         return err;
6786
6787                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
6788                 if (err < 0)
6789                         goto fail_elem;
6790         }
6791
6792         if (nla[NFTA_SET_ELEM_KEY_END]) {
6793                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
6794                                             nla[NFTA_SET_ELEM_KEY_END]);
6795                 if (err < 0)
6796                         goto fail_elem;
6797
6798                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
6799                 if (err < 0)
6800                         goto fail_elem_key_end;
6801         }
6802
6803         err = -ENOMEM;
6804         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
6805                                       elem.key_end.val.data, NULL, 0, 0,
6806                                       GFP_KERNEL_ACCOUNT);
6807         if (IS_ERR(elem.priv)) {
6808                 err = PTR_ERR(elem.priv);
6809                 goto fail_elem_key_end;
6810         }
6811
6812         ext = nft_set_elem_ext(set, elem.priv);
6813         if (flags)
6814                 *nft_set_ext_flags(ext) = flags;
6815
6816         trans = nft_trans_elem_alloc(ctx, NFT_MSG_DELSETELEM, set);
6817         if (trans == NULL)
6818                 goto fail_trans;
6819
6820         err = nft_setelem_deactivate(ctx->net, set, &elem, flags);
6821         if (err < 0)
6822                 goto fail_ops;
6823
6824         nft_setelem_data_deactivate(ctx->net, set, &elem);
6825
6826         nft_trans_elem(trans) = elem;
6827         nft_trans_commit_list_add_tail(ctx->net, trans);
6828         return 0;
6829
6830 fail_ops:
6831         kfree(trans);
6832 fail_trans:
6833         kfree(elem.priv);
6834 fail_elem_key_end:
6835         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
6836 fail_elem:
6837         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
6838         return err;
6839 }
6840
6841 static int nft_setelem_flush(const struct nft_ctx *ctx,
6842                              struct nft_set *set,
6843                              const struct nft_set_iter *iter,
6844                              struct nft_set_elem *elem)
6845 {
6846         struct nft_trans *trans;
6847         int err;
6848
6849         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
6850                                     sizeof(struct nft_trans_elem), GFP_ATOMIC);
6851         if (!trans)
6852                 return -ENOMEM;
6853
6854         if (!set->ops->flush(ctx->net, set, elem->priv)) {
6855                 err = -ENOENT;
6856                 goto err1;
6857         }
6858         set->ndeact++;
6859
6860         nft_setelem_data_deactivate(ctx->net, set, elem);
6861         nft_trans_elem_set(trans) = set;
6862         nft_trans_elem(trans) = *elem;
6863         nft_trans_commit_list_add_tail(ctx->net, trans);
6864
6865         return 0;
6866 err1:
6867         kfree(trans);
6868         return err;
6869 }
6870
6871 static int __nft_set_catchall_flush(const struct nft_ctx *ctx,
6872                                     struct nft_set *set,
6873                                     struct nft_set_elem *elem)
6874 {
6875         struct nft_trans *trans;
6876
6877         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
6878                                     sizeof(struct nft_trans_elem), GFP_KERNEL);
6879         if (!trans)
6880                 return -ENOMEM;
6881
6882         nft_setelem_data_deactivate(ctx->net, set, elem);
6883         nft_trans_elem_set(trans) = set;
6884         nft_trans_elem(trans) = *elem;
6885         nft_trans_commit_list_add_tail(ctx->net, trans);
6886
6887         return 0;
6888 }
6889
6890 static int nft_set_catchall_flush(const struct nft_ctx *ctx,
6891                                   struct nft_set *set)
6892 {
6893         u8 genmask = nft_genmask_next(ctx->net);
6894         struct nft_set_elem_catchall *catchall;
6895         struct nft_set_elem elem;
6896         struct nft_set_ext *ext;
6897         int ret = 0;
6898
6899         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
6900                 ext = nft_set_elem_ext(set, catchall->elem);
6901                 if (!nft_set_elem_active(ext, genmask) ||
6902                     nft_set_elem_mark_busy(ext))
6903                         continue;
6904
6905                 elem.priv = catchall->elem;
6906                 ret = __nft_set_catchall_flush(ctx, set, &elem);
6907                 if (ret < 0)
6908                         break;
6909         }
6910
6911         return ret;
6912 }
6913
6914 static int nft_set_flush(struct nft_ctx *ctx, struct nft_set *set, u8 genmask)
6915 {
6916         struct nft_set_iter iter = {
6917                 .genmask        = genmask,
6918                 .fn             = nft_setelem_flush,
6919         };
6920
6921         set->ops->walk(ctx, set, &iter);
6922         if (!iter.err)
6923                 iter.err = nft_set_catchall_flush(ctx, set);
6924
6925         return iter.err;
6926 }
6927
6928 static int nf_tables_delsetelem(struct sk_buff *skb,
6929                                 const struct nfnl_info *info,
6930                                 const struct nlattr * const nla[])
6931 {
6932         struct netlink_ext_ack *extack = info->extack;
6933         u8 genmask = nft_genmask_next(info->net);
6934         u8 family = info->nfmsg->nfgen_family;
6935         struct net *net = info->net;
6936         const struct nlattr *attr;
6937         struct nft_table *table;
6938         struct nft_set *set;
6939         struct nft_ctx ctx;
6940         int rem, err = 0;
6941
6942         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
6943                                  genmask, NETLINK_CB(skb).portid);
6944         if (IS_ERR(table)) {
6945                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
6946                 return PTR_ERR(table);
6947         }
6948
6949         set = nft_set_lookup(table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
6950         if (IS_ERR(set))
6951                 return PTR_ERR(set);
6952         if (!list_empty(&set->bindings) && set->flags & NFT_SET_CONSTANT)
6953                 return -EBUSY;
6954
6955         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
6956
6957         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
6958                 return nft_set_flush(&ctx, set, genmask);
6959
6960         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
6961                 err = nft_del_setelem(&ctx, set, attr);
6962                 if (err == -ENOENT &&
6963                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYSETELEM)
6964                         continue;
6965
6966                 if (err < 0) {
6967                         NL_SET_BAD_ATTR(extack, attr);
6968                         break;
6969                 }
6970         }
6971         return err;
6972 }
6973
6974 void nft_set_gc_batch_release(struct rcu_head *rcu)
6975 {
6976         struct nft_set_gc_batch *gcb;
6977         unsigned int i;
6978
6979         gcb = container_of(rcu, struct nft_set_gc_batch, head.rcu);
6980         for (i = 0; i < gcb->head.cnt; i++)
6981                 nft_set_elem_destroy(gcb->head.set, gcb->elems[i], true);
6982         kfree(gcb);
6983 }
6984
6985 struct nft_set_gc_batch *nft_set_gc_batch_alloc(const struct nft_set *set,
6986                                                 gfp_t gfp)
6987 {
6988         struct nft_set_gc_batch *gcb;
6989
6990         gcb = kzalloc(sizeof(*gcb), gfp);
6991         if (gcb == NULL)
6992                 return gcb;
6993         gcb->head.set = set;
6994         return gcb;
6995 }
6996
6997 /*
6998  * Stateful objects
6999  */
7000
7001 /**
7002  *      nft_register_obj- register nf_tables stateful object type
7003  *      @obj_type: object type
7004  *
7005  *      Registers the object type for use with nf_tables. Returns zero on
7006  *      success or a negative errno code otherwise.
7007  */
7008 int nft_register_obj(struct nft_object_type *obj_type)
7009 {
7010         if (obj_type->type == NFT_OBJECT_UNSPEC)
7011                 return -EINVAL;
7012
7013         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7014         list_add_rcu(&obj_type->list, &nf_tables_objects);
7015         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7016         return 0;
7017 }
7018 EXPORT_SYMBOL_GPL(nft_register_obj);
7019
7020 /**
7021  *      nft_unregister_obj - unregister nf_tables object type
7022  *      @obj_type: object type
7023  *
7024  *      Unregisters the object type for use with nf_tables.
7025  */
7026 void nft_unregister_obj(struct nft_object_type *obj_type)
7027 {
7028         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7029         list_del_rcu(&obj_type->list);
7030         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7031 }
7032 EXPORT_SYMBOL_GPL(nft_unregister_obj);
7033
7034 struct nft_object *nft_obj_lookup(const struct net *net,
7035                                   const struct nft_table *table,
7036                                   const struct nlattr *nla, u32 objtype,
7037                                   u8 genmask)
7038 {
7039         struct nft_object_hash_key k = { .table = table };
7040         char search[NFT_OBJ_MAXNAMELEN];
7041         struct rhlist_head *tmp, *list;
7042         struct nft_object *obj;
7043
7044         nla_strscpy(search, nla, sizeof(search));
7045         k.name = search;
7046
7047         WARN_ON_ONCE(!rcu_read_lock_held() &&
7048                      !lockdep_commit_lock_is_held(net));
7049
7050         rcu_read_lock();
7051         list = rhltable_lookup(&nft_objname_ht, &k, nft_objname_ht_params);
7052         if (!list)
7053                 goto out;
7054
7055         rhl_for_each_entry_rcu(obj, tmp, list, rhlhead) {
7056                 if (objtype == obj->ops->type->type &&
7057                     nft_active_genmask(obj, genmask)) {
7058                         rcu_read_unlock();
7059                         return obj;
7060                 }
7061         }
7062 out:
7063         rcu_read_unlock();
7064         return ERR_PTR(-ENOENT);
7065 }
7066 EXPORT_SYMBOL_GPL(nft_obj_lookup);
7067
7068 static struct nft_object *nft_obj_lookup_byhandle(const struct nft_table *table,
7069                                                   const struct nlattr *nla,
7070                                                   u32 objtype, u8 genmask)
7071 {
7072         struct nft_object *obj;
7073
7074         list_for_each_entry(obj, &table->objects, list) {
7075                 if (be64_to_cpu(nla_get_be64(nla)) == obj->handle &&
7076                     objtype == obj->ops->type->type &&
7077                     nft_active_genmask(obj, genmask))
7078                         return obj;
7079         }
7080         return ERR_PTR(-ENOENT);
7081 }
7082
7083 static const struct nla_policy nft_obj_policy[NFTA_OBJ_MAX + 1] = {
7084         [NFTA_OBJ_TABLE]        = { .type = NLA_STRING,
7085                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
7086         [NFTA_OBJ_NAME]         = { .type = NLA_STRING,
7087                                     .len = NFT_OBJ_MAXNAMELEN - 1 },
7088         [NFTA_OBJ_TYPE]         = { .type = NLA_U32 },
7089         [NFTA_OBJ_DATA]         = { .type = NLA_NESTED },
7090         [NFTA_OBJ_HANDLE]       = { .type = NLA_U64},
7091         [NFTA_OBJ_USERDATA]     = { .type = NLA_BINARY,
7092                                     .len = NFT_USERDATA_MAXLEN },
7093 };
7094
7095 static struct nft_object *nft_obj_init(const struct nft_ctx *ctx,
7096                                        const struct nft_object_type *type,
7097                                        const struct nlattr *attr)
7098 {
7099         struct nlattr **tb;
7100         const struct nft_object_ops *ops;
7101         struct nft_object *obj;
7102         int err = -ENOMEM;
7103
7104         tb = kmalloc_array(type->maxattr + 1, sizeof(*tb), GFP_KERNEL);
7105         if (!tb)
7106                 goto err1;
7107
7108         if (attr) {
7109                 err = nla_parse_nested_deprecated(tb, type->maxattr, attr,
7110                                                   type->policy, NULL);
7111                 if (err < 0)
7112                         goto err2;
7113         } else {
7114                 memset(tb, 0, sizeof(tb[0]) * (type->maxattr + 1));
7115         }
7116
7117         if (type->select_ops) {
7118                 ops = type->select_ops(ctx, (const struct nlattr * const *)tb);
7119                 if (IS_ERR(ops)) {
7120                         err = PTR_ERR(ops);
7121                         goto err2;
7122                 }
7123         } else {
7124                 ops = type->ops;
7125         }
7126
7127         err = -ENOMEM;
7128         obj = kzalloc(sizeof(*obj) + ops->size, GFP_KERNEL_ACCOUNT);
7129         if (!obj)
7130                 goto err2;
7131
7132         err = ops->init(ctx, (const struct nlattr * const *)tb, obj);
7133         if (err < 0)
7134                 goto err3;
7135
7136         obj->ops = ops;
7137
7138         kfree(tb);
7139         return obj;
7140 err3:
7141         kfree(obj);
7142 err2:
7143         kfree(tb);
7144 err1:
7145         return ERR_PTR(err);
7146 }
7147
7148 static int nft_object_dump(struct sk_buff *skb, unsigned int attr,
7149                            struct nft_object *obj, bool reset)
7150 {
7151         struct nlattr *nest;
7152
7153         nest = nla_nest_start_noflag(skb, attr);
7154         if (!nest)
7155                 goto nla_put_failure;
7156         if (obj->ops->dump(skb, obj, reset) < 0)
7157                 goto nla_put_failure;
7158         nla_nest_end(skb, nest);
7159         return 0;
7160
7161 nla_put_failure:
7162         return -1;
7163 }
7164
7165 static const struct nft_object_type *__nft_obj_type_get(u32 objtype)
7166 {
7167         const struct nft_object_type *type;
7168
7169         list_for_each_entry(type, &nf_tables_objects, list) {
7170                 if (objtype == type->type)
7171                         return type;
7172         }
7173         return NULL;
7174 }
7175
7176 static const struct nft_object_type *
7177 nft_obj_type_get(struct net *net, u32 objtype)
7178 {
7179         const struct nft_object_type *type;
7180
7181         type = __nft_obj_type_get(objtype);
7182         if (type != NULL && try_module_get(type->owner))
7183                 return type;
7184
7185         lockdep_nfnl_nft_mutex_not_held();
7186 #ifdef CONFIG_MODULES
7187         if (type == NULL) {
7188                 if (nft_request_module(net, "nft-obj-%u", objtype) == -EAGAIN)
7189                         return ERR_PTR(-EAGAIN);
7190         }
7191 #endif
7192         return ERR_PTR(-ENOENT);
7193 }
7194
7195 static int nf_tables_updobj(const struct nft_ctx *ctx,
7196                             const struct nft_object_type *type,
7197                             const struct nlattr *attr,
7198                             struct nft_object *obj)
7199 {
7200         struct nft_object *newobj;
7201         struct nft_trans *trans;
7202         int err = -ENOMEM;
7203
7204         if (!try_module_get(type->owner))
7205                 return -ENOENT;
7206
7207         trans = nft_trans_alloc(ctx, NFT_MSG_NEWOBJ,
7208                                 sizeof(struct nft_trans_obj));
7209         if (!trans)
7210                 goto err_trans;
7211
7212         newobj = nft_obj_init(ctx, type, attr);
7213         if (IS_ERR(newobj)) {
7214                 err = PTR_ERR(newobj);
7215                 goto err_free_trans;
7216         }
7217
7218         nft_trans_obj(trans) = obj;
7219         nft_trans_obj_update(trans) = true;
7220         nft_trans_obj_newobj(trans) = newobj;
7221         nft_trans_commit_list_add_tail(ctx->net, trans);
7222
7223         return 0;
7224
7225 err_free_trans:
7226         kfree(trans);
7227 err_trans:
7228         module_put(type->owner);
7229         return err;
7230 }
7231
7232 static int nf_tables_newobj(struct sk_buff *skb, const struct nfnl_info *info,
7233                             const struct nlattr * const nla[])
7234 {
7235         struct netlink_ext_ack *extack = info->extack;
7236         u8 genmask = nft_genmask_next(info->net);
7237         u8 family = info->nfmsg->nfgen_family;
7238         const struct nft_object_type *type;
7239         struct net *net = info->net;
7240         struct nft_table *table;
7241         struct nft_object *obj;
7242         struct nft_ctx ctx;
7243         u32 objtype;
7244         int err;
7245
7246         if (!nla[NFTA_OBJ_TYPE] ||
7247             !nla[NFTA_OBJ_NAME] ||
7248             !nla[NFTA_OBJ_DATA])
7249                 return -EINVAL;
7250
7251         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
7252                                  NETLINK_CB(skb).portid);
7253         if (IS_ERR(table)) {
7254                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7255                 return PTR_ERR(table);
7256         }
7257
7258         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7259         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
7260         if (IS_ERR(obj)) {
7261                 err = PTR_ERR(obj);
7262                 if (err != -ENOENT) {
7263                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7264                         return err;
7265                 }
7266         } else {
7267                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
7268                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7269                         return -EEXIST;
7270                 }
7271                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
7272                         return -EOPNOTSUPP;
7273
7274                 type = __nft_obj_type_get(objtype);
7275                 if (WARN_ON_ONCE(!type))
7276                         return -ENOENT;
7277
7278                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7279
7280                 return nf_tables_updobj(&ctx, type, nla[NFTA_OBJ_DATA], obj);
7281         }
7282
7283         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7284
7285         type = nft_obj_type_get(net, objtype);
7286         if (IS_ERR(type))
7287                 return PTR_ERR(type);
7288
7289         obj = nft_obj_init(&ctx, type, nla[NFTA_OBJ_DATA]);
7290         if (IS_ERR(obj)) {
7291                 err = PTR_ERR(obj);
7292                 goto err_init;
7293         }
7294         obj->key.table = table;
7295         obj->handle = nf_tables_alloc_handle(table);
7296
7297         obj->key.name = nla_strdup(nla[NFTA_OBJ_NAME], GFP_KERNEL_ACCOUNT);
7298         if (!obj->key.name) {
7299                 err = -ENOMEM;
7300                 goto err_strdup;
7301         }
7302
7303         if (nla[NFTA_OBJ_USERDATA]) {
7304                 obj->udata = nla_memdup(nla[NFTA_OBJ_USERDATA], GFP_KERNEL_ACCOUNT);
7305                 if (obj->udata == NULL)
7306                         goto err_userdata;
7307
7308                 obj->udlen = nla_len(nla[NFTA_OBJ_USERDATA]);
7309         }
7310
7311         err = nft_trans_obj_add(&ctx, NFT_MSG_NEWOBJ, obj);
7312         if (err < 0)
7313                 goto err_trans;
7314
7315         err = rhltable_insert(&nft_objname_ht, &obj->rhlhead,
7316                               nft_objname_ht_params);
7317         if (err < 0)
7318                 goto err_obj_ht;
7319
7320         list_add_tail_rcu(&obj->list, &table->objects);
7321         table->use++;
7322         return 0;
7323 err_obj_ht:
7324         /* queued in transaction log */
7325         INIT_LIST_HEAD(&obj->list);
7326         return err;
7327 err_trans:
7328         kfree(obj->udata);
7329 err_userdata:
7330         kfree(obj->key.name);
7331 err_strdup:
7332         if (obj->ops->destroy)
7333                 obj->ops->destroy(&ctx, obj);
7334         kfree(obj);
7335 err_init:
7336         module_put(type->owner);
7337         return err;
7338 }
7339
7340 static int nf_tables_fill_obj_info(struct sk_buff *skb, struct net *net,
7341                                    u32 portid, u32 seq, int event, u32 flags,
7342                                    int family, const struct nft_table *table,
7343                                    struct nft_object *obj, bool reset)
7344 {
7345         struct nlmsghdr *nlh;
7346
7347         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
7348         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
7349                            NFNETLINK_V0, nft_base_seq(net));
7350         if (!nlh)
7351                 goto nla_put_failure;
7352
7353         if (nla_put_string(skb, NFTA_OBJ_TABLE, table->name) ||
7354             nla_put_string(skb, NFTA_OBJ_NAME, obj->key.name) ||
7355             nla_put_be64(skb, NFTA_OBJ_HANDLE, cpu_to_be64(obj->handle),
7356                          NFTA_OBJ_PAD))
7357                 goto nla_put_failure;
7358
7359         if (event == NFT_MSG_DELOBJ) {
7360                 nlmsg_end(skb, nlh);
7361                 return 0;
7362         }
7363
7364         if (nla_put_be32(skb, NFTA_OBJ_TYPE, htonl(obj->ops->type->type)) ||
7365             nla_put_be32(skb, NFTA_OBJ_USE, htonl(obj->use)) ||
7366             nft_object_dump(skb, NFTA_OBJ_DATA, obj, reset))
7367                 goto nla_put_failure;
7368
7369         if (obj->udata &&
7370             nla_put(skb, NFTA_OBJ_USERDATA, obj->udlen, obj->udata))
7371                 goto nla_put_failure;
7372
7373         nlmsg_end(skb, nlh);
7374         return 0;
7375
7376 nla_put_failure:
7377         nlmsg_trim(skb, nlh);
7378         return -1;
7379 }
7380
7381 struct nft_obj_filter {
7382         char            *table;
7383         u32             type;
7384 };
7385
7386 static int nf_tables_dump_obj(struct sk_buff *skb, struct netlink_callback *cb)
7387 {
7388         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
7389         const struct nft_table *table;
7390         unsigned int idx = 0, s_idx = cb->args[0];
7391         struct nft_obj_filter *filter = cb->data;
7392         struct net *net = sock_net(skb->sk);
7393         int family = nfmsg->nfgen_family;
7394         struct nftables_pernet *nft_net;
7395         struct nft_object *obj;
7396         bool reset = false;
7397
7398         if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
7399                 reset = true;
7400
7401         rcu_read_lock();
7402         nft_net = nft_pernet(net);
7403         cb->seq = READ_ONCE(nft_net->base_seq);
7404
7405         list_for_each_entry_rcu(table, &nft_net->tables, list) {
7406                 if (family != NFPROTO_UNSPEC && family != table->family)
7407                         continue;
7408
7409                 list_for_each_entry_rcu(obj, &table->objects, list) {
7410                         if (!nft_is_active(net, obj))
7411                                 goto cont;
7412                         if (idx < s_idx)
7413                                 goto cont;
7414                         if (idx > s_idx)
7415                                 memset(&cb->args[1], 0,
7416                                        sizeof(cb->args) - sizeof(cb->args[0]));
7417                         if (filter && filter->table &&
7418                             strcmp(filter->table, table->name))
7419                                 goto cont;
7420                         if (filter &&
7421                             filter->type != NFT_OBJECT_UNSPEC &&
7422                             obj->ops->type->type != filter->type)
7423                                 goto cont;
7424                         if (reset) {
7425                                 char *buf = kasprintf(GFP_ATOMIC,
7426                                                       "%s:%u",
7427                                                       table->name,
7428                                                       nft_net->base_seq);
7429
7430                                 audit_log_nfcfg(buf,
7431                                                 family,
7432                                                 obj->handle,
7433                                                 AUDIT_NFT_OP_OBJ_RESET,
7434                                                 GFP_ATOMIC);
7435                                 kfree(buf);
7436                         }
7437
7438                         if (nf_tables_fill_obj_info(skb, net, NETLINK_CB(cb->skb).portid,
7439                                                     cb->nlh->nlmsg_seq,
7440                                                     NFT_MSG_NEWOBJ,
7441                                                     NLM_F_MULTI | NLM_F_APPEND,
7442                                                     table->family, table,
7443                                                     obj, reset) < 0)
7444                                 goto done;
7445
7446                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
7447 cont:
7448                         idx++;
7449                 }
7450         }
7451 done:
7452         rcu_read_unlock();
7453
7454         cb->args[0] = idx;
7455         return skb->len;
7456 }
7457
7458 static int nf_tables_dump_obj_start(struct netlink_callback *cb)
7459 {
7460         const struct nlattr * const *nla = cb->data;
7461         struct nft_obj_filter *filter = NULL;
7462
7463         if (nla[NFTA_OBJ_TABLE] || nla[NFTA_OBJ_TYPE]) {
7464                 filter = kzalloc(sizeof(*filter), GFP_ATOMIC);
7465                 if (!filter)
7466                         return -ENOMEM;
7467
7468                 if (nla[NFTA_OBJ_TABLE]) {
7469                         filter->table = nla_strdup(nla[NFTA_OBJ_TABLE], GFP_ATOMIC);
7470                         if (!filter->table) {
7471                                 kfree(filter);
7472                                 return -ENOMEM;
7473                         }
7474                 }
7475
7476                 if (nla[NFTA_OBJ_TYPE])
7477                         filter->type = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7478         }
7479
7480         cb->data = filter;
7481         return 0;
7482 }
7483
7484 static int nf_tables_dump_obj_done(struct netlink_callback *cb)
7485 {
7486         struct nft_obj_filter *filter = cb->data;
7487
7488         if (filter) {
7489                 kfree(filter->table);
7490                 kfree(filter);
7491         }
7492
7493         return 0;
7494 }
7495
7496 /* called with rcu_read_lock held */
7497 static int nf_tables_getobj(struct sk_buff *skb, const struct nfnl_info *info,
7498                             const struct nlattr * const nla[])
7499 {
7500         struct netlink_ext_ack *extack = info->extack;
7501         u8 genmask = nft_genmask_cur(info->net);
7502         u8 family = info->nfmsg->nfgen_family;
7503         const struct nft_table *table;
7504         struct net *net = info->net;
7505         struct nft_object *obj;
7506         struct sk_buff *skb2;
7507         bool reset = false;
7508         u32 objtype;
7509         int err;
7510
7511         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
7512                 struct netlink_dump_control c = {
7513                         .start = nf_tables_dump_obj_start,
7514                         .dump = nf_tables_dump_obj,
7515                         .done = nf_tables_dump_obj_done,
7516                         .module = THIS_MODULE,
7517                         .data = (void *)nla,
7518                 };
7519
7520                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
7521         }
7522
7523         if (!nla[NFTA_OBJ_NAME] ||
7524             !nla[NFTA_OBJ_TYPE])
7525                 return -EINVAL;
7526
7527         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask, 0);
7528         if (IS_ERR(table)) {
7529                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7530                 return PTR_ERR(table);
7531         }
7532
7533         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7534         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
7535         if (IS_ERR(obj)) {
7536                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7537                 return PTR_ERR(obj);
7538         }
7539
7540         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
7541         if (!skb2)
7542                 return -ENOMEM;
7543
7544         if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
7545                 reset = true;
7546
7547         if (reset) {
7548                 const struct nftables_pernet *nft_net;
7549                 char *buf;
7550
7551                 nft_net = nft_pernet(net);
7552                 buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, nft_net->base_seq);
7553
7554                 audit_log_nfcfg(buf,
7555                                 family,
7556                                 obj->handle,
7557                                 AUDIT_NFT_OP_OBJ_RESET,
7558                                 GFP_ATOMIC);
7559                 kfree(buf);
7560         }
7561
7562         err = nf_tables_fill_obj_info(skb2, net, NETLINK_CB(skb).portid,
7563                                       info->nlh->nlmsg_seq, NFT_MSG_NEWOBJ, 0,
7564                                       family, table, obj, reset);
7565         if (err < 0)
7566                 goto err_fill_obj_info;
7567
7568         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
7569
7570 err_fill_obj_info:
7571         kfree_skb(skb2);
7572         return err;
7573 }
7574
7575 static void nft_obj_destroy(const struct nft_ctx *ctx, struct nft_object *obj)
7576 {
7577         if (obj->ops->destroy)
7578                 obj->ops->destroy(ctx, obj);
7579
7580         module_put(obj->ops->type->owner);
7581         kfree(obj->key.name);
7582         kfree(obj->udata);
7583         kfree(obj);
7584 }
7585
7586 static int nf_tables_delobj(struct sk_buff *skb, const struct nfnl_info *info,
7587                             const struct nlattr * const nla[])
7588 {
7589         struct netlink_ext_ack *extack = info->extack;
7590         u8 genmask = nft_genmask_next(info->net);
7591         u8 family = info->nfmsg->nfgen_family;
7592         struct net *net = info->net;
7593         const struct nlattr *attr;
7594         struct nft_table *table;
7595         struct nft_object *obj;
7596         struct nft_ctx ctx;
7597         u32 objtype;
7598
7599         if (!nla[NFTA_OBJ_TYPE] ||
7600             (!nla[NFTA_OBJ_NAME] && !nla[NFTA_OBJ_HANDLE]))
7601                 return -EINVAL;
7602
7603         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
7604                                  NETLINK_CB(skb).portid);
7605         if (IS_ERR(table)) {
7606                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7607                 return PTR_ERR(table);
7608         }
7609
7610         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7611         if (nla[NFTA_OBJ_HANDLE]) {
7612                 attr = nla[NFTA_OBJ_HANDLE];
7613                 obj = nft_obj_lookup_byhandle(table, attr, objtype, genmask);
7614         } else {
7615                 attr = nla[NFTA_OBJ_NAME];
7616                 obj = nft_obj_lookup(net, table, attr, objtype, genmask);
7617         }
7618
7619         if (IS_ERR(obj)) {
7620                 if (PTR_ERR(obj) == -ENOENT &&
7621                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYOBJ)
7622                         return 0;
7623
7624                 NL_SET_BAD_ATTR(extack, attr);
7625                 return PTR_ERR(obj);
7626         }
7627         if (obj->use > 0) {
7628                 NL_SET_BAD_ATTR(extack, attr);
7629                 return -EBUSY;
7630         }
7631
7632         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7633
7634         return nft_delobj(&ctx, obj);
7635 }
7636
7637 void nft_obj_notify(struct net *net, const struct nft_table *table,
7638                     struct nft_object *obj, u32 portid, u32 seq, int event,
7639                     u16 flags, int family, int report, gfp_t gfp)
7640 {
7641         struct nftables_pernet *nft_net = nft_pernet(net);
7642         struct sk_buff *skb;
7643         int err;
7644         char *buf = kasprintf(gfp, "%s:%u",
7645                               table->name, nft_net->base_seq);
7646
7647         audit_log_nfcfg(buf,
7648                         family,
7649                         obj->handle,
7650                         event == NFT_MSG_NEWOBJ ?
7651                                  AUDIT_NFT_OP_OBJ_REGISTER :
7652                                  AUDIT_NFT_OP_OBJ_UNREGISTER,
7653                         gfp);
7654         kfree(buf);
7655
7656         if (!report &&
7657             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
7658                 return;
7659
7660         skb = nlmsg_new(NLMSG_GOODSIZE, gfp);
7661         if (skb == NULL)
7662                 goto err;
7663
7664         err = nf_tables_fill_obj_info(skb, net, portid, seq, event,
7665                                       flags & (NLM_F_CREATE | NLM_F_EXCL),
7666                                       family, table, obj, false);
7667         if (err < 0) {
7668                 kfree_skb(skb);
7669                 goto err;
7670         }
7671
7672         nft_notify_enqueue(skb, report, &nft_net->notify_list);
7673         return;
7674 err:
7675         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
7676 }
7677 EXPORT_SYMBOL_GPL(nft_obj_notify);
7678
7679 static void nf_tables_obj_notify(const struct nft_ctx *ctx,
7680                                  struct nft_object *obj, int event)
7681 {
7682         nft_obj_notify(ctx->net, ctx->table, obj, ctx->portid, ctx->seq, event,
7683                        ctx->flags, ctx->family, ctx->report, GFP_KERNEL);
7684 }
7685
7686 /*
7687  * Flow tables
7688  */
7689 void nft_register_flowtable_type(struct nf_flowtable_type *type)
7690 {
7691         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7692         list_add_tail_rcu(&type->list, &nf_tables_flowtables);
7693         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7694 }
7695 EXPORT_SYMBOL_GPL(nft_register_flowtable_type);
7696
7697 void nft_unregister_flowtable_type(struct nf_flowtable_type *type)
7698 {
7699         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7700         list_del_rcu(&type->list);
7701         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7702 }
7703 EXPORT_SYMBOL_GPL(nft_unregister_flowtable_type);
7704
7705 static const struct nla_policy nft_flowtable_policy[NFTA_FLOWTABLE_MAX + 1] = {
7706         [NFTA_FLOWTABLE_TABLE]          = { .type = NLA_STRING,
7707                                             .len = NFT_NAME_MAXLEN - 1 },
7708         [NFTA_FLOWTABLE_NAME]           = { .type = NLA_STRING,
7709                                             .len = NFT_NAME_MAXLEN - 1 },
7710         [NFTA_FLOWTABLE_HOOK]           = { .type = NLA_NESTED },
7711         [NFTA_FLOWTABLE_HANDLE]         = { .type = NLA_U64 },
7712         [NFTA_FLOWTABLE_FLAGS]          = { .type = NLA_U32 },
7713 };
7714
7715 struct nft_flowtable *nft_flowtable_lookup(const struct nft_table *table,
7716                                            const struct nlattr *nla, u8 genmask)
7717 {
7718         struct nft_flowtable *flowtable;
7719
7720         list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
7721                 if (!nla_strcmp(nla, flowtable->name) &&
7722                     nft_active_genmask(flowtable, genmask))
7723                         return flowtable;
7724         }
7725         return ERR_PTR(-ENOENT);
7726 }
7727 EXPORT_SYMBOL_GPL(nft_flowtable_lookup);
7728
7729 void nf_tables_deactivate_flowtable(const struct nft_ctx *ctx,
7730                                     struct nft_flowtable *flowtable,
7731                                     enum nft_trans_phase phase)
7732 {
7733         switch (phase) {
7734         case NFT_TRANS_PREPARE_ERROR:
7735         case NFT_TRANS_PREPARE:
7736         case NFT_TRANS_ABORT:
7737         case NFT_TRANS_RELEASE:
7738                 flowtable->use--;
7739                 fallthrough;
7740         default:
7741                 return;
7742         }
7743 }
7744 EXPORT_SYMBOL_GPL(nf_tables_deactivate_flowtable);
7745
7746 static struct nft_flowtable *
7747 nft_flowtable_lookup_byhandle(const struct nft_table *table,
7748                               const struct nlattr *nla, u8 genmask)
7749 {
7750        struct nft_flowtable *flowtable;
7751
7752        list_for_each_entry(flowtable, &table->flowtables, list) {
7753                if (be64_to_cpu(nla_get_be64(nla)) == flowtable->handle &&
7754                    nft_active_genmask(flowtable, genmask))
7755                        return flowtable;
7756        }
7757        return ERR_PTR(-ENOENT);
7758 }
7759
7760 struct nft_flowtable_hook {
7761         u32                     num;
7762         int                     priority;
7763         struct list_head        list;
7764 };
7765
7766 static const struct nla_policy nft_flowtable_hook_policy[NFTA_FLOWTABLE_HOOK_MAX + 1] = {
7767         [NFTA_FLOWTABLE_HOOK_NUM]       = { .type = NLA_U32 },
7768         [NFTA_FLOWTABLE_HOOK_PRIORITY]  = { .type = NLA_U32 },
7769         [NFTA_FLOWTABLE_HOOK_DEVS]      = { .type = NLA_NESTED },
7770 };
7771
7772 static int nft_flowtable_parse_hook(const struct nft_ctx *ctx,
7773                                     const struct nlattr * const nla[],
7774                                     struct nft_flowtable_hook *flowtable_hook,
7775                                     struct nft_flowtable *flowtable,
7776                                     struct netlink_ext_ack *extack, bool add)
7777 {
7778         struct nlattr *tb[NFTA_FLOWTABLE_HOOK_MAX + 1];
7779         struct nft_hook *hook;
7780         int hooknum, priority;
7781         int err;
7782
7783         INIT_LIST_HEAD(&flowtable_hook->list);
7784
7785         err = nla_parse_nested_deprecated(tb, NFTA_FLOWTABLE_HOOK_MAX,
7786                                           nla[NFTA_FLOWTABLE_HOOK],
7787                                           nft_flowtable_hook_policy, NULL);
7788         if (err < 0)
7789                 return err;
7790
7791         if (add) {
7792                 if (!tb[NFTA_FLOWTABLE_HOOK_NUM] ||
7793                     !tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
7794                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
7795                         return -ENOENT;
7796                 }
7797
7798                 hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
7799                 if (hooknum != NF_NETDEV_INGRESS)
7800                         return -EOPNOTSUPP;
7801
7802                 priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
7803
7804                 flowtable_hook->priority        = priority;
7805                 flowtable_hook->num             = hooknum;
7806         } else {
7807                 if (tb[NFTA_FLOWTABLE_HOOK_NUM]) {
7808                         hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
7809                         if (hooknum != flowtable->hooknum)
7810                                 return -EOPNOTSUPP;
7811                 }
7812
7813                 if (tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
7814                         priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
7815                         if (priority != flowtable->data.priority)
7816                                 return -EOPNOTSUPP;
7817                 }
7818
7819                 flowtable_hook->priority        = flowtable->data.priority;
7820                 flowtable_hook->num             = flowtable->hooknum;
7821         }
7822
7823         if (tb[NFTA_FLOWTABLE_HOOK_DEVS]) {
7824                 err = nf_tables_parse_netdev_hooks(ctx->net,
7825                                                    tb[NFTA_FLOWTABLE_HOOK_DEVS],
7826                                                    &flowtable_hook->list,
7827                                                    extack);
7828                 if (err < 0)
7829                         return err;
7830         }
7831
7832         list_for_each_entry(hook, &flowtable_hook->list, list) {
7833                 hook->ops.pf            = NFPROTO_NETDEV;
7834                 hook->ops.hooknum       = flowtable_hook->num;
7835                 hook->ops.priority      = flowtable_hook->priority;
7836                 hook->ops.priv          = &flowtable->data;
7837                 hook->ops.hook          = flowtable->data.type->hook;
7838         }
7839
7840         return err;
7841 }
7842
7843 static const struct nf_flowtable_type *__nft_flowtable_type_get(u8 family)
7844 {
7845         const struct nf_flowtable_type *type;
7846
7847         list_for_each_entry(type, &nf_tables_flowtables, list) {
7848                 if (family == type->family)
7849                         return type;
7850         }
7851         return NULL;
7852 }
7853
7854 static const struct nf_flowtable_type *
7855 nft_flowtable_type_get(struct net *net, u8 family)
7856 {
7857         const struct nf_flowtable_type *type;
7858
7859         type = __nft_flowtable_type_get(family);
7860         if (type != NULL && try_module_get(type->owner))
7861                 return type;
7862
7863         lockdep_nfnl_nft_mutex_not_held();
7864 #ifdef CONFIG_MODULES
7865         if (type == NULL) {
7866                 if (nft_request_module(net, "nf-flowtable-%u", family) == -EAGAIN)
7867                         return ERR_PTR(-EAGAIN);
7868         }
7869 #endif
7870         return ERR_PTR(-ENOENT);
7871 }
7872
7873 /* Only called from error and netdev event paths. */
7874 static void nft_unregister_flowtable_hook(struct net *net,
7875                                           struct nft_flowtable *flowtable,
7876                                           struct nft_hook *hook)
7877 {
7878         nf_unregister_net_hook(net, &hook->ops);
7879         flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
7880                                     FLOW_BLOCK_UNBIND);
7881 }
7882
7883 static void __nft_unregister_flowtable_net_hooks(struct net *net,
7884                                                  struct list_head *hook_list,
7885                                                  bool release_netdev)
7886 {
7887         struct nft_hook *hook, *next;
7888
7889         list_for_each_entry_safe(hook, next, hook_list, list) {
7890                 nf_unregister_net_hook(net, &hook->ops);
7891                 if (release_netdev) {
7892                         list_del(&hook->list);
7893                         kfree_rcu(hook, rcu);
7894                 }
7895         }
7896 }
7897
7898 static void nft_unregister_flowtable_net_hooks(struct net *net,
7899                                                struct list_head *hook_list)
7900 {
7901         __nft_unregister_flowtable_net_hooks(net, hook_list, false);
7902 }
7903
7904 static int nft_register_flowtable_net_hooks(struct net *net,
7905                                             struct nft_table *table,
7906                                             struct list_head *hook_list,
7907                                             struct nft_flowtable *flowtable)
7908 {
7909         struct nft_hook *hook, *hook2, *next;
7910         struct nft_flowtable *ft;
7911         int err, i = 0;
7912
7913         list_for_each_entry(hook, hook_list, list) {
7914                 list_for_each_entry(ft, &table->flowtables, list) {
7915                         if (!nft_is_active_next(net, ft))
7916                                 continue;
7917
7918                         list_for_each_entry(hook2, &ft->hook_list, list) {
7919                                 if (hook->ops.dev == hook2->ops.dev &&
7920                                     hook->ops.pf == hook2->ops.pf) {
7921                                         err = -EEXIST;
7922                                         goto err_unregister_net_hooks;
7923                                 }
7924                         }
7925                 }
7926
7927                 err = flowtable->data.type->setup(&flowtable->data,
7928                                                   hook->ops.dev,
7929                                                   FLOW_BLOCK_BIND);
7930                 if (err < 0)
7931                         goto err_unregister_net_hooks;
7932
7933                 err = nf_register_net_hook(net, &hook->ops);
7934                 if (err < 0) {
7935                         flowtable->data.type->setup(&flowtable->data,
7936                                                     hook->ops.dev,
7937                                                     FLOW_BLOCK_UNBIND);
7938                         goto err_unregister_net_hooks;
7939                 }
7940
7941                 i++;
7942         }
7943
7944         return 0;
7945
7946 err_unregister_net_hooks:
7947         list_for_each_entry_safe(hook, next, hook_list, list) {
7948                 if (i-- <= 0)
7949                         break;
7950
7951                 nft_unregister_flowtable_hook(net, flowtable, hook);
7952                 list_del_rcu(&hook->list);
7953                 kfree_rcu(hook, rcu);
7954         }
7955
7956         return err;
7957 }
7958
7959 static void nft_hooks_destroy(struct list_head *hook_list)
7960 {
7961         struct nft_hook *hook, *next;
7962
7963         list_for_each_entry_safe(hook, next, hook_list, list) {
7964                 list_del_rcu(&hook->list);
7965                 kfree_rcu(hook, rcu);
7966         }
7967 }
7968
7969 static int nft_flowtable_update(struct nft_ctx *ctx, const struct nlmsghdr *nlh,
7970                                 struct nft_flowtable *flowtable,
7971                                 struct netlink_ext_ack *extack)
7972 {
7973         const struct nlattr * const *nla = ctx->nla;
7974         struct nft_flowtable_hook flowtable_hook;
7975         struct nft_hook *hook, *next;
7976         struct nft_trans *trans;
7977         bool unregister = false;
7978         u32 flags;
7979         int err;
7980
7981         err = nft_flowtable_parse_hook(ctx, nla, &flowtable_hook, flowtable,
7982                                        extack, false);
7983         if (err < 0)
7984                 return err;
7985
7986         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
7987                 if (nft_hook_list_find(&flowtable->hook_list, hook)) {
7988                         list_del(&hook->list);
7989                         kfree(hook);
7990                 }
7991         }
7992
7993         if (nla[NFTA_FLOWTABLE_FLAGS]) {
7994                 flags = ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
7995                 if (flags & ~NFT_FLOWTABLE_MASK) {
7996                         err = -EOPNOTSUPP;
7997                         goto err_flowtable_update_hook;
7998                 }
7999                 if ((flowtable->data.flags & NFT_FLOWTABLE_HW_OFFLOAD) ^
8000                     (flags & NFT_FLOWTABLE_HW_OFFLOAD)) {
8001                         err = -EOPNOTSUPP;
8002                         goto err_flowtable_update_hook;
8003                 }
8004         } else {
8005                 flags = flowtable->data.flags;
8006         }
8007
8008         err = nft_register_flowtable_net_hooks(ctx->net, ctx->table,
8009                                                &flowtable_hook.list, flowtable);
8010         if (err < 0)
8011                 goto err_flowtable_update_hook;
8012
8013         trans = nft_trans_alloc(ctx, NFT_MSG_NEWFLOWTABLE,
8014                                 sizeof(struct nft_trans_flowtable));
8015         if (!trans) {
8016                 unregister = true;
8017                 err = -ENOMEM;
8018                 goto err_flowtable_update_hook;
8019         }
8020
8021         nft_trans_flowtable_flags(trans) = flags;
8022         nft_trans_flowtable(trans) = flowtable;
8023         nft_trans_flowtable_update(trans) = true;
8024         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
8025         list_splice(&flowtable_hook.list, &nft_trans_flowtable_hooks(trans));
8026
8027         nft_trans_commit_list_add_tail(ctx->net, trans);
8028
8029         return 0;
8030
8031 err_flowtable_update_hook:
8032         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
8033                 if (unregister)
8034                         nft_unregister_flowtable_hook(ctx->net, flowtable, hook);
8035                 list_del_rcu(&hook->list);
8036                 kfree_rcu(hook, rcu);
8037         }
8038
8039         return err;
8040
8041 }
8042
8043 static int nf_tables_newflowtable(struct sk_buff *skb,
8044                                   const struct nfnl_info *info,
8045                                   const struct nlattr * const nla[])
8046 {
8047         struct netlink_ext_ack *extack = info->extack;
8048         struct nft_flowtable_hook flowtable_hook;
8049         u8 genmask = nft_genmask_next(info->net);
8050         u8 family = info->nfmsg->nfgen_family;
8051         const struct nf_flowtable_type *type;
8052         struct nft_flowtable *flowtable;
8053         struct nft_hook *hook, *next;
8054         struct net *net = info->net;
8055         struct nft_table *table;
8056         struct nft_ctx ctx;
8057         int err;
8058
8059         if (!nla[NFTA_FLOWTABLE_TABLE] ||
8060             !nla[NFTA_FLOWTABLE_NAME] ||
8061             !nla[NFTA_FLOWTABLE_HOOK])
8062                 return -EINVAL;
8063
8064         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8065                                  genmask, NETLINK_CB(skb).portid);
8066         if (IS_ERR(table)) {
8067                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
8068                 return PTR_ERR(table);
8069         }
8070
8071         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
8072                                          genmask);
8073         if (IS_ERR(flowtable)) {
8074                 err = PTR_ERR(flowtable);
8075                 if (err != -ENOENT) {
8076                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8077                         return err;
8078                 }
8079         } else {
8080                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
8081                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8082                         return -EEXIST;
8083                 }
8084
8085                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8086
8087                 return nft_flowtable_update(&ctx, info->nlh, flowtable, extack);
8088         }
8089
8090         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8091
8092         flowtable = kzalloc(sizeof(*flowtable), GFP_KERNEL_ACCOUNT);
8093         if (!flowtable)
8094                 return -ENOMEM;
8095
8096         flowtable->table = table;
8097         flowtable->handle = nf_tables_alloc_handle(table);
8098         INIT_LIST_HEAD(&flowtable->hook_list);
8099
8100         flowtable->name = nla_strdup(nla[NFTA_FLOWTABLE_NAME], GFP_KERNEL_ACCOUNT);
8101         if (!flowtable->name) {
8102                 err = -ENOMEM;
8103                 goto err1;
8104         }
8105
8106         type = nft_flowtable_type_get(net, family);
8107         if (IS_ERR(type)) {
8108                 err = PTR_ERR(type);
8109                 goto err2;
8110         }
8111
8112         if (nla[NFTA_FLOWTABLE_FLAGS]) {
8113                 flowtable->data.flags =
8114                         ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
8115                 if (flowtable->data.flags & ~NFT_FLOWTABLE_MASK) {
8116                         err = -EOPNOTSUPP;
8117                         goto err3;
8118                 }
8119         }
8120
8121         write_pnet(&flowtable->data.net, net);
8122         flowtable->data.type = type;
8123         err = type->init(&flowtable->data);
8124         if (err < 0)
8125                 goto err3;
8126
8127         err = nft_flowtable_parse_hook(&ctx, nla, &flowtable_hook, flowtable,
8128                                        extack, true);
8129         if (err < 0)
8130                 goto err4;
8131
8132         list_splice(&flowtable_hook.list, &flowtable->hook_list);
8133         flowtable->data.priority = flowtable_hook.priority;
8134         flowtable->hooknum = flowtable_hook.num;
8135
8136         err = nft_register_flowtable_net_hooks(ctx.net, table,
8137                                                &flowtable->hook_list,
8138                                                flowtable);
8139         if (err < 0) {
8140                 nft_hooks_destroy(&flowtable->hook_list);
8141                 goto err4;
8142         }
8143
8144         err = nft_trans_flowtable_add(&ctx, NFT_MSG_NEWFLOWTABLE, flowtable);
8145         if (err < 0)
8146                 goto err5;
8147
8148         list_add_tail_rcu(&flowtable->list, &table->flowtables);
8149         table->use++;
8150
8151         return 0;
8152 err5:
8153         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
8154                 nft_unregister_flowtable_hook(net, flowtable, hook);
8155                 list_del_rcu(&hook->list);
8156                 kfree_rcu(hook, rcu);
8157         }
8158 err4:
8159         flowtable->data.type->free(&flowtable->data);
8160 err3:
8161         module_put(type->owner);
8162 err2:
8163         kfree(flowtable->name);
8164 err1:
8165         kfree(flowtable);
8166         return err;
8167 }
8168
8169 static void nft_flowtable_hook_release(struct nft_flowtable_hook *flowtable_hook)
8170 {
8171         struct nft_hook *this, *next;
8172
8173         list_for_each_entry_safe(this, next, &flowtable_hook->list, list) {
8174                 list_del(&this->list);
8175                 kfree(this);
8176         }
8177 }
8178
8179 static int nft_delflowtable_hook(struct nft_ctx *ctx,
8180                                  struct nft_flowtable *flowtable,
8181                                  struct netlink_ext_ack *extack)
8182 {
8183         const struct nlattr * const *nla = ctx->nla;
8184         struct nft_flowtable_hook flowtable_hook;
8185         LIST_HEAD(flowtable_del_list);
8186         struct nft_hook *this, *hook;
8187         struct nft_trans *trans;
8188         int err;
8189
8190         err = nft_flowtable_parse_hook(ctx, nla, &flowtable_hook, flowtable,
8191                                        extack, false);
8192         if (err < 0)
8193                 return err;
8194
8195         list_for_each_entry(this, &flowtable_hook.list, list) {
8196                 hook = nft_hook_list_find(&flowtable->hook_list, this);
8197                 if (!hook) {
8198                         err = -ENOENT;
8199                         goto err_flowtable_del_hook;
8200                 }
8201                 list_move(&hook->list, &flowtable_del_list);
8202         }
8203
8204         trans = nft_trans_alloc(ctx, NFT_MSG_DELFLOWTABLE,
8205                                 sizeof(struct nft_trans_flowtable));
8206         if (!trans) {
8207                 err = -ENOMEM;
8208                 goto err_flowtable_del_hook;
8209         }
8210
8211         nft_trans_flowtable(trans) = flowtable;
8212         nft_trans_flowtable_update(trans) = true;
8213         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
8214         list_splice(&flowtable_del_list, &nft_trans_flowtable_hooks(trans));
8215         nft_flowtable_hook_release(&flowtable_hook);
8216
8217         nft_trans_commit_list_add_tail(ctx->net, trans);
8218
8219         return 0;
8220
8221 err_flowtable_del_hook:
8222         list_splice(&flowtable_del_list, &flowtable->hook_list);
8223         nft_flowtable_hook_release(&flowtable_hook);
8224
8225         return err;
8226 }
8227
8228 static int nf_tables_delflowtable(struct sk_buff *skb,
8229                                   const struct nfnl_info *info,
8230                                   const struct nlattr * const nla[])
8231 {
8232         struct netlink_ext_ack *extack = info->extack;
8233         u8 genmask = nft_genmask_next(info->net);
8234         u8 family = info->nfmsg->nfgen_family;
8235         struct nft_flowtable *flowtable;
8236         struct net *net = info->net;
8237         const struct nlattr *attr;
8238         struct nft_table *table;
8239         struct nft_ctx ctx;
8240
8241         if (!nla[NFTA_FLOWTABLE_TABLE] ||
8242             (!nla[NFTA_FLOWTABLE_NAME] &&
8243              !nla[NFTA_FLOWTABLE_HANDLE]))
8244                 return -EINVAL;
8245
8246         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8247                                  genmask, NETLINK_CB(skb).portid);
8248         if (IS_ERR(table)) {
8249                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
8250                 return PTR_ERR(table);
8251         }
8252
8253         if (nla[NFTA_FLOWTABLE_HANDLE]) {
8254                 attr = nla[NFTA_FLOWTABLE_HANDLE];
8255                 flowtable = nft_flowtable_lookup_byhandle(table, attr, genmask);
8256         } else {
8257                 attr = nla[NFTA_FLOWTABLE_NAME];
8258                 flowtable = nft_flowtable_lookup(table, attr, genmask);
8259         }
8260
8261         if (IS_ERR(flowtable)) {
8262                 if (PTR_ERR(flowtable) == -ENOENT &&
8263                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYFLOWTABLE)
8264                         return 0;
8265
8266                 NL_SET_BAD_ATTR(extack, attr);
8267                 return PTR_ERR(flowtable);
8268         }
8269
8270         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8271
8272         if (nla[NFTA_FLOWTABLE_HOOK])
8273                 return nft_delflowtable_hook(&ctx, flowtable, extack);
8274
8275         if (flowtable->use > 0) {
8276                 NL_SET_BAD_ATTR(extack, attr);
8277                 return -EBUSY;
8278         }
8279
8280         return nft_delflowtable(&ctx, flowtable);
8281 }
8282
8283 static int nf_tables_fill_flowtable_info(struct sk_buff *skb, struct net *net,
8284                                          u32 portid, u32 seq, int event,
8285                                          u32 flags, int family,
8286                                          struct nft_flowtable *flowtable,
8287                                          struct list_head *hook_list)
8288 {
8289         struct nlattr *nest, *nest_devs;
8290         struct nft_hook *hook;
8291         struct nlmsghdr *nlh;
8292
8293         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
8294         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
8295                            NFNETLINK_V0, nft_base_seq(net));
8296         if (!nlh)
8297                 goto nla_put_failure;
8298
8299         if (nla_put_string(skb, NFTA_FLOWTABLE_TABLE, flowtable->table->name) ||
8300             nla_put_string(skb, NFTA_FLOWTABLE_NAME, flowtable->name) ||
8301             nla_put_be64(skb, NFTA_FLOWTABLE_HANDLE, cpu_to_be64(flowtable->handle),
8302                          NFTA_FLOWTABLE_PAD))
8303                 goto nla_put_failure;
8304
8305         if (event == NFT_MSG_DELFLOWTABLE && !hook_list) {
8306                 nlmsg_end(skb, nlh);
8307                 return 0;
8308         }
8309
8310         if (nla_put_be32(skb, NFTA_FLOWTABLE_USE, htonl(flowtable->use)) ||
8311             nla_put_be32(skb, NFTA_FLOWTABLE_FLAGS, htonl(flowtable->data.flags)))
8312                 goto nla_put_failure;
8313
8314         nest = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK);
8315         if (!nest)
8316                 goto nla_put_failure;
8317         if (nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_NUM, htonl(flowtable->hooknum)) ||
8318             nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_PRIORITY, htonl(flowtable->data.priority)))
8319                 goto nla_put_failure;
8320
8321         nest_devs = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK_DEVS);
8322         if (!nest_devs)
8323                 goto nla_put_failure;
8324
8325         if (!hook_list)
8326                 hook_list = &flowtable->hook_list;
8327
8328         list_for_each_entry_rcu(hook, hook_list, list) {
8329                 if (nla_put_string(skb, NFTA_DEVICE_NAME, hook->ops.dev->name))
8330                         goto nla_put_failure;
8331         }
8332         nla_nest_end(skb, nest_devs);
8333         nla_nest_end(skb, nest);
8334
8335         nlmsg_end(skb, nlh);
8336         return 0;
8337
8338 nla_put_failure:
8339         nlmsg_trim(skb, nlh);
8340         return -1;
8341 }
8342
8343 struct nft_flowtable_filter {
8344         char            *table;
8345 };
8346
8347 static int nf_tables_dump_flowtable(struct sk_buff *skb,
8348                                     struct netlink_callback *cb)
8349 {
8350         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
8351         struct nft_flowtable_filter *filter = cb->data;
8352         unsigned int idx = 0, s_idx = cb->args[0];
8353         struct net *net = sock_net(skb->sk);
8354         int family = nfmsg->nfgen_family;
8355         struct nft_flowtable *flowtable;
8356         struct nftables_pernet *nft_net;
8357         const struct nft_table *table;
8358
8359         rcu_read_lock();
8360         nft_net = nft_pernet(net);
8361         cb->seq = READ_ONCE(nft_net->base_seq);
8362
8363         list_for_each_entry_rcu(table, &nft_net->tables, list) {
8364                 if (family != NFPROTO_UNSPEC && family != table->family)
8365                         continue;
8366
8367                 list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
8368                         if (!nft_is_active(net, flowtable))
8369                                 goto cont;
8370                         if (idx < s_idx)
8371                                 goto cont;
8372                         if (idx > s_idx)
8373                                 memset(&cb->args[1], 0,
8374                                        sizeof(cb->args) - sizeof(cb->args[0]));
8375                         if (filter && filter->table &&
8376                             strcmp(filter->table, table->name))
8377                                 goto cont;
8378
8379                         if (nf_tables_fill_flowtable_info(skb, net, NETLINK_CB(cb->skb).portid,
8380                                                           cb->nlh->nlmsg_seq,
8381                                                           NFT_MSG_NEWFLOWTABLE,
8382                                                           NLM_F_MULTI | NLM_F_APPEND,
8383                                                           table->family,
8384                                                           flowtable, NULL) < 0)
8385                                 goto done;
8386
8387                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
8388 cont:
8389                         idx++;
8390                 }
8391         }
8392 done:
8393         rcu_read_unlock();
8394
8395         cb->args[0] = idx;
8396         return skb->len;
8397 }
8398
8399 static int nf_tables_dump_flowtable_start(struct netlink_callback *cb)
8400 {
8401         const struct nlattr * const *nla = cb->data;
8402         struct nft_flowtable_filter *filter = NULL;
8403
8404         if (nla[NFTA_FLOWTABLE_TABLE]) {
8405                 filter = kzalloc(sizeof(*filter), GFP_ATOMIC);
8406                 if (!filter)
8407                         return -ENOMEM;
8408
8409                 filter->table = nla_strdup(nla[NFTA_FLOWTABLE_TABLE],
8410                                            GFP_ATOMIC);
8411                 if (!filter->table) {
8412                         kfree(filter);
8413                         return -ENOMEM;
8414                 }
8415         }
8416
8417         cb->data = filter;
8418         return 0;
8419 }
8420
8421 static int nf_tables_dump_flowtable_done(struct netlink_callback *cb)
8422 {
8423         struct nft_flowtable_filter *filter = cb->data;
8424
8425         if (!filter)
8426                 return 0;
8427
8428         kfree(filter->table);
8429         kfree(filter);
8430
8431         return 0;
8432 }
8433
8434 /* called with rcu_read_lock held */
8435 static int nf_tables_getflowtable(struct sk_buff *skb,
8436                                   const struct nfnl_info *info,
8437                                   const struct nlattr * const nla[])
8438 {
8439         u8 genmask = nft_genmask_cur(info->net);
8440         u8 family = info->nfmsg->nfgen_family;
8441         struct nft_flowtable *flowtable;
8442         const struct nft_table *table;
8443         struct net *net = info->net;
8444         struct sk_buff *skb2;
8445         int err;
8446
8447         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
8448                 struct netlink_dump_control c = {
8449                         .start = nf_tables_dump_flowtable_start,
8450                         .dump = nf_tables_dump_flowtable,
8451                         .done = nf_tables_dump_flowtable_done,
8452                         .module = THIS_MODULE,
8453                         .data = (void *)nla,
8454                 };
8455
8456                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
8457         }
8458
8459         if (!nla[NFTA_FLOWTABLE_NAME])
8460                 return -EINVAL;
8461
8462         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8463                                  genmask, 0);
8464         if (IS_ERR(table))
8465                 return PTR_ERR(table);
8466
8467         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
8468                                          genmask);
8469         if (IS_ERR(flowtable))
8470                 return PTR_ERR(flowtable);
8471
8472         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
8473         if (!skb2)
8474                 return -ENOMEM;
8475
8476         err = nf_tables_fill_flowtable_info(skb2, net, NETLINK_CB(skb).portid,
8477                                             info->nlh->nlmsg_seq,
8478                                             NFT_MSG_NEWFLOWTABLE, 0, family,
8479                                             flowtable, NULL);
8480         if (err < 0)
8481                 goto err_fill_flowtable_info;
8482
8483         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
8484
8485 err_fill_flowtable_info:
8486         kfree_skb(skb2);
8487         return err;
8488 }
8489
8490 static void nf_tables_flowtable_notify(struct nft_ctx *ctx,
8491                                        struct nft_flowtable *flowtable,
8492                                        struct list_head *hook_list, int event)
8493 {
8494         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
8495         struct sk_buff *skb;
8496         u16 flags = 0;
8497         int err;
8498
8499         if (!ctx->report &&
8500             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
8501                 return;
8502
8503         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
8504         if (skb == NULL)
8505                 goto err;
8506
8507         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
8508                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
8509
8510         err = nf_tables_fill_flowtable_info(skb, ctx->net, ctx->portid,
8511                                             ctx->seq, event, flags,
8512                                             ctx->family, flowtable, hook_list);
8513         if (err < 0) {
8514                 kfree_skb(skb);
8515                 goto err;
8516         }
8517
8518         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
8519         return;
8520 err:
8521         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
8522 }
8523
8524 static void nf_tables_flowtable_destroy(struct nft_flowtable *flowtable)
8525 {
8526         struct nft_hook *hook, *next;
8527
8528         flowtable->data.type->free(&flowtable->data);
8529         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
8530                 flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
8531                                             FLOW_BLOCK_UNBIND);
8532                 list_del_rcu(&hook->list);
8533                 kfree(hook);
8534         }
8535         kfree(flowtable->name);
8536         module_put(flowtable->data.type->owner);
8537         kfree(flowtable);
8538 }
8539
8540 static int nf_tables_fill_gen_info(struct sk_buff *skb, struct net *net,
8541                                    u32 portid, u32 seq)
8542 {
8543         struct nftables_pernet *nft_net = nft_pernet(net);
8544         struct nlmsghdr *nlh;
8545         char buf[TASK_COMM_LEN];
8546         int event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWGEN);
8547
8548         nlh = nfnl_msg_put(skb, portid, seq, event, 0, AF_UNSPEC,
8549                            NFNETLINK_V0, nft_base_seq(net));
8550         if (!nlh)
8551                 goto nla_put_failure;
8552
8553         if (nla_put_be32(skb, NFTA_GEN_ID, htonl(nft_net->base_seq)) ||
8554             nla_put_be32(skb, NFTA_GEN_PROC_PID, htonl(task_pid_nr(current))) ||
8555             nla_put_string(skb, NFTA_GEN_PROC_NAME, get_task_comm(buf, current)))
8556                 goto nla_put_failure;
8557
8558         nlmsg_end(skb, nlh);
8559         return 0;
8560
8561 nla_put_failure:
8562         nlmsg_trim(skb, nlh);
8563         return -EMSGSIZE;
8564 }
8565
8566 static void nft_flowtable_event(unsigned long event, struct net_device *dev,
8567                                 struct nft_flowtable *flowtable)
8568 {
8569         struct nft_hook *hook;
8570
8571         list_for_each_entry(hook, &flowtable->hook_list, list) {
8572                 if (hook->ops.dev != dev)
8573                         continue;
8574
8575                 /* flow_offload_netdev_event() cleans up entries for us. */
8576                 nft_unregister_flowtable_hook(dev_net(dev), flowtable, hook);
8577                 list_del_rcu(&hook->list);
8578                 kfree_rcu(hook, rcu);
8579                 break;
8580         }
8581 }
8582
8583 static int nf_tables_flowtable_event(struct notifier_block *this,
8584                                      unsigned long event, void *ptr)
8585 {
8586         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
8587         struct nft_flowtable *flowtable;
8588         struct nftables_pernet *nft_net;
8589         struct nft_table *table;
8590         struct net *net;
8591
8592         if (event != NETDEV_UNREGISTER)
8593                 return 0;
8594
8595         net = dev_net(dev);
8596         nft_net = nft_pernet(net);
8597         mutex_lock(&nft_net->commit_mutex);
8598         list_for_each_entry(table, &nft_net->tables, list) {
8599                 list_for_each_entry(flowtable, &table->flowtables, list) {
8600                         nft_flowtable_event(event, dev, flowtable);
8601                 }
8602         }
8603         mutex_unlock(&nft_net->commit_mutex);
8604
8605         return NOTIFY_DONE;
8606 }
8607
8608 static struct notifier_block nf_tables_flowtable_notifier = {
8609         .notifier_call  = nf_tables_flowtable_event,
8610 };
8611
8612 static void nf_tables_gen_notify(struct net *net, struct sk_buff *skb,
8613                                  int event)
8614 {
8615         struct nlmsghdr *nlh = nlmsg_hdr(skb);
8616         struct sk_buff *skb2;
8617         int err;
8618
8619         if (!nlmsg_report(nlh) &&
8620             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
8621                 return;
8622
8623         skb2 = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
8624         if (skb2 == NULL)
8625                 goto err;
8626
8627         err = nf_tables_fill_gen_info(skb2, net, NETLINK_CB(skb).portid,
8628                                       nlh->nlmsg_seq);
8629         if (err < 0) {
8630                 kfree_skb(skb2);
8631                 goto err;
8632         }
8633
8634         nfnetlink_send(skb2, net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
8635                        nlmsg_report(nlh), GFP_KERNEL);
8636         return;
8637 err:
8638         nfnetlink_set_err(net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
8639                           -ENOBUFS);
8640 }
8641
8642 static int nf_tables_getgen(struct sk_buff *skb, const struct nfnl_info *info,
8643                             const struct nlattr * const nla[])
8644 {
8645         struct sk_buff *skb2;
8646         int err;
8647
8648         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
8649         if (skb2 == NULL)
8650                 return -ENOMEM;
8651
8652         err = nf_tables_fill_gen_info(skb2, info->net, NETLINK_CB(skb).portid,
8653                                       info->nlh->nlmsg_seq);
8654         if (err < 0)
8655                 goto err_fill_gen_info;
8656
8657         return nfnetlink_unicast(skb2, info->net, NETLINK_CB(skb).portid);
8658
8659 err_fill_gen_info:
8660         kfree_skb(skb2);
8661         return err;
8662 }
8663
8664 static const struct nfnl_callback nf_tables_cb[NFT_MSG_MAX] = {
8665         [NFT_MSG_NEWTABLE] = {
8666                 .call           = nf_tables_newtable,
8667                 .type           = NFNL_CB_BATCH,
8668                 .attr_count     = NFTA_TABLE_MAX,
8669                 .policy         = nft_table_policy,
8670         },
8671         [NFT_MSG_GETTABLE] = {
8672                 .call           = nf_tables_gettable,
8673                 .type           = NFNL_CB_RCU,
8674                 .attr_count     = NFTA_TABLE_MAX,
8675                 .policy         = nft_table_policy,
8676         },
8677         [NFT_MSG_DELTABLE] = {
8678                 .call           = nf_tables_deltable,
8679                 .type           = NFNL_CB_BATCH,
8680                 .attr_count     = NFTA_TABLE_MAX,
8681                 .policy         = nft_table_policy,
8682         },
8683         [NFT_MSG_DESTROYTABLE] = {
8684                 .call           = nf_tables_deltable,
8685                 .type           = NFNL_CB_BATCH,
8686                 .attr_count     = NFTA_TABLE_MAX,
8687                 .policy         = nft_table_policy,
8688         },
8689         [NFT_MSG_NEWCHAIN] = {
8690                 .call           = nf_tables_newchain,
8691                 .type           = NFNL_CB_BATCH,
8692                 .attr_count     = NFTA_CHAIN_MAX,
8693                 .policy         = nft_chain_policy,
8694         },
8695         [NFT_MSG_GETCHAIN] = {
8696                 .call           = nf_tables_getchain,
8697                 .type           = NFNL_CB_RCU,
8698                 .attr_count     = NFTA_CHAIN_MAX,
8699                 .policy         = nft_chain_policy,
8700         },
8701         [NFT_MSG_DELCHAIN] = {
8702                 .call           = nf_tables_delchain,
8703                 .type           = NFNL_CB_BATCH,
8704                 .attr_count     = NFTA_CHAIN_MAX,
8705                 .policy         = nft_chain_policy,
8706         },
8707         [NFT_MSG_DESTROYCHAIN] = {
8708                 .call           = nf_tables_delchain,
8709                 .type           = NFNL_CB_BATCH,
8710                 .attr_count     = NFTA_CHAIN_MAX,
8711                 .policy         = nft_chain_policy,
8712         },
8713         [NFT_MSG_NEWRULE] = {
8714                 .call           = nf_tables_newrule,
8715                 .type           = NFNL_CB_BATCH,
8716                 .attr_count     = NFTA_RULE_MAX,
8717                 .policy         = nft_rule_policy,
8718         },
8719         [NFT_MSG_GETRULE] = {
8720                 .call           = nf_tables_getrule,
8721                 .type           = NFNL_CB_RCU,
8722                 .attr_count     = NFTA_RULE_MAX,
8723                 .policy         = nft_rule_policy,
8724         },
8725         [NFT_MSG_GETRULE_RESET] = {
8726                 .call           = nf_tables_getrule,
8727                 .type           = NFNL_CB_RCU,
8728                 .attr_count     = NFTA_RULE_MAX,
8729                 .policy         = nft_rule_policy,
8730         },
8731         [NFT_MSG_DELRULE] = {
8732                 .call           = nf_tables_delrule,
8733                 .type           = NFNL_CB_BATCH,
8734                 .attr_count     = NFTA_RULE_MAX,
8735                 .policy         = nft_rule_policy,
8736         },
8737         [NFT_MSG_DESTROYRULE] = {
8738                 .call           = nf_tables_delrule,
8739                 .type           = NFNL_CB_BATCH,
8740                 .attr_count     = NFTA_RULE_MAX,
8741                 .policy         = nft_rule_policy,
8742         },
8743         [NFT_MSG_NEWSET] = {
8744                 .call           = nf_tables_newset,
8745                 .type           = NFNL_CB_BATCH,
8746                 .attr_count     = NFTA_SET_MAX,
8747                 .policy         = nft_set_policy,
8748         },
8749         [NFT_MSG_GETSET] = {
8750                 .call           = nf_tables_getset,
8751                 .type           = NFNL_CB_RCU,
8752                 .attr_count     = NFTA_SET_MAX,
8753                 .policy         = nft_set_policy,
8754         },
8755         [NFT_MSG_DELSET] = {
8756                 .call           = nf_tables_delset,
8757                 .type           = NFNL_CB_BATCH,
8758                 .attr_count     = NFTA_SET_MAX,
8759                 .policy         = nft_set_policy,
8760         },
8761         [NFT_MSG_DESTROYSET] = {
8762                 .call           = nf_tables_delset,
8763                 .type           = NFNL_CB_BATCH,
8764                 .attr_count     = NFTA_SET_MAX,
8765                 .policy         = nft_set_policy,
8766         },
8767         [NFT_MSG_NEWSETELEM] = {
8768                 .call           = nf_tables_newsetelem,
8769                 .type           = NFNL_CB_BATCH,
8770                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
8771                 .policy         = nft_set_elem_list_policy,
8772         },
8773         [NFT_MSG_GETSETELEM] = {
8774                 .call           = nf_tables_getsetelem,
8775                 .type           = NFNL_CB_RCU,
8776                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
8777                 .policy         = nft_set_elem_list_policy,
8778         },
8779         [NFT_MSG_DELSETELEM] = {
8780                 .call           = nf_tables_delsetelem,
8781                 .type           = NFNL_CB_BATCH,
8782                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
8783                 .policy         = nft_set_elem_list_policy,
8784         },
8785         [NFT_MSG_DESTROYSETELEM] = {
8786                 .call           = nf_tables_delsetelem,
8787                 .type           = NFNL_CB_BATCH,
8788                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
8789                 .policy         = nft_set_elem_list_policy,
8790         },
8791         [NFT_MSG_GETGEN] = {
8792                 .call           = nf_tables_getgen,
8793                 .type           = NFNL_CB_RCU,
8794         },
8795         [NFT_MSG_NEWOBJ] = {
8796                 .call           = nf_tables_newobj,
8797                 .type           = NFNL_CB_BATCH,
8798                 .attr_count     = NFTA_OBJ_MAX,
8799                 .policy         = nft_obj_policy,
8800         },
8801         [NFT_MSG_GETOBJ] = {
8802                 .call           = nf_tables_getobj,
8803                 .type           = NFNL_CB_RCU,
8804                 .attr_count     = NFTA_OBJ_MAX,
8805                 .policy         = nft_obj_policy,
8806         },
8807         [NFT_MSG_DELOBJ] = {
8808                 .call           = nf_tables_delobj,
8809                 .type           = NFNL_CB_BATCH,
8810                 .attr_count     = NFTA_OBJ_MAX,
8811                 .policy         = nft_obj_policy,
8812         },
8813         [NFT_MSG_DESTROYOBJ] = {
8814                 .call           = nf_tables_delobj,
8815                 .type           = NFNL_CB_BATCH,
8816                 .attr_count     = NFTA_OBJ_MAX,
8817                 .policy         = nft_obj_policy,
8818         },
8819         [NFT_MSG_GETOBJ_RESET] = {
8820                 .call           = nf_tables_getobj,
8821                 .type           = NFNL_CB_RCU,
8822                 .attr_count     = NFTA_OBJ_MAX,
8823                 .policy         = nft_obj_policy,
8824         },
8825         [NFT_MSG_NEWFLOWTABLE] = {
8826                 .call           = nf_tables_newflowtable,
8827                 .type           = NFNL_CB_BATCH,
8828                 .attr_count     = NFTA_FLOWTABLE_MAX,
8829                 .policy         = nft_flowtable_policy,
8830         },
8831         [NFT_MSG_GETFLOWTABLE] = {
8832                 .call           = nf_tables_getflowtable,
8833                 .type           = NFNL_CB_RCU,
8834                 .attr_count     = NFTA_FLOWTABLE_MAX,
8835                 .policy         = nft_flowtable_policy,
8836         },
8837         [NFT_MSG_DELFLOWTABLE] = {
8838                 .call           = nf_tables_delflowtable,
8839                 .type           = NFNL_CB_BATCH,
8840                 .attr_count     = NFTA_FLOWTABLE_MAX,
8841                 .policy         = nft_flowtable_policy,
8842         },
8843         [NFT_MSG_DESTROYFLOWTABLE] = {
8844                 .call           = nf_tables_delflowtable,
8845                 .type           = NFNL_CB_BATCH,
8846                 .attr_count     = NFTA_FLOWTABLE_MAX,
8847                 .policy         = nft_flowtable_policy,
8848         },
8849 };
8850
8851 static int nf_tables_validate(struct net *net)
8852 {
8853         struct nftables_pernet *nft_net = nft_pernet(net);
8854         struct nft_table *table;
8855
8856         list_for_each_entry(table, &nft_net->tables, list) {
8857                 switch (table->validate_state) {
8858                 case NFT_VALIDATE_SKIP:
8859                         continue;
8860                 case NFT_VALIDATE_NEED:
8861                         nft_validate_state_update(table, NFT_VALIDATE_DO);
8862                         fallthrough;
8863                 case NFT_VALIDATE_DO:
8864                         if (nft_table_validate(net, table) < 0)
8865                                 return -EAGAIN;
8866
8867                         nft_validate_state_update(table, NFT_VALIDATE_SKIP);
8868                 }
8869
8870                 break;
8871         }
8872
8873         return 0;
8874 }
8875
8876 /* a drop policy has to be deferred until all rules have been activated,
8877  * otherwise a large ruleset that contains a drop-policy base chain will
8878  * cause all packets to get dropped until the full transaction has been
8879  * processed.
8880  *
8881  * We defer the drop policy until the transaction has been finalized.
8882  */
8883 static void nft_chain_commit_drop_policy(struct nft_trans *trans)
8884 {
8885         struct nft_base_chain *basechain;
8886
8887         if (nft_trans_chain_policy(trans) != NF_DROP)
8888                 return;
8889
8890         if (!nft_is_base_chain(trans->ctx.chain))
8891                 return;
8892
8893         basechain = nft_base_chain(trans->ctx.chain);
8894         basechain->policy = NF_DROP;
8895 }
8896
8897 static void nft_chain_commit_update(struct nft_trans *trans)
8898 {
8899         struct nft_base_chain *basechain;
8900
8901         if (nft_trans_chain_name(trans)) {
8902                 rhltable_remove(&trans->ctx.table->chains_ht,
8903                                 &trans->ctx.chain->rhlhead,
8904                                 nft_chain_ht_params);
8905                 swap(trans->ctx.chain->name, nft_trans_chain_name(trans));
8906                 rhltable_insert_key(&trans->ctx.table->chains_ht,
8907                                     trans->ctx.chain->name,
8908                                     &trans->ctx.chain->rhlhead,
8909                                     nft_chain_ht_params);
8910         }
8911
8912         if (!nft_is_base_chain(trans->ctx.chain))
8913                 return;
8914
8915         nft_chain_stats_replace(trans);
8916
8917         basechain = nft_base_chain(trans->ctx.chain);
8918
8919         switch (nft_trans_chain_policy(trans)) {
8920         case NF_DROP:
8921         case NF_ACCEPT:
8922                 basechain->policy = nft_trans_chain_policy(trans);
8923                 break;
8924         }
8925 }
8926
8927 static void nft_obj_commit_update(struct nft_trans *trans)
8928 {
8929         struct nft_object *newobj;
8930         struct nft_object *obj;
8931
8932         obj = nft_trans_obj(trans);
8933         newobj = nft_trans_obj_newobj(trans);
8934
8935         if (obj->ops->update)
8936                 obj->ops->update(obj, newobj);
8937
8938         nft_obj_destroy(&trans->ctx, newobj);
8939 }
8940
8941 static void nft_commit_release(struct nft_trans *trans)
8942 {
8943         switch (trans->msg_type) {
8944         case NFT_MSG_DELTABLE:
8945         case NFT_MSG_DESTROYTABLE:
8946                 nf_tables_table_destroy(&trans->ctx);
8947                 break;
8948         case NFT_MSG_NEWCHAIN:
8949                 free_percpu(nft_trans_chain_stats(trans));
8950                 kfree(nft_trans_chain_name(trans));
8951                 break;
8952         case NFT_MSG_DELCHAIN:
8953         case NFT_MSG_DESTROYCHAIN:
8954                 if (nft_trans_chain_update(trans))
8955                         nft_hooks_destroy(&nft_trans_chain_hooks(trans));
8956                 else
8957                         nf_tables_chain_destroy(&trans->ctx);
8958                 break;
8959         case NFT_MSG_DELRULE:
8960         case NFT_MSG_DESTROYRULE:
8961                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
8962                 break;
8963         case NFT_MSG_DELSET:
8964         case NFT_MSG_DESTROYSET:
8965                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
8966                 break;
8967         case NFT_MSG_DELSETELEM:
8968         case NFT_MSG_DESTROYSETELEM:
8969                 nf_tables_set_elem_destroy(&trans->ctx,
8970                                            nft_trans_elem_set(trans),
8971                                            nft_trans_elem(trans).priv);
8972                 break;
8973         case NFT_MSG_DELOBJ:
8974         case NFT_MSG_DESTROYOBJ:
8975                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
8976                 break;
8977         case NFT_MSG_DELFLOWTABLE:
8978         case NFT_MSG_DESTROYFLOWTABLE:
8979                 if (nft_trans_flowtable_update(trans))
8980                         nft_hooks_destroy(&nft_trans_flowtable_hooks(trans));
8981                 else
8982                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
8983                 break;
8984         }
8985
8986         if (trans->put_net)
8987                 put_net(trans->ctx.net);
8988
8989         kfree(trans);
8990 }
8991
8992 static void nf_tables_trans_destroy_work(struct work_struct *w)
8993 {
8994         struct nft_trans *trans, *next;
8995         LIST_HEAD(head);
8996
8997         spin_lock(&nf_tables_destroy_list_lock);
8998         list_splice_init(&nf_tables_destroy_list, &head);
8999         spin_unlock(&nf_tables_destroy_list_lock);
9000
9001         if (list_empty(&head))
9002                 return;
9003
9004         synchronize_rcu();
9005
9006         list_for_each_entry_safe(trans, next, &head, list) {
9007                 list_del(&trans->list);
9008                 nft_commit_release(trans);
9009         }
9010 }
9011
9012 void nf_tables_trans_destroy_flush_work(void)
9013 {
9014         flush_work(&trans_destroy_work);
9015 }
9016 EXPORT_SYMBOL_GPL(nf_tables_trans_destroy_flush_work);
9017
9018 static bool nft_expr_reduce(struct nft_regs_track *track,
9019                             const struct nft_expr *expr)
9020 {
9021         return false;
9022 }
9023
9024 static int nf_tables_commit_chain_prepare(struct net *net, struct nft_chain *chain)
9025 {
9026         const struct nft_expr *expr, *last;
9027         struct nft_regs_track track = {};
9028         unsigned int size, data_size;
9029         void *data, *data_boundary;
9030         struct nft_rule_dp *prule;
9031         struct nft_rule *rule;
9032
9033         /* already handled or inactive chain? */
9034         if (chain->blob_next || !nft_is_active_next(net, chain))
9035                 return 0;
9036
9037         data_size = 0;
9038         list_for_each_entry(rule, &chain->rules, list) {
9039                 if (nft_is_active_next(net, rule)) {
9040                         data_size += sizeof(*prule) + rule->dlen;
9041                         if (data_size > INT_MAX)
9042                                 return -ENOMEM;
9043                 }
9044         }
9045
9046         chain->blob_next = nf_tables_chain_alloc_rules(chain, data_size);
9047         if (!chain->blob_next)
9048                 return -ENOMEM;
9049
9050         data = (void *)chain->blob_next->data;
9051         data_boundary = data + data_size;
9052         size = 0;
9053
9054         list_for_each_entry(rule, &chain->rules, list) {
9055                 if (!nft_is_active_next(net, rule))
9056                         continue;
9057
9058                 prule = (struct nft_rule_dp *)data;
9059                 data += offsetof(struct nft_rule_dp, data);
9060                 if (WARN_ON_ONCE(data > data_boundary))
9061                         return -ENOMEM;
9062
9063                 size = 0;
9064                 track.last = nft_expr_last(rule);
9065                 nft_rule_for_each_expr(expr, last, rule) {
9066                         track.cur = expr;
9067
9068                         if (nft_expr_reduce(&track, expr)) {
9069                                 expr = track.cur;
9070                                 continue;
9071                         }
9072
9073                         if (WARN_ON_ONCE(data + size + expr->ops->size > data_boundary))
9074                                 return -ENOMEM;
9075
9076                         memcpy(data + size, expr, expr->ops->size);
9077                         size += expr->ops->size;
9078                 }
9079                 if (WARN_ON_ONCE(size >= 1 << 12))
9080                         return -ENOMEM;
9081
9082                 prule->handle = rule->handle;
9083                 prule->dlen = size;
9084                 prule->is_last = 0;
9085
9086                 data += size;
9087                 size = 0;
9088                 chain->blob_next->size += (unsigned long)(data - (void *)prule);
9089         }
9090
9091         if (WARN_ON_ONCE(data > data_boundary))
9092                 return -ENOMEM;
9093
9094         prule = (struct nft_rule_dp *)data;
9095         nft_last_rule(chain, prule);
9096
9097         return 0;
9098 }
9099
9100 static void nf_tables_commit_chain_prepare_cancel(struct net *net)
9101 {
9102         struct nftables_pernet *nft_net = nft_pernet(net);
9103         struct nft_trans *trans, *next;
9104
9105         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
9106                 struct nft_chain *chain = trans->ctx.chain;
9107
9108                 if (trans->msg_type == NFT_MSG_NEWRULE ||
9109                     trans->msg_type == NFT_MSG_DELRULE) {
9110                         kvfree(chain->blob_next);
9111                         chain->blob_next = NULL;
9112                 }
9113         }
9114 }
9115
9116 static void __nf_tables_commit_chain_free_rules(struct rcu_head *h)
9117 {
9118         struct nft_rule_dp_last *l = container_of(h, struct nft_rule_dp_last, h);
9119
9120         kvfree(l->blob);
9121 }
9122
9123 static void nf_tables_commit_chain_free_rules_old(struct nft_rule_blob *blob)
9124 {
9125         struct nft_rule_dp_last *last;
9126
9127         /* last rule trailer is after end marker */
9128         last = (void *)blob + sizeof(*blob) + blob->size;
9129         last->blob = blob;
9130
9131         call_rcu(&last->h, __nf_tables_commit_chain_free_rules);
9132 }
9133
9134 static void nf_tables_commit_chain(struct net *net, struct nft_chain *chain)
9135 {
9136         struct nft_rule_blob *g0, *g1;
9137         bool next_genbit;
9138
9139         next_genbit = nft_gencursor_next(net);
9140
9141         g0 = rcu_dereference_protected(chain->blob_gen_0,
9142                                        lockdep_commit_lock_is_held(net));
9143         g1 = rcu_dereference_protected(chain->blob_gen_1,
9144                                        lockdep_commit_lock_is_held(net));
9145
9146         /* No changes to this chain? */
9147         if (chain->blob_next == NULL) {
9148                 /* chain had no change in last or next generation */
9149                 if (g0 == g1)
9150                         return;
9151                 /*
9152                  * chain had no change in this generation; make sure next
9153                  * one uses same rules as current generation.
9154                  */
9155                 if (next_genbit) {
9156                         rcu_assign_pointer(chain->blob_gen_1, g0);
9157                         nf_tables_commit_chain_free_rules_old(g1);
9158                 } else {
9159                         rcu_assign_pointer(chain->blob_gen_0, g1);
9160                         nf_tables_commit_chain_free_rules_old(g0);
9161                 }
9162
9163                 return;
9164         }
9165
9166         if (next_genbit)
9167                 rcu_assign_pointer(chain->blob_gen_1, chain->blob_next);
9168         else
9169                 rcu_assign_pointer(chain->blob_gen_0, chain->blob_next);
9170
9171         chain->blob_next = NULL;
9172
9173         if (g0 == g1)
9174                 return;
9175
9176         if (next_genbit)
9177                 nf_tables_commit_chain_free_rules_old(g1);
9178         else
9179                 nf_tables_commit_chain_free_rules_old(g0);
9180 }
9181
9182 static void nft_obj_del(struct nft_object *obj)
9183 {
9184         rhltable_remove(&nft_objname_ht, &obj->rhlhead, nft_objname_ht_params);
9185         list_del_rcu(&obj->list);
9186 }
9187
9188 void nft_chain_del(struct nft_chain *chain)
9189 {
9190         struct nft_table *table = chain->table;
9191
9192         WARN_ON_ONCE(rhltable_remove(&table->chains_ht, &chain->rhlhead,
9193                                      nft_chain_ht_params));
9194         list_del_rcu(&chain->list);
9195 }
9196
9197 static void nf_tables_module_autoload_cleanup(struct net *net)
9198 {
9199         struct nftables_pernet *nft_net = nft_pernet(net);
9200         struct nft_module_request *req, *next;
9201
9202         WARN_ON_ONCE(!list_empty(&nft_net->commit_list));
9203         list_for_each_entry_safe(req, next, &nft_net->module_list, list) {
9204                 WARN_ON_ONCE(!req->done);
9205                 list_del(&req->list);
9206                 kfree(req);
9207         }
9208 }
9209
9210 static void nf_tables_commit_release(struct net *net)
9211 {
9212         struct nftables_pernet *nft_net = nft_pernet(net);
9213         struct nft_trans *trans;
9214
9215         /* all side effects have to be made visible.
9216          * For example, if a chain named 'foo' has been deleted, a
9217          * new transaction must not find it anymore.
9218          *
9219          * Memory reclaim happens asynchronously from work queue
9220          * to prevent expensive synchronize_rcu() in commit phase.
9221          */
9222         if (list_empty(&nft_net->commit_list)) {
9223                 nf_tables_module_autoload_cleanup(net);
9224                 mutex_unlock(&nft_net->commit_mutex);
9225                 return;
9226         }
9227
9228         trans = list_last_entry(&nft_net->commit_list,
9229                                 struct nft_trans, list);
9230         get_net(trans->ctx.net);
9231         WARN_ON_ONCE(trans->put_net);
9232
9233         trans->put_net = true;
9234         spin_lock(&nf_tables_destroy_list_lock);
9235         list_splice_tail_init(&nft_net->commit_list, &nf_tables_destroy_list);
9236         spin_unlock(&nf_tables_destroy_list_lock);
9237
9238         nf_tables_module_autoload_cleanup(net);
9239         schedule_work(&trans_destroy_work);
9240
9241         mutex_unlock(&nft_net->commit_mutex);
9242 }
9243
9244 static void nft_commit_notify(struct net *net, u32 portid)
9245 {
9246         struct nftables_pernet *nft_net = nft_pernet(net);
9247         struct sk_buff *batch_skb = NULL, *nskb, *skb;
9248         unsigned char *data;
9249         int len;
9250
9251         list_for_each_entry_safe(skb, nskb, &nft_net->notify_list, list) {
9252                 if (!batch_skb) {
9253 new_batch:
9254                         batch_skb = skb;
9255                         len = NLMSG_GOODSIZE - skb->len;
9256                         list_del(&skb->list);
9257                         continue;
9258                 }
9259                 len -= skb->len;
9260                 if (len > 0 && NFT_CB(skb).report == NFT_CB(batch_skb).report) {
9261                         data = skb_put(batch_skb, skb->len);
9262                         memcpy(data, skb->data, skb->len);
9263                         list_del(&skb->list);
9264                         kfree_skb(skb);
9265                         continue;
9266                 }
9267                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
9268                                NFT_CB(batch_skb).report, GFP_KERNEL);
9269                 goto new_batch;
9270         }
9271
9272         if (batch_skb) {
9273                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
9274                                NFT_CB(batch_skb).report, GFP_KERNEL);
9275         }
9276
9277         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
9278 }
9279
9280 static int nf_tables_commit_audit_alloc(struct list_head *adl,
9281                                         struct nft_table *table)
9282 {
9283         struct nft_audit_data *adp;
9284
9285         list_for_each_entry(adp, adl, list) {
9286                 if (adp->table == table)
9287                         return 0;
9288         }
9289         adp = kzalloc(sizeof(*adp), GFP_KERNEL);
9290         if (!adp)
9291                 return -ENOMEM;
9292         adp->table = table;
9293         list_add(&adp->list, adl);
9294         return 0;
9295 }
9296
9297 static void nf_tables_commit_audit_free(struct list_head *adl)
9298 {
9299         struct nft_audit_data *adp, *adn;
9300
9301         list_for_each_entry_safe(adp, adn, adl, list) {
9302                 list_del(&adp->list);
9303                 kfree(adp);
9304         }
9305 }
9306
9307 static void nf_tables_commit_audit_collect(struct list_head *adl,
9308                                            struct nft_table *table, u32 op)
9309 {
9310         struct nft_audit_data *adp;
9311
9312         list_for_each_entry(adp, adl, list) {
9313                 if (adp->table == table)
9314                         goto found;
9315         }
9316         WARN_ONCE(1, "table=%s not expected in commit list", table->name);
9317         return;
9318 found:
9319         adp->entries++;
9320         if (!adp->op || adp->op > op)
9321                 adp->op = op;
9322 }
9323
9324 #define AUNFTABLENAMELEN (NFT_TABLE_MAXNAMELEN + 22)
9325
9326 static void nf_tables_commit_audit_log(struct list_head *adl, u32 generation)
9327 {
9328         struct nft_audit_data *adp, *adn;
9329         char aubuf[AUNFTABLENAMELEN];
9330
9331         list_for_each_entry_safe(adp, adn, adl, list) {
9332                 snprintf(aubuf, AUNFTABLENAMELEN, "%s:%u", adp->table->name,
9333                          generation);
9334                 audit_log_nfcfg(aubuf, adp->table->family, adp->entries,
9335                                 nft2audit_op[adp->op], GFP_KERNEL);
9336                 list_del(&adp->list);
9337                 kfree(adp);
9338         }
9339 }
9340
9341 static void nft_set_commit_update(struct list_head *set_update_list)
9342 {
9343         struct nft_set *set, *next;
9344
9345         list_for_each_entry_safe(set, next, set_update_list, pending_update) {
9346                 list_del_init(&set->pending_update);
9347
9348                 if (!set->ops->commit)
9349                         continue;
9350
9351                 set->ops->commit(set);
9352         }
9353 }
9354
9355 static int nf_tables_commit(struct net *net, struct sk_buff *skb)
9356 {
9357         struct nftables_pernet *nft_net = nft_pernet(net);
9358         struct nft_trans *trans, *next;
9359         LIST_HEAD(set_update_list);
9360         struct nft_trans_elem *te;
9361         struct nft_chain *chain;
9362         struct nft_table *table;
9363         unsigned int base_seq;
9364         LIST_HEAD(adl);
9365         int err;
9366
9367         if (list_empty(&nft_net->commit_list)) {
9368                 mutex_unlock(&nft_net->commit_mutex);
9369                 return 0;
9370         }
9371
9372         /* 0. Validate ruleset, otherwise roll back for error reporting. */
9373         if (nf_tables_validate(net) < 0)
9374                 return -EAGAIN;
9375
9376         err = nft_flow_rule_offload_commit(net);
9377         if (err < 0)
9378                 return err;
9379
9380         /* 1.  Allocate space for next generation rules_gen_X[] */
9381         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
9382                 int ret;
9383
9384                 ret = nf_tables_commit_audit_alloc(&adl, trans->ctx.table);
9385                 if (ret) {
9386                         nf_tables_commit_chain_prepare_cancel(net);
9387                         nf_tables_commit_audit_free(&adl);
9388                         return ret;
9389                 }
9390                 if (trans->msg_type == NFT_MSG_NEWRULE ||
9391                     trans->msg_type == NFT_MSG_DELRULE) {
9392                         chain = trans->ctx.chain;
9393
9394                         ret = nf_tables_commit_chain_prepare(net, chain);
9395                         if (ret < 0) {
9396                                 nf_tables_commit_chain_prepare_cancel(net);
9397                                 nf_tables_commit_audit_free(&adl);
9398                                 return ret;
9399                         }
9400                 }
9401         }
9402
9403         /* step 2.  Make rules_gen_X visible to packet path */
9404         list_for_each_entry(table, &nft_net->tables, list) {
9405                 list_for_each_entry(chain, &table->chains, list)
9406                         nf_tables_commit_chain(net, chain);
9407         }
9408
9409         /*
9410          * Bump generation counter, invalidate any dump in progress.
9411          * Cannot fail after this point.
9412          */
9413         base_seq = READ_ONCE(nft_net->base_seq);
9414         while (++base_seq == 0)
9415                 ;
9416
9417         WRITE_ONCE(nft_net->base_seq, base_seq);
9418
9419         /* step 3. Start new generation, rules_gen_X now in use. */
9420         net->nft.gencursor = nft_gencursor_next(net);
9421
9422         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
9423                 nf_tables_commit_audit_collect(&adl, trans->ctx.table,
9424                                                trans->msg_type);
9425                 switch (trans->msg_type) {
9426                 case NFT_MSG_NEWTABLE:
9427                         if (nft_trans_table_update(trans)) {
9428                                 if (!(trans->ctx.table->flags & __NFT_TABLE_F_UPDATE)) {
9429                                         nft_trans_destroy(trans);
9430                                         break;
9431                                 }
9432                                 if (trans->ctx.table->flags & NFT_TABLE_F_DORMANT)
9433                                         nf_tables_table_disable(net, trans->ctx.table);
9434
9435                                 trans->ctx.table->flags &= ~__NFT_TABLE_F_UPDATE;
9436                         } else {
9437                                 nft_clear(net, trans->ctx.table);
9438                         }
9439                         nf_tables_table_notify(&trans->ctx, NFT_MSG_NEWTABLE);
9440                         nft_trans_destroy(trans);
9441                         break;
9442                 case NFT_MSG_DELTABLE:
9443                 case NFT_MSG_DESTROYTABLE:
9444                         list_del_rcu(&trans->ctx.table->list);
9445                         nf_tables_table_notify(&trans->ctx, trans->msg_type);
9446                         break;
9447                 case NFT_MSG_NEWCHAIN:
9448                         if (nft_trans_chain_update(trans)) {
9449                                 nft_chain_commit_update(trans);
9450                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN,
9451                                                        &nft_trans_chain_hooks(trans));
9452                                 list_splice(&nft_trans_chain_hooks(trans),
9453                                             &nft_trans_basechain(trans)->hook_list);
9454                                 /* trans destroyed after rcu grace period */
9455                         } else {
9456                                 nft_chain_commit_drop_policy(trans);
9457                                 nft_clear(net, trans->ctx.chain);
9458                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN, NULL);
9459                                 nft_trans_destroy(trans);
9460                         }
9461                         break;
9462                 case NFT_MSG_DELCHAIN:
9463                 case NFT_MSG_DESTROYCHAIN:
9464                         if (nft_trans_chain_update(trans)) {
9465                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_DELCHAIN,
9466                                                        &nft_trans_chain_hooks(trans));
9467                                 nft_netdev_unregister_hooks(net,
9468                                                             &nft_trans_chain_hooks(trans),
9469                                                             true);
9470                         } else {
9471                                 nft_chain_del(trans->ctx.chain);
9472                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_DELCHAIN,
9473                                                        NULL);
9474                                 nf_tables_unregister_hook(trans->ctx.net,
9475                                                           trans->ctx.table,
9476                                                           trans->ctx.chain);
9477                         }
9478                         break;
9479                 case NFT_MSG_NEWRULE:
9480                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
9481                         nf_tables_rule_notify(&trans->ctx,
9482                                               nft_trans_rule(trans),
9483                                               NFT_MSG_NEWRULE);
9484                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
9485                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
9486
9487                         nft_trans_destroy(trans);
9488                         break;
9489                 case NFT_MSG_DELRULE:
9490                 case NFT_MSG_DESTROYRULE:
9491                         list_del_rcu(&nft_trans_rule(trans)->list);
9492                         nf_tables_rule_notify(&trans->ctx,
9493                                               nft_trans_rule(trans),
9494                                               trans->msg_type);
9495                         nft_rule_expr_deactivate(&trans->ctx,
9496                                                  nft_trans_rule(trans),
9497                                                  NFT_TRANS_COMMIT);
9498
9499                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
9500                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
9501                         break;
9502                 case NFT_MSG_NEWSET:
9503                         if (nft_trans_set_update(trans)) {
9504                                 struct nft_set *set = nft_trans_set(trans);
9505
9506                                 WRITE_ONCE(set->timeout, nft_trans_set_timeout(trans));
9507                                 WRITE_ONCE(set->gc_int, nft_trans_set_gc_int(trans));
9508                         } else {
9509                                 nft_clear(net, nft_trans_set(trans));
9510                                 /* This avoids hitting -EBUSY when deleting the table
9511                                  * from the transaction.
9512                                  */
9513                                 if (nft_set_is_anonymous(nft_trans_set(trans)) &&
9514                                     !list_empty(&nft_trans_set(trans)->bindings))
9515                                         trans->ctx.table->use--;
9516                         }
9517                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
9518                                              NFT_MSG_NEWSET, GFP_KERNEL);
9519                         nft_trans_destroy(trans);
9520                         break;
9521                 case NFT_MSG_DELSET:
9522                 case NFT_MSG_DESTROYSET:
9523                         list_del_rcu(&nft_trans_set(trans)->list);
9524                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
9525                                              trans->msg_type, GFP_KERNEL);
9526                         break;
9527                 case NFT_MSG_NEWSETELEM:
9528                         te = (struct nft_trans_elem *)trans->data;
9529
9530                         nft_setelem_activate(net, te->set, &te->elem);
9531                         nf_tables_setelem_notify(&trans->ctx, te->set,
9532                                                  &te->elem,
9533                                                  NFT_MSG_NEWSETELEM);
9534                         if (te->set->ops->commit &&
9535                             list_empty(&te->set->pending_update)) {
9536                                 list_add_tail(&te->set->pending_update,
9537                                               &set_update_list);
9538                         }
9539                         nft_trans_destroy(trans);
9540                         break;
9541                 case NFT_MSG_DELSETELEM:
9542                 case NFT_MSG_DESTROYSETELEM:
9543                         te = (struct nft_trans_elem *)trans->data;
9544
9545                         nf_tables_setelem_notify(&trans->ctx, te->set,
9546                                                  &te->elem,
9547                                                  trans->msg_type);
9548                         nft_setelem_remove(net, te->set, &te->elem);
9549                         if (!nft_setelem_is_catchall(te->set, &te->elem)) {
9550                                 atomic_dec(&te->set->nelems);
9551                                 te->set->ndeact--;
9552                         }
9553                         if (te->set->ops->commit &&
9554                             list_empty(&te->set->pending_update)) {
9555                                 list_add_tail(&te->set->pending_update,
9556                                               &set_update_list);
9557                         }
9558                         break;
9559                 case NFT_MSG_NEWOBJ:
9560                         if (nft_trans_obj_update(trans)) {
9561                                 nft_obj_commit_update(trans);
9562                                 nf_tables_obj_notify(&trans->ctx,
9563                                                      nft_trans_obj(trans),
9564                                                      NFT_MSG_NEWOBJ);
9565                         } else {
9566                                 nft_clear(net, nft_trans_obj(trans));
9567                                 nf_tables_obj_notify(&trans->ctx,
9568                                                      nft_trans_obj(trans),
9569                                                      NFT_MSG_NEWOBJ);
9570                                 nft_trans_destroy(trans);
9571                         }
9572                         break;
9573                 case NFT_MSG_DELOBJ:
9574                 case NFT_MSG_DESTROYOBJ:
9575                         nft_obj_del(nft_trans_obj(trans));
9576                         nf_tables_obj_notify(&trans->ctx, nft_trans_obj(trans),
9577                                              trans->msg_type);
9578                         break;
9579                 case NFT_MSG_NEWFLOWTABLE:
9580                         if (nft_trans_flowtable_update(trans)) {
9581                                 nft_trans_flowtable(trans)->data.flags =
9582                                         nft_trans_flowtable_flags(trans);
9583                                 nf_tables_flowtable_notify(&trans->ctx,
9584                                                            nft_trans_flowtable(trans),
9585                                                            &nft_trans_flowtable_hooks(trans),
9586                                                            NFT_MSG_NEWFLOWTABLE);
9587                                 list_splice(&nft_trans_flowtable_hooks(trans),
9588                                             &nft_trans_flowtable(trans)->hook_list);
9589                         } else {
9590                                 nft_clear(net, nft_trans_flowtable(trans));
9591                                 nf_tables_flowtable_notify(&trans->ctx,
9592                                                            nft_trans_flowtable(trans),
9593                                                            NULL,
9594                                                            NFT_MSG_NEWFLOWTABLE);
9595                         }
9596                         nft_trans_destroy(trans);
9597                         break;
9598                 case NFT_MSG_DELFLOWTABLE:
9599                 case NFT_MSG_DESTROYFLOWTABLE:
9600                         if (nft_trans_flowtable_update(trans)) {
9601                                 nf_tables_flowtable_notify(&trans->ctx,
9602                                                            nft_trans_flowtable(trans),
9603                                                            &nft_trans_flowtable_hooks(trans),
9604                                                            trans->msg_type);
9605                                 nft_unregister_flowtable_net_hooks(net,
9606                                                                    &nft_trans_flowtable_hooks(trans));
9607                         } else {
9608                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
9609                                 nf_tables_flowtable_notify(&trans->ctx,
9610                                                            nft_trans_flowtable(trans),
9611                                                            NULL,
9612                                                            trans->msg_type);
9613                                 nft_unregister_flowtable_net_hooks(net,
9614                                                 &nft_trans_flowtable(trans)->hook_list);
9615                         }
9616                         break;
9617                 }
9618         }
9619
9620         nft_set_commit_update(&set_update_list);
9621
9622         nft_commit_notify(net, NETLINK_CB(skb).portid);
9623         nf_tables_gen_notify(net, skb, NFT_MSG_NEWGEN);
9624         nf_tables_commit_audit_log(&adl, nft_net->base_seq);
9625         nf_tables_commit_release(net);
9626
9627         return 0;
9628 }
9629
9630 static void nf_tables_module_autoload(struct net *net)
9631 {
9632         struct nftables_pernet *nft_net = nft_pernet(net);
9633         struct nft_module_request *req, *next;
9634         LIST_HEAD(module_list);
9635
9636         list_splice_init(&nft_net->module_list, &module_list);
9637         mutex_unlock(&nft_net->commit_mutex);
9638         list_for_each_entry_safe(req, next, &module_list, list) {
9639                 request_module("%s", req->module);
9640                 req->done = true;
9641         }
9642         mutex_lock(&nft_net->commit_mutex);
9643         list_splice(&module_list, &nft_net->module_list);
9644 }
9645
9646 static void nf_tables_abort_release(struct nft_trans *trans)
9647 {
9648         switch (trans->msg_type) {
9649         case NFT_MSG_NEWTABLE:
9650                 nf_tables_table_destroy(&trans->ctx);
9651                 break;
9652         case NFT_MSG_NEWCHAIN:
9653                 if (nft_trans_chain_update(trans))
9654                         nft_hooks_destroy(&nft_trans_chain_hooks(trans));
9655                 else
9656                         nf_tables_chain_destroy(&trans->ctx);
9657                 break;
9658         case NFT_MSG_NEWRULE:
9659                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
9660                 break;
9661         case NFT_MSG_NEWSET:
9662                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
9663                 break;
9664         case NFT_MSG_NEWSETELEM:
9665                 nft_set_elem_destroy(nft_trans_elem_set(trans),
9666                                      nft_trans_elem(trans).priv, true);
9667                 break;
9668         case NFT_MSG_NEWOBJ:
9669                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
9670                 break;
9671         case NFT_MSG_NEWFLOWTABLE:
9672                 if (nft_trans_flowtable_update(trans))
9673                         nft_hooks_destroy(&nft_trans_flowtable_hooks(trans));
9674                 else
9675                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
9676                 break;
9677         }
9678         kfree(trans);
9679 }
9680
9681 static void nft_set_abort_update(struct list_head *set_update_list)
9682 {
9683         struct nft_set *set, *next;
9684
9685         list_for_each_entry_safe(set, next, set_update_list, pending_update) {
9686                 list_del_init(&set->pending_update);
9687
9688                 if (!set->ops->abort)
9689                         continue;
9690
9691                 set->ops->abort(set);
9692         }
9693 }
9694
9695 static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action)
9696 {
9697         struct nftables_pernet *nft_net = nft_pernet(net);
9698         struct nft_trans *trans, *next;
9699         LIST_HEAD(set_update_list);
9700         struct nft_trans_elem *te;
9701
9702         if (action == NFNL_ABORT_VALIDATE &&
9703             nf_tables_validate(net) < 0)
9704                 return -EAGAIN;
9705
9706         list_for_each_entry_safe_reverse(trans, next, &nft_net->commit_list,
9707                                          list) {
9708                 switch (trans->msg_type) {
9709                 case NFT_MSG_NEWTABLE:
9710                         if (nft_trans_table_update(trans)) {
9711                                 if (!(trans->ctx.table->flags & __NFT_TABLE_F_UPDATE)) {
9712                                         nft_trans_destroy(trans);
9713                                         break;
9714                                 }
9715                                 if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_DORMANT) {
9716                                         nf_tables_table_disable(net, trans->ctx.table);
9717                                         trans->ctx.table->flags |= NFT_TABLE_F_DORMANT;
9718                                 } else if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_AWAKEN) {
9719                                         trans->ctx.table->flags &= ~NFT_TABLE_F_DORMANT;
9720                                 }
9721                                 trans->ctx.table->flags &= ~__NFT_TABLE_F_UPDATE;
9722                                 nft_trans_destroy(trans);
9723                         } else {
9724                                 list_del_rcu(&trans->ctx.table->list);
9725                         }
9726                         break;
9727                 case NFT_MSG_DELTABLE:
9728                 case NFT_MSG_DESTROYTABLE:
9729                         nft_clear(trans->ctx.net, trans->ctx.table);
9730                         nft_trans_destroy(trans);
9731                         break;
9732                 case NFT_MSG_NEWCHAIN:
9733                         if (nft_trans_chain_update(trans)) {
9734                                 nft_netdev_unregister_hooks(net,
9735                                                             &nft_trans_chain_hooks(trans),
9736                                                             true);
9737                                 free_percpu(nft_trans_chain_stats(trans));
9738                                 kfree(nft_trans_chain_name(trans));
9739                                 nft_trans_destroy(trans);
9740                         } else {
9741                                 if (nft_trans_chain_bound(trans)) {
9742                                         nft_trans_destroy(trans);
9743                                         break;
9744                                 }
9745                                 trans->ctx.table->use--;
9746                                 nft_chain_del(trans->ctx.chain);
9747                                 nf_tables_unregister_hook(trans->ctx.net,
9748                                                           trans->ctx.table,
9749                                                           trans->ctx.chain);
9750                         }
9751                         break;
9752                 case NFT_MSG_DELCHAIN:
9753                 case NFT_MSG_DESTROYCHAIN:
9754                         if (nft_trans_chain_update(trans)) {
9755                                 list_splice(&nft_trans_chain_hooks(trans),
9756                                             &nft_trans_basechain(trans)->hook_list);
9757                         } else {
9758                                 trans->ctx.table->use++;
9759                                 nft_clear(trans->ctx.net, trans->ctx.chain);
9760                         }
9761                         nft_trans_destroy(trans);
9762                         break;
9763                 case NFT_MSG_NEWRULE:
9764                         if (nft_trans_rule_bound(trans)) {
9765                                 nft_trans_destroy(trans);
9766                                 break;
9767                         }
9768                         trans->ctx.chain->use--;
9769                         list_del_rcu(&nft_trans_rule(trans)->list);
9770                         nft_rule_expr_deactivate(&trans->ctx,
9771                                                  nft_trans_rule(trans),
9772                                                  NFT_TRANS_ABORT);
9773                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
9774                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
9775                         break;
9776                 case NFT_MSG_DELRULE:
9777                 case NFT_MSG_DESTROYRULE:
9778                         trans->ctx.chain->use++;
9779                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
9780                         nft_rule_expr_activate(&trans->ctx, nft_trans_rule(trans));
9781                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
9782                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
9783
9784                         nft_trans_destroy(trans);
9785                         break;
9786                 case NFT_MSG_NEWSET:
9787                         if (nft_trans_set_update(trans)) {
9788                                 nft_trans_destroy(trans);
9789                                 break;
9790                         }
9791                         trans->ctx.table->use--;
9792                         if (nft_trans_set_bound(trans)) {
9793                                 nft_trans_destroy(trans);
9794                                 break;
9795                         }
9796                         list_del_rcu(&nft_trans_set(trans)->list);
9797                         break;
9798                 case NFT_MSG_DELSET:
9799                 case NFT_MSG_DESTROYSET:
9800                         trans->ctx.table->use++;
9801                         nft_clear(trans->ctx.net, nft_trans_set(trans));
9802                         nft_trans_destroy(trans);
9803                         break;
9804                 case NFT_MSG_NEWSETELEM:
9805                         if (nft_trans_elem_set_bound(trans)) {
9806                                 nft_trans_destroy(trans);
9807                                 break;
9808                         }
9809                         te = (struct nft_trans_elem *)trans->data;
9810                         nft_setelem_remove(net, te->set, &te->elem);
9811                         if (!nft_setelem_is_catchall(te->set, &te->elem))
9812                                 atomic_dec(&te->set->nelems);
9813
9814                         if (te->set->ops->abort &&
9815                             list_empty(&te->set->pending_update)) {
9816                                 list_add_tail(&te->set->pending_update,
9817                                               &set_update_list);
9818                         }
9819                         break;
9820                 case NFT_MSG_DELSETELEM:
9821                 case NFT_MSG_DESTROYSETELEM:
9822                         te = (struct nft_trans_elem *)trans->data;
9823
9824                         nft_setelem_data_activate(net, te->set, &te->elem);
9825                         nft_setelem_activate(net, te->set, &te->elem);
9826                         if (!nft_setelem_is_catchall(te->set, &te->elem))
9827                                 te->set->ndeact--;
9828
9829                         if (te->set->ops->abort &&
9830                             list_empty(&te->set->pending_update)) {
9831                                 list_add_tail(&te->set->pending_update,
9832                                               &set_update_list);
9833                         }
9834                         nft_trans_destroy(trans);
9835                         break;
9836                 case NFT_MSG_NEWOBJ:
9837                         if (nft_trans_obj_update(trans)) {
9838                                 nft_obj_destroy(&trans->ctx, nft_trans_obj_newobj(trans));
9839                                 nft_trans_destroy(trans);
9840                         } else {
9841                                 trans->ctx.table->use--;
9842                                 nft_obj_del(nft_trans_obj(trans));
9843                         }
9844                         break;
9845                 case NFT_MSG_DELOBJ:
9846                 case NFT_MSG_DESTROYOBJ:
9847                         trans->ctx.table->use++;
9848                         nft_clear(trans->ctx.net, nft_trans_obj(trans));
9849                         nft_trans_destroy(trans);
9850                         break;
9851                 case NFT_MSG_NEWFLOWTABLE:
9852                         if (nft_trans_flowtable_update(trans)) {
9853                                 nft_unregister_flowtable_net_hooks(net,
9854                                                 &nft_trans_flowtable_hooks(trans));
9855                         } else {
9856                                 trans->ctx.table->use--;
9857                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
9858                                 nft_unregister_flowtable_net_hooks(net,
9859                                                 &nft_trans_flowtable(trans)->hook_list);
9860                         }
9861                         break;
9862                 case NFT_MSG_DELFLOWTABLE:
9863                 case NFT_MSG_DESTROYFLOWTABLE:
9864                         if (nft_trans_flowtable_update(trans)) {
9865                                 list_splice(&nft_trans_flowtable_hooks(trans),
9866                                             &nft_trans_flowtable(trans)->hook_list);
9867                         } else {
9868                                 trans->ctx.table->use++;
9869                                 nft_clear(trans->ctx.net, nft_trans_flowtable(trans));
9870                         }
9871                         nft_trans_destroy(trans);
9872                         break;
9873                 }
9874         }
9875
9876         nft_set_abort_update(&set_update_list);
9877
9878         synchronize_rcu();
9879
9880         list_for_each_entry_safe_reverse(trans, next,
9881                                          &nft_net->commit_list, list) {
9882                 list_del(&trans->list);
9883                 nf_tables_abort_release(trans);
9884         }
9885
9886         if (action == NFNL_ABORT_AUTOLOAD)
9887                 nf_tables_module_autoload(net);
9888         else
9889                 nf_tables_module_autoload_cleanup(net);
9890
9891         return 0;
9892 }
9893
9894 static int nf_tables_abort(struct net *net, struct sk_buff *skb,
9895                            enum nfnl_abort_action action)
9896 {
9897         struct nftables_pernet *nft_net = nft_pernet(net);
9898         int ret = __nf_tables_abort(net, action);
9899
9900         mutex_unlock(&nft_net->commit_mutex);
9901
9902         return ret;
9903 }
9904
9905 static bool nf_tables_valid_genid(struct net *net, u32 genid)
9906 {
9907         struct nftables_pernet *nft_net = nft_pernet(net);
9908         bool genid_ok;
9909
9910         mutex_lock(&nft_net->commit_mutex);
9911
9912         genid_ok = genid == 0 || nft_net->base_seq == genid;
9913         if (!genid_ok)
9914                 mutex_unlock(&nft_net->commit_mutex);
9915
9916         /* else, commit mutex has to be released by commit or abort function */
9917         return genid_ok;
9918 }
9919
9920 static const struct nfnetlink_subsystem nf_tables_subsys = {
9921         .name           = "nf_tables",
9922         .subsys_id      = NFNL_SUBSYS_NFTABLES,
9923         .cb_count       = NFT_MSG_MAX,
9924         .cb             = nf_tables_cb,
9925         .commit         = nf_tables_commit,
9926         .abort          = nf_tables_abort,
9927         .valid_genid    = nf_tables_valid_genid,
9928         .owner          = THIS_MODULE,
9929 };
9930
9931 int nft_chain_validate_dependency(const struct nft_chain *chain,
9932                                   enum nft_chain_types type)
9933 {
9934         const struct nft_base_chain *basechain;
9935
9936         if (nft_is_base_chain(chain)) {
9937                 basechain = nft_base_chain(chain);
9938                 if (basechain->type->type != type)
9939                         return -EOPNOTSUPP;
9940         }
9941         return 0;
9942 }
9943 EXPORT_SYMBOL_GPL(nft_chain_validate_dependency);
9944
9945 int nft_chain_validate_hooks(const struct nft_chain *chain,
9946                              unsigned int hook_flags)
9947 {
9948         struct nft_base_chain *basechain;
9949
9950         if (nft_is_base_chain(chain)) {
9951                 basechain = nft_base_chain(chain);
9952
9953                 if ((1 << basechain->ops.hooknum) & hook_flags)
9954                         return 0;
9955
9956                 return -EOPNOTSUPP;
9957         }
9958
9959         return 0;
9960 }
9961 EXPORT_SYMBOL_GPL(nft_chain_validate_hooks);
9962
9963 /*
9964  * Loop detection - walk through the ruleset beginning at the destination chain
9965  * of a new jump until either the source chain is reached (loop) or all
9966  * reachable chains have been traversed.
9967  *
9968  * The loop check is performed whenever a new jump verdict is added to an
9969  * expression or verdict map or a verdict map is bound to a new chain.
9970  */
9971
9972 static int nf_tables_check_loops(const struct nft_ctx *ctx,
9973                                  const struct nft_chain *chain);
9974
9975 static int nft_check_loops(const struct nft_ctx *ctx,
9976                            const struct nft_set_ext *ext)
9977 {
9978         const struct nft_data *data;
9979         int ret;
9980
9981         data = nft_set_ext_data(ext);
9982         switch (data->verdict.code) {
9983         case NFT_JUMP:
9984         case NFT_GOTO:
9985                 ret = nf_tables_check_loops(ctx, data->verdict.chain);
9986                 break;
9987         default:
9988                 ret = 0;
9989                 break;
9990         }
9991
9992         return ret;
9993 }
9994
9995 static int nf_tables_loop_check_setelem(const struct nft_ctx *ctx,
9996                                         struct nft_set *set,
9997                                         const struct nft_set_iter *iter,
9998                                         struct nft_set_elem *elem)
9999 {
10000         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
10001
10002         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
10003             *nft_set_ext_flags(ext) & NFT_SET_ELEM_INTERVAL_END)
10004                 return 0;
10005
10006         return nft_check_loops(ctx, ext);
10007 }
10008
10009 static int nft_set_catchall_loops(const struct nft_ctx *ctx,
10010                                   struct nft_set *set)
10011 {
10012         u8 genmask = nft_genmask_next(ctx->net);
10013         struct nft_set_elem_catchall *catchall;
10014         struct nft_set_ext *ext;
10015         int ret = 0;
10016
10017         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
10018                 ext = nft_set_elem_ext(set, catchall->elem);
10019                 if (!nft_set_elem_active(ext, genmask))
10020                         continue;
10021
10022                 ret = nft_check_loops(ctx, ext);
10023                 if (ret < 0)
10024                         return ret;
10025         }
10026
10027         return ret;
10028 }
10029
10030 static int nf_tables_check_loops(const struct nft_ctx *ctx,
10031                                  const struct nft_chain *chain)
10032 {
10033         const struct nft_rule *rule;
10034         const struct nft_expr *expr, *last;
10035         struct nft_set *set;
10036         struct nft_set_binding *binding;
10037         struct nft_set_iter iter;
10038
10039         if (ctx->chain == chain)
10040                 return -ELOOP;
10041
10042         list_for_each_entry(rule, &chain->rules, list) {
10043                 nft_rule_for_each_expr(expr, last, rule) {
10044                         struct nft_immediate_expr *priv;
10045                         const struct nft_data *data;
10046                         int err;
10047
10048                         if (strcmp(expr->ops->type->name, "immediate"))
10049                                 continue;
10050
10051                         priv = nft_expr_priv(expr);
10052                         if (priv->dreg != NFT_REG_VERDICT)
10053                                 continue;
10054
10055                         data = &priv->data;
10056                         switch (data->verdict.code) {
10057                         case NFT_JUMP:
10058                         case NFT_GOTO:
10059                                 err = nf_tables_check_loops(ctx,
10060                                                         data->verdict.chain);
10061                                 if (err < 0)
10062                                         return err;
10063                                 break;
10064                         default:
10065                                 break;
10066                         }
10067                 }
10068         }
10069
10070         list_for_each_entry(set, &ctx->table->sets, list) {
10071                 if (!nft_is_active_next(ctx->net, set))
10072                         continue;
10073                 if (!(set->flags & NFT_SET_MAP) ||
10074                     set->dtype != NFT_DATA_VERDICT)
10075                         continue;
10076
10077                 list_for_each_entry(binding, &set->bindings, list) {
10078                         if (!(binding->flags & NFT_SET_MAP) ||
10079                             binding->chain != chain)
10080                                 continue;
10081
10082                         iter.genmask    = nft_genmask_next(ctx->net);
10083                         iter.skip       = 0;
10084                         iter.count      = 0;
10085                         iter.err        = 0;
10086                         iter.fn         = nf_tables_loop_check_setelem;
10087
10088                         set->ops->walk(ctx, set, &iter);
10089                         if (!iter.err)
10090                                 iter.err = nft_set_catchall_loops(ctx, set);
10091
10092                         if (iter.err < 0)
10093                                 return iter.err;
10094                 }
10095         }
10096
10097         return 0;
10098 }
10099
10100 /**
10101  *      nft_parse_u32_check - fetch u32 attribute and check for maximum value
10102  *
10103  *      @attr: netlink attribute to fetch value from
10104  *      @max: maximum value to be stored in dest
10105  *      @dest: pointer to the variable
10106  *
10107  *      Parse, check and store a given u32 netlink attribute into variable.
10108  *      This function returns -ERANGE if the value goes over maximum value.
10109  *      Otherwise a 0 is returned and the attribute value is stored in the
10110  *      destination variable.
10111  */
10112 int nft_parse_u32_check(const struct nlattr *attr, int max, u32 *dest)
10113 {
10114         u32 val;
10115
10116         val = ntohl(nla_get_be32(attr));
10117         if (val > max)
10118                 return -ERANGE;
10119
10120         *dest = val;
10121         return 0;
10122 }
10123 EXPORT_SYMBOL_GPL(nft_parse_u32_check);
10124
10125 static int nft_parse_register(const struct nlattr *attr, u32 *preg)
10126 {
10127         unsigned int reg;
10128
10129         reg = ntohl(nla_get_be32(attr));
10130         switch (reg) {
10131         case NFT_REG_VERDICT...NFT_REG_4:
10132                 *preg = reg * NFT_REG_SIZE / NFT_REG32_SIZE;
10133                 break;
10134         case NFT_REG32_00...NFT_REG32_15:
10135                 *preg = reg + NFT_REG_SIZE / NFT_REG32_SIZE - NFT_REG32_00;
10136                 break;
10137         default:
10138                 return -ERANGE;
10139         }
10140
10141         return 0;
10142 }
10143
10144 /**
10145  *      nft_dump_register - dump a register value to a netlink attribute
10146  *
10147  *      @skb: socket buffer
10148  *      @attr: attribute number
10149  *      @reg: register number
10150  *
10151  *      Construct a netlink attribute containing the register number. For
10152  *      compatibility reasons, register numbers being a multiple of 4 are
10153  *      translated to the corresponding 128 bit register numbers.
10154  */
10155 int nft_dump_register(struct sk_buff *skb, unsigned int attr, unsigned int reg)
10156 {
10157         if (reg % (NFT_REG_SIZE / NFT_REG32_SIZE) == 0)
10158                 reg = reg / (NFT_REG_SIZE / NFT_REG32_SIZE);
10159         else
10160                 reg = reg - NFT_REG_SIZE / NFT_REG32_SIZE + NFT_REG32_00;
10161
10162         return nla_put_be32(skb, attr, htonl(reg));
10163 }
10164 EXPORT_SYMBOL_GPL(nft_dump_register);
10165
10166 static int nft_validate_register_load(enum nft_registers reg, unsigned int len)
10167 {
10168         if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
10169                 return -EINVAL;
10170         if (len == 0)
10171                 return -EINVAL;
10172         if (reg * NFT_REG32_SIZE + len > sizeof_field(struct nft_regs, data))
10173                 return -ERANGE;
10174
10175         return 0;
10176 }
10177
10178 int nft_parse_register_load(const struct nlattr *attr, u8 *sreg, u32 len)
10179 {
10180         u32 reg;
10181         int err;
10182
10183         err = nft_parse_register(attr, &reg);
10184         if (err < 0)
10185                 return err;
10186
10187         err = nft_validate_register_load(reg, len);
10188         if (err < 0)
10189                 return err;
10190
10191         *sreg = reg;
10192         return 0;
10193 }
10194 EXPORT_SYMBOL_GPL(nft_parse_register_load);
10195
10196 static int nft_validate_register_store(const struct nft_ctx *ctx,
10197                                        enum nft_registers reg,
10198                                        const struct nft_data *data,
10199                                        enum nft_data_types type,
10200                                        unsigned int len)
10201 {
10202         int err;
10203
10204         switch (reg) {
10205         case NFT_REG_VERDICT:
10206                 if (type != NFT_DATA_VERDICT)
10207                         return -EINVAL;
10208
10209                 if (data != NULL &&
10210                     (data->verdict.code == NFT_GOTO ||
10211                      data->verdict.code == NFT_JUMP)) {
10212                         err = nf_tables_check_loops(ctx, data->verdict.chain);
10213                         if (err < 0)
10214                                 return err;
10215                 }
10216
10217                 return 0;
10218         default:
10219                 if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
10220                         return -EINVAL;
10221                 if (len == 0)
10222                         return -EINVAL;
10223                 if (reg * NFT_REG32_SIZE + len >
10224                     sizeof_field(struct nft_regs, data))
10225                         return -ERANGE;
10226
10227                 if (data != NULL && type != NFT_DATA_VALUE)
10228                         return -EINVAL;
10229                 return 0;
10230         }
10231 }
10232
10233 int nft_parse_register_store(const struct nft_ctx *ctx,
10234                              const struct nlattr *attr, u8 *dreg,
10235                              const struct nft_data *data,
10236                              enum nft_data_types type, unsigned int len)
10237 {
10238         int err;
10239         u32 reg;
10240
10241         err = nft_parse_register(attr, &reg);
10242         if (err < 0)
10243                 return err;
10244
10245         err = nft_validate_register_store(ctx, reg, data, type, len);
10246         if (err < 0)
10247                 return err;
10248
10249         *dreg = reg;
10250         return 0;
10251 }
10252 EXPORT_SYMBOL_GPL(nft_parse_register_store);
10253
10254 static const struct nla_policy nft_verdict_policy[NFTA_VERDICT_MAX + 1] = {
10255         [NFTA_VERDICT_CODE]     = { .type = NLA_U32 },
10256         [NFTA_VERDICT_CHAIN]    = { .type = NLA_STRING,
10257                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
10258         [NFTA_VERDICT_CHAIN_ID] = { .type = NLA_U32 },
10259 };
10260
10261 static int nft_verdict_init(const struct nft_ctx *ctx, struct nft_data *data,
10262                             struct nft_data_desc *desc, const struct nlattr *nla)
10263 {
10264         u8 genmask = nft_genmask_next(ctx->net);
10265         struct nlattr *tb[NFTA_VERDICT_MAX + 1];
10266         struct nft_chain *chain;
10267         int err;
10268
10269         err = nla_parse_nested_deprecated(tb, NFTA_VERDICT_MAX, nla,
10270                                           nft_verdict_policy, NULL);
10271         if (err < 0)
10272                 return err;
10273
10274         if (!tb[NFTA_VERDICT_CODE])
10275                 return -EINVAL;
10276         data->verdict.code = ntohl(nla_get_be32(tb[NFTA_VERDICT_CODE]));
10277
10278         switch (data->verdict.code) {
10279         default:
10280                 switch (data->verdict.code & NF_VERDICT_MASK) {
10281                 case NF_ACCEPT:
10282                 case NF_DROP:
10283                 case NF_QUEUE:
10284                         break;
10285                 default:
10286                         return -EINVAL;
10287                 }
10288                 fallthrough;
10289         case NFT_CONTINUE:
10290         case NFT_BREAK:
10291         case NFT_RETURN:
10292                 break;
10293         case NFT_JUMP:
10294         case NFT_GOTO:
10295                 if (tb[NFTA_VERDICT_CHAIN]) {
10296                         chain = nft_chain_lookup(ctx->net, ctx->table,
10297                                                  tb[NFTA_VERDICT_CHAIN],
10298                                                  genmask);
10299                 } else if (tb[NFTA_VERDICT_CHAIN_ID]) {
10300                         chain = nft_chain_lookup_byid(ctx->net, ctx->table,
10301                                                       tb[NFTA_VERDICT_CHAIN_ID]);
10302                         if (IS_ERR(chain))
10303                                 return PTR_ERR(chain);
10304                 } else {
10305                         return -EINVAL;
10306                 }
10307
10308                 if (IS_ERR(chain))
10309                         return PTR_ERR(chain);
10310                 if (nft_is_base_chain(chain))
10311                         return -EOPNOTSUPP;
10312                 if (nft_chain_is_bound(chain))
10313                         return -EINVAL;
10314                 if (desc->flags & NFT_DATA_DESC_SETELEM &&
10315                     chain->flags & NFT_CHAIN_BINDING)
10316                         return -EINVAL;
10317
10318                 chain->use++;
10319                 data->verdict.chain = chain;
10320                 break;
10321         }
10322
10323         desc->len = sizeof(data->verdict);
10324
10325         return 0;
10326 }
10327
10328 static void nft_verdict_uninit(const struct nft_data *data)
10329 {
10330         struct nft_chain *chain;
10331
10332         switch (data->verdict.code) {
10333         case NFT_JUMP:
10334         case NFT_GOTO:
10335                 chain = data->verdict.chain;
10336                 chain->use--;
10337                 break;
10338         }
10339 }
10340
10341 int nft_verdict_dump(struct sk_buff *skb, int type, const struct nft_verdict *v)
10342 {
10343         struct nlattr *nest;
10344
10345         nest = nla_nest_start_noflag(skb, type);
10346         if (!nest)
10347                 goto nla_put_failure;
10348
10349         if (nla_put_be32(skb, NFTA_VERDICT_CODE, htonl(v->code)))
10350                 goto nla_put_failure;
10351
10352         switch (v->code) {
10353         case NFT_JUMP:
10354         case NFT_GOTO:
10355                 if (nla_put_string(skb, NFTA_VERDICT_CHAIN,
10356                                    v->chain->name))
10357                         goto nla_put_failure;
10358         }
10359         nla_nest_end(skb, nest);
10360         return 0;
10361
10362 nla_put_failure:
10363         return -1;
10364 }
10365
10366 static int nft_value_init(const struct nft_ctx *ctx,
10367                           struct nft_data *data, struct nft_data_desc *desc,
10368                           const struct nlattr *nla)
10369 {
10370         unsigned int len;
10371
10372         len = nla_len(nla);
10373         if (len == 0)
10374                 return -EINVAL;
10375         if (len > desc->size)
10376                 return -EOVERFLOW;
10377         if (desc->len) {
10378                 if (len != desc->len)
10379                         return -EINVAL;
10380         } else {
10381                 desc->len = len;
10382         }
10383
10384         nla_memcpy(data->data, nla, len);
10385
10386         return 0;
10387 }
10388
10389 static int nft_value_dump(struct sk_buff *skb, const struct nft_data *data,
10390                           unsigned int len)
10391 {
10392         return nla_put(skb, NFTA_DATA_VALUE, len, data->data);
10393 }
10394
10395 static const struct nla_policy nft_data_policy[NFTA_DATA_MAX + 1] = {
10396         [NFTA_DATA_VALUE]       = { .type = NLA_BINARY },
10397         [NFTA_DATA_VERDICT]     = { .type = NLA_NESTED },
10398 };
10399
10400 /**
10401  *      nft_data_init - parse nf_tables data netlink attributes
10402  *
10403  *      @ctx: context of the expression using the data
10404  *      @data: destination struct nft_data
10405  *      @desc: data description
10406  *      @nla: netlink attribute containing data
10407  *
10408  *      Parse the netlink data attributes and initialize a struct nft_data.
10409  *      The type and length of data are returned in the data description.
10410  *
10411  *      The caller can indicate that it only wants to accept data of type
10412  *      NFT_DATA_VALUE by passing NULL for the ctx argument.
10413  */
10414 int nft_data_init(const struct nft_ctx *ctx, struct nft_data *data,
10415                   struct nft_data_desc *desc, const struct nlattr *nla)
10416 {
10417         struct nlattr *tb[NFTA_DATA_MAX + 1];
10418         int err;
10419
10420         if (WARN_ON_ONCE(!desc->size))
10421                 return -EINVAL;
10422
10423         err = nla_parse_nested_deprecated(tb, NFTA_DATA_MAX, nla,
10424                                           nft_data_policy, NULL);
10425         if (err < 0)
10426                 return err;
10427
10428         if (tb[NFTA_DATA_VALUE]) {
10429                 if (desc->type != NFT_DATA_VALUE)
10430                         return -EINVAL;
10431
10432                 err = nft_value_init(ctx, data, desc, tb[NFTA_DATA_VALUE]);
10433         } else if (tb[NFTA_DATA_VERDICT] && ctx != NULL) {
10434                 if (desc->type != NFT_DATA_VERDICT)
10435                         return -EINVAL;
10436
10437                 err = nft_verdict_init(ctx, data, desc, tb[NFTA_DATA_VERDICT]);
10438         } else {
10439                 err = -EINVAL;
10440         }
10441
10442         return err;
10443 }
10444 EXPORT_SYMBOL_GPL(nft_data_init);
10445
10446 /**
10447  *      nft_data_release - release a nft_data item
10448  *
10449  *      @data: struct nft_data to release
10450  *      @type: type of data
10451  *
10452  *      Release a nft_data item. NFT_DATA_VALUE types can be silently discarded,
10453  *      all others need to be released by calling this function.
10454  */
10455 void nft_data_release(const struct nft_data *data, enum nft_data_types type)
10456 {
10457         if (type < NFT_DATA_VERDICT)
10458                 return;
10459         switch (type) {
10460         case NFT_DATA_VERDICT:
10461                 return nft_verdict_uninit(data);
10462         default:
10463                 WARN_ON(1);
10464         }
10465 }
10466 EXPORT_SYMBOL_GPL(nft_data_release);
10467
10468 int nft_data_dump(struct sk_buff *skb, int attr, const struct nft_data *data,
10469                   enum nft_data_types type, unsigned int len)
10470 {
10471         struct nlattr *nest;
10472         int err;
10473
10474         nest = nla_nest_start_noflag(skb, attr);
10475         if (nest == NULL)
10476                 return -1;
10477
10478         switch (type) {
10479         case NFT_DATA_VALUE:
10480                 err = nft_value_dump(skb, data, len);
10481                 break;
10482         case NFT_DATA_VERDICT:
10483                 err = nft_verdict_dump(skb, NFTA_DATA_VERDICT, &data->verdict);
10484                 break;
10485         default:
10486                 err = -EINVAL;
10487                 WARN_ON(1);
10488         }
10489
10490         nla_nest_end(skb, nest);
10491         return err;
10492 }
10493 EXPORT_SYMBOL_GPL(nft_data_dump);
10494
10495 int __nft_release_basechain(struct nft_ctx *ctx)
10496 {
10497         struct nft_rule *rule, *nr;
10498
10499         if (WARN_ON(!nft_is_base_chain(ctx->chain)))
10500                 return 0;
10501
10502         nf_tables_unregister_hook(ctx->net, ctx->chain->table, ctx->chain);
10503         list_for_each_entry_safe(rule, nr, &ctx->chain->rules, list) {
10504                 list_del(&rule->list);
10505                 ctx->chain->use--;
10506                 nf_tables_rule_release(ctx, rule);
10507         }
10508         nft_chain_del(ctx->chain);
10509         ctx->table->use--;
10510         nf_tables_chain_destroy(ctx);
10511
10512         return 0;
10513 }
10514 EXPORT_SYMBOL_GPL(__nft_release_basechain);
10515
10516 static void __nft_release_hook(struct net *net, struct nft_table *table)
10517 {
10518         struct nft_flowtable *flowtable;
10519         struct nft_chain *chain;
10520
10521         list_for_each_entry(chain, &table->chains, list)
10522                 __nf_tables_unregister_hook(net, table, chain, true);
10523         list_for_each_entry(flowtable, &table->flowtables, list)
10524                 __nft_unregister_flowtable_net_hooks(net, &flowtable->hook_list,
10525                                                      true);
10526 }
10527
10528 static void __nft_release_hooks(struct net *net)
10529 {
10530         struct nftables_pernet *nft_net = nft_pernet(net);
10531         struct nft_table *table;
10532
10533         list_for_each_entry(table, &nft_net->tables, list) {
10534                 if (nft_table_has_owner(table))
10535                         continue;
10536
10537                 __nft_release_hook(net, table);
10538         }
10539 }
10540
10541 static void __nft_release_table(struct net *net, struct nft_table *table)
10542 {
10543         struct nft_flowtable *flowtable, *nf;
10544         struct nft_chain *chain, *nc;
10545         struct nft_object *obj, *ne;
10546         struct nft_rule *rule, *nr;
10547         struct nft_set *set, *ns;
10548         struct nft_ctx ctx = {
10549                 .net    = net,
10550                 .family = NFPROTO_NETDEV,
10551         };
10552
10553         ctx.family = table->family;
10554         ctx.table = table;
10555         list_for_each_entry(chain, &table->chains, list) {
10556                 ctx.chain = chain;
10557                 list_for_each_entry_safe(rule, nr, &chain->rules, list) {
10558                         list_del(&rule->list);
10559                         chain->use--;
10560                         nf_tables_rule_release(&ctx, rule);
10561                 }
10562         }
10563         list_for_each_entry_safe(flowtable, nf, &table->flowtables, list) {
10564                 list_del(&flowtable->list);
10565                 table->use--;
10566                 nf_tables_flowtable_destroy(flowtable);
10567         }
10568         list_for_each_entry_safe(set, ns, &table->sets, list) {
10569                 list_del(&set->list);
10570                 table->use--;
10571                 nft_set_destroy(&ctx, set);
10572         }
10573         list_for_each_entry_safe(obj, ne, &table->objects, list) {
10574                 nft_obj_del(obj);
10575                 table->use--;
10576                 nft_obj_destroy(&ctx, obj);
10577         }
10578         list_for_each_entry_safe(chain, nc, &table->chains, list) {
10579                 ctx.chain = chain;
10580                 nft_chain_del(chain);
10581                 table->use--;
10582                 nf_tables_chain_destroy(&ctx);
10583         }
10584         nf_tables_table_destroy(&ctx);
10585 }
10586
10587 static void __nft_release_tables(struct net *net)
10588 {
10589         struct nftables_pernet *nft_net = nft_pernet(net);
10590         struct nft_table *table, *nt;
10591
10592         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
10593                 if (nft_table_has_owner(table))
10594                         continue;
10595
10596                 list_del(&table->list);
10597
10598                 __nft_release_table(net, table);
10599         }
10600 }
10601
10602 static int nft_rcv_nl_event(struct notifier_block *this, unsigned long event,
10603                             void *ptr)
10604 {
10605         struct nft_table *table, *to_delete[8];
10606         struct nftables_pernet *nft_net;
10607         struct netlink_notify *n = ptr;
10608         struct net *net = n->net;
10609         unsigned int deleted;
10610         bool restart = false;
10611
10612         if (event != NETLINK_URELEASE || n->protocol != NETLINK_NETFILTER)
10613                 return NOTIFY_DONE;
10614
10615         nft_net = nft_pernet(net);
10616         deleted = 0;
10617         mutex_lock(&nft_net->commit_mutex);
10618         if (!list_empty(&nf_tables_destroy_list))
10619                 rcu_barrier();
10620 again:
10621         list_for_each_entry(table, &nft_net->tables, list) {
10622                 if (nft_table_has_owner(table) &&
10623                     n->portid == table->nlpid) {
10624                         __nft_release_hook(net, table);
10625                         list_del_rcu(&table->list);
10626                         to_delete[deleted++] = table;
10627                         if (deleted >= ARRAY_SIZE(to_delete))
10628                                 break;
10629                 }
10630         }
10631         if (deleted) {
10632                 restart = deleted >= ARRAY_SIZE(to_delete);
10633                 synchronize_rcu();
10634                 while (deleted)
10635                         __nft_release_table(net, to_delete[--deleted]);
10636
10637                 if (restart)
10638                         goto again;
10639         }
10640         mutex_unlock(&nft_net->commit_mutex);
10641
10642         return NOTIFY_DONE;
10643 }
10644
10645 static struct notifier_block nft_nl_notifier = {
10646         .notifier_call  = nft_rcv_nl_event,
10647 };
10648
10649 static int __net_init nf_tables_init_net(struct net *net)
10650 {
10651         struct nftables_pernet *nft_net = nft_pernet(net);
10652
10653         INIT_LIST_HEAD(&nft_net->tables);
10654         INIT_LIST_HEAD(&nft_net->commit_list);
10655         INIT_LIST_HEAD(&nft_net->module_list);
10656         INIT_LIST_HEAD(&nft_net->notify_list);
10657         mutex_init(&nft_net->commit_mutex);
10658         nft_net->base_seq = 1;
10659
10660         return 0;
10661 }
10662
10663 static void __net_exit nf_tables_pre_exit_net(struct net *net)
10664 {
10665         struct nftables_pernet *nft_net = nft_pernet(net);
10666
10667         mutex_lock(&nft_net->commit_mutex);
10668         __nft_release_hooks(net);
10669         mutex_unlock(&nft_net->commit_mutex);
10670 }
10671
10672 static void __net_exit nf_tables_exit_net(struct net *net)
10673 {
10674         struct nftables_pernet *nft_net = nft_pernet(net);
10675
10676         mutex_lock(&nft_net->commit_mutex);
10677         if (!list_empty(&nft_net->commit_list) ||
10678             !list_empty(&nft_net->module_list))
10679                 __nf_tables_abort(net, NFNL_ABORT_NONE);
10680         __nft_release_tables(net);
10681         mutex_unlock(&nft_net->commit_mutex);
10682         WARN_ON_ONCE(!list_empty(&nft_net->tables));
10683         WARN_ON_ONCE(!list_empty(&nft_net->module_list));
10684         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
10685 }
10686
10687 static struct pernet_operations nf_tables_net_ops = {
10688         .init           = nf_tables_init_net,
10689         .pre_exit       = nf_tables_pre_exit_net,
10690         .exit           = nf_tables_exit_net,
10691         .id             = &nf_tables_net_id,
10692         .size           = sizeof(struct nftables_pernet),
10693 };
10694
10695 static int __init nf_tables_module_init(void)
10696 {
10697         int err;
10698
10699         err = register_pernet_subsys(&nf_tables_net_ops);
10700         if (err < 0)
10701                 return err;
10702
10703         err = nft_chain_filter_init();
10704         if (err < 0)
10705                 goto err_chain_filter;
10706
10707         err = nf_tables_core_module_init();
10708         if (err < 0)
10709                 goto err_core_module;
10710
10711         err = register_netdevice_notifier(&nf_tables_flowtable_notifier);
10712         if (err < 0)
10713                 goto err_netdev_notifier;
10714
10715         err = rhltable_init(&nft_objname_ht, &nft_objname_ht_params);
10716         if (err < 0)
10717                 goto err_rht_objname;
10718
10719         err = nft_offload_init();
10720         if (err < 0)
10721                 goto err_offload;
10722
10723         err = netlink_register_notifier(&nft_nl_notifier);
10724         if (err < 0)
10725                 goto err_netlink_notifier;
10726
10727         /* must be last */
10728         err = nfnetlink_subsys_register(&nf_tables_subsys);
10729         if (err < 0)
10730                 goto err_nfnl_subsys;
10731
10732         nft_chain_route_init();
10733
10734         return err;
10735
10736 err_nfnl_subsys:
10737         netlink_unregister_notifier(&nft_nl_notifier);
10738 err_netlink_notifier:
10739         nft_offload_exit();
10740 err_offload:
10741         rhltable_destroy(&nft_objname_ht);
10742 err_rht_objname:
10743         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
10744 err_netdev_notifier:
10745         nf_tables_core_module_exit();
10746 err_core_module:
10747         nft_chain_filter_fini();
10748 err_chain_filter:
10749         unregister_pernet_subsys(&nf_tables_net_ops);
10750         return err;
10751 }
10752
10753 static void __exit nf_tables_module_exit(void)
10754 {
10755         nfnetlink_subsys_unregister(&nf_tables_subsys);
10756         netlink_unregister_notifier(&nft_nl_notifier);
10757         nft_offload_exit();
10758         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
10759         nft_chain_filter_fini();
10760         nft_chain_route_fini();
10761         unregister_pernet_subsys(&nf_tables_net_ops);
10762         cancel_work_sync(&trans_destroy_work);
10763         rcu_barrier();
10764         rhltable_destroy(&nft_objname_ht);
10765         nf_tables_core_module_exit();
10766 }
10767
10768 module_init(nf_tables_module_init);
10769 module_exit(nf_tables_module_exit);
10770
10771 MODULE_LICENSE("GPL");
10772 MODULE_AUTHOR("Patrick McHardy <kaber@trash.net>");
10773 MODULE_ALIAS_NFNL_SUBSYS(NFNL_SUBSYS_NFTABLES);