net/netfilter/nf_tables_api.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2007-2009 Patrick McHardy <kaber@trash.net>
   4  *
   5  * Development of this code funded by Astaro AG (http://www.astaro.com/)
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/init.h>
  10 #include <linux/list.h>
  11 #include <linux/skbuff.h>
  12 #include <linux/netlink.h>
  13 #include <linux/vmalloc.h>
  14 #include <linux/rhashtable.h>
  15 #include <linux/audit.h>
  16 #include <linux/netfilter.h>
  17 #include <linux/netfilter/nfnetlink.h>
  18 #include <linux/netfilter/nf_tables.h>
  19 #include <net/netfilter/nf_flow_table.h>
  20 #include <net/netfilter/nf_tables_core.h>
  21 #include <net/netfilter/nf_tables.h>
  22 #include <net/netfilter/nf_tables_offload.h>
  23 #include <net/net_namespace.h>
  24 #include <net/sock.h>
  25
  26 #define NFT_MODULE_AUTOLOAD_LIMIT (MODULE_NAME_LEN - sizeof("nft-expr-255-"))
  27
  28 unsigned int nf_tables_net_id __read_mostly;
  29
  30 static LIST_HEAD(nf_tables_expressions);
  31 static LIST_HEAD(nf_tables_objects);
  32 static LIST_HEAD(nf_tables_flowtables);
  33 static LIST_HEAD(nf_tables_destroy_list);
  34 static LIST_HEAD(nf_tables_gc_list);
  35 static DEFINE_SPINLOCK(nf_tables_destroy_list_lock);
  36 static DEFINE_SPINLOCK(nf_tables_gc_list_lock);
  37
  38 enum {
  39         NFT_VALIDATE_SKIP       = 0,
  40         NFT_VALIDATE_NEED,
  41         NFT_VALIDATE_DO,
  42 };
  43
  44 static struct rhltable nft_objname_ht;
  45
  46 static u32 nft_chain_hash(const void *data, u32 len, u32 seed);
  47 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed);
  48 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *, const void *);
  49
  50 static u32 nft_objname_hash(const void *data, u32 len, u32 seed);
  51 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed);
  52 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *, const void *);
  53
  54 static const struct rhashtable_params nft_chain_ht_params = {
  55         .head_offset            = offsetof(struct nft_chain, rhlhead),
  56         .key_offset             = offsetof(struct nft_chain, name),
  57         .hashfn                 = nft_chain_hash,
  58         .obj_hashfn             = nft_chain_hash_obj,
  59         .obj_cmpfn              = nft_chain_hash_cmp,
  60         .automatic_shrinking    = true,
  61 };
  62
  63 static const struct rhashtable_params nft_objname_ht_params = {
  64         .head_offset            = offsetof(struct nft_object, rhlhead),
  65         .key_offset             = offsetof(struct nft_object, key),
  66         .hashfn                 = nft_objname_hash,
  67         .obj_hashfn             = nft_objname_hash_obj,
  68         .obj_cmpfn              = nft_objname_hash_cmp,
  69         .automatic_shrinking    = true,
  70 };
  71
  72 struct nft_audit_data {
  73         struct nft_table *table;
  74         int entries;
  75         int op;
  76         struct list_head list;
  77 };
  78
  79 static const u8 nft2audit_op[NFT_MSG_MAX] = { // enum nf_tables_msg_types
  80         [NFT_MSG_NEWTABLE]      = AUDIT_NFT_OP_TABLE_REGISTER,
  81         [NFT_MSG_GETTABLE]      = AUDIT_NFT_OP_INVALID,
  82         [NFT_MSG_DELTABLE]      = AUDIT_NFT_OP_TABLE_UNREGISTER,
  83         [NFT_MSG_NEWCHAIN]      = AUDIT_NFT_OP_CHAIN_REGISTER,
  84         [NFT_MSG_GETCHAIN]      = AUDIT_NFT_OP_INVALID,
  85         [NFT_MSG_DELCHAIN]      = AUDIT_NFT_OP_CHAIN_UNREGISTER,
  86         [NFT_MSG_NEWRULE]       = AUDIT_NFT_OP_RULE_REGISTER,
  87         [NFT_MSG_GETRULE]       = AUDIT_NFT_OP_INVALID,
  88         [NFT_MSG_DELRULE]       = AUDIT_NFT_OP_RULE_UNREGISTER,
  89         [NFT_MSG_NEWSET]        = AUDIT_NFT_OP_SET_REGISTER,
  90         [NFT_MSG_GETSET]        = AUDIT_NFT_OP_INVALID,
  91         [NFT_MSG_DELSET]        = AUDIT_NFT_OP_SET_UNREGISTER,
  92         [NFT_MSG_NEWSETELEM]    = AUDIT_NFT_OP_SETELEM_REGISTER,
  93         [NFT_MSG_GETSETELEM]    = AUDIT_NFT_OP_INVALID,
  94         [NFT_MSG_DELSETELEM]    = AUDIT_NFT_OP_SETELEM_UNREGISTER,
  95         [NFT_MSG_NEWGEN]        = AUDIT_NFT_OP_GEN_REGISTER,
  96         [NFT_MSG_GETGEN]        = AUDIT_NFT_OP_INVALID,
  97         [NFT_MSG_TRACE]         = AUDIT_NFT_OP_INVALID,
  98         [NFT_MSG_NEWOBJ]        = AUDIT_NFT_OP_OBJ_REGISTER,
  99         [NFT_MSG_GETOBJ]        = AUDIT_NFT_OP_INVALID,
 100         [NFT_MSG_DELOBJ]        = AUDIT_NFT_OP_OBJ_UNREGISTER,
 101         [NFT_MSG_GETOBJ_RESET]  = AUDIT_NFT_OP_OBJ_RESET,
 102         [NFT_MSG_NEWFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_REGISTER,
 103         [NFT_MSG_GETFLOWTABLE]  = AUDIT_NFT_OP_INVALID,
 104         [NFT_MSG_DELFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_UNREGISTER,
 105         [NFT_MSG_GETSETELEM_RESET] = AUDIT_NFT_OP_SETELEM_RESET,
 106 };
 107
 108 static void nft_validate_state_update(struct nft_table *table, u8 new_validate_state)
 109 {
 110         switch (table->validate_state) {
 111         case NFT_VALIDATE_SKIP:
 112                 WARN_ON_ONCE(new_validate_state == NFT_VALIDATE_DO);
 113                 break;
 114         case NFT_VALIDATE_NEED:
 115                 break;
 116         case NFT_VALIDATE_DO:
 117                 if (new_validate_state == NFT_VALIDATE_NEED)
 118                         return;
 119         }
 120
 121         table->validate_state = new_validate_state;
 122 }
 123 static void nf_tables_trans_destroy_work(struct work_struct *w);
 124 static DECLARE_WORK(trans_destroy_work, nf_tables_trans_destroy_work);
 125
 126 static void nft_trans_gc_work(struct work_struct *work);
 127 static DECLARE_WORK(trans_gc_work, nft_trans_gc_work);
 128
 129 static void nft_ctx_init(struct nft_ctx *ctx,
 130                          struct net *net,
 131                          const struct sk_buff *skb,
 132                          const struct nlmsghdr *nlh,
 133                          u8 family,
 134                          struct nft_table *table,
 135                          struct nft_chain *chain,
 136                          const struct nlattr * const *nla)
 137 {
 138         ctx->net        = net;
 139         ctx->family     = family;
 140         ctx->level      = 0;
 141         ctx->table      = table;
 142         ctx->chain      = chain;
 143         ctx->nla        = nla;
 144         ctx->portid     = NETLINK_CB(skb).portid;
 145         ctx->report     = nlmsg_report(nlh);
 146         ctx->flags      = nlh->nlmsg_flags;
 147         ctx->seq        = nlh->nlmsg_seq;
 148 }
 149
 150 static struct nft_trans *nft_trans_alloc_gfp(const struct nft_ctx *ctx,
 151                                              int msg_type, u32 size, gfp_t gfp)
 152 {
 153         struct nft_trans *trans;
 154
 155         trans = kzalloc(sizeof(struct nft_trans) + size, gfp);
 156         if (trans == NULL)
 157                 return NULL;
 158
 159         INIT_LIST_HEAD(&trans->list);
 160         INIT_LIST_HEAD(&trans->binding_list);
 161         trans->msg_type = msg_type;
 162         trans->ctx      = *ctx;
 163
 164         return trans;
 165 }
 166
 167 static struct nft_trans *nft_trans_alloc(const struct nft_ctx *ctx,
 168                                          int msg_type, u32 size)
 169 {
 170         return nft_trans_alloc_gfp(ctx, msg_type, size, GFP_KERNEL);
 171 }
 172
 173 static void nft_trans_list_del(struct nft_trans *trans)
 174 {
 175         list_del(&trans->list);
 176         list_del(&trans->binding_list);
 177 }
 178
 179 static void nft_trans_destroy(struct nft_trans *trans)
 180 {
 181         nft_trans_list_del(trans);
 182         kfree(trans);
 183 }
 184
 185 static void __nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set,
 186                                  bool bind)
 187 {
 188         struct nftables_pernet *nft_net;
 189         struct net *net = ctx->net;
 190         struct nft_trans *trans;
 191
 192         if (!nft_set_is_anonymous(set))
 193                 return;
 194
 195         nft_net = nft_pernet(net);
 196         list_for_each_entry_reverse(trans, &nft_net->commit_list, list) {
 197                 switch (trans->msg_type) {
 198                 case NFT_MSG_NEWSET:
 199                         if (nft_trans_set(trans) == set)
 200                                 nft_trans_set_bound(trans) = bind;
 201                         break;
 202                 case NFT_MSG_NEWSETELEM:
 203                         if (nft_trans_elem_set(trans) == set)
 204                                 nft_trans_elem_set_bound(trans) = bind;
 205                         break;
 206                 }
 207         }
 208 }
 209
 210 static void nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set)
 211 {
 212         return __nft_set_trans_bind(ctx, set, true);
 213 }
 214
 215 static void nft_set_trans_unbind(const struct nft_ctx *ctx, struct nft_set *set)
 216 {
 217         return __nft_set_trans_bind(ctx, set, false);
 218 }
 219
 220 static void __nft_chain_trans_bind(const struct nft_ctx *ctx,
 221                                    struct nft_chain *chain, bool bind)
 222 {
 223         struct nftables_pernet *nft_net;
 224         struct net *net = ctx->net;
 225         struct nft_trans *trans;
 226
 227         if (!nft_chain_binding(chain))
 228                 return;
 229
 230         nft_net = nft_pernet(net);
 231         list_for_each_entry_reverse(trans, &nft_net->commit_list, list) {
 232                 switch (trans->msg_type) {
 233                 case NFT_MSG_NEWCHAIN:
 234                         if (nft_trans_chain(trans) == chain)
 235                                 nft_trans_chain_bound(trans) = bind;
 236                         break;
 237                 case NFT_MSG_NEWRULE:
 238                         if (trans->ctx.chain == chain)
 239                                 nft_trans_rule_bound(trans) = bind;
 240                         break;
 241                 }
 242         }
 243 }
 244
 245 static void nft_chain_trans_bind(const struct nft_ctx *ctx,
 246                                  struct nft_chain *chain)
 247 {
 248         __nft_chain_trans_bind(ctx, chain, true);
 249 }
 250
 251 int nf_tables_bind_chain(const struct nft_ctx *ctx, struct nft_chain *chain)
 252 {
 253         if (!nft_chain_binding(chain))
 254                 return 0;
 255
 256         if (nft_chain_binding(ctx->chain))
 257                 return -EOPNOTSUPP;
 258
 259         if (chain->bound)
 260                 return -EBUSY;
 261
 262         if (!nft_use_inc(&chain->use))
 263                 return -EMFILE;
 264
 265         chain->bound = true;
 266         nft_chain_trans_bind(ctx, chain);
 267
 268         return 0;
 269 }
 270
 271 void nf_tables_unbind_chain(const struct nft_ctx *ctx, struct nft_chain *chain)
 272 {
 273         __nft_chain_trans_bind(ctx, chain, false);
 274 }
 275
 276 static int nft_netdev_register_hooks(struct net *net,
 277                                      struct list_head *hook_list)
 278 {
 279         struct nft_hook *hook;
 280         int err, j;
 281
 282         j = 0;
 283         list_for_each_entry(hook, hook_list, list) {
 284                 err = nf_register_net_hook(net, &hook->ops);
 285                 if (err < 0)
 286                         goto err_register;
 287
 288                 j++;
 289         }
 290         return 0;
 291
 292 err_register:
 293         list_for_each_entry(hook, hook_list, list) {
 294                 if (j-- <= 0)
 295                         break;
 296
 297                 nf_unregister_net_hook(net, &hook->ops);
 298         }
 299         return err;
 300 }
 301
 302 static void nft_netdev_unregister_hooks(struct net *net,
 303                                         struct list_head *hook_list,
 304                                         bool release_netdev)
 305 {
 306         struct nft_hook *hook, *next;
 307
 308         list_for_each_entry_safe(hook, next, hook_list, list) {
 309                 nf_unregister_net_hook(net, &hook->ops);
 310                 if (release_netdev) {
 311                         list_del(&hook->list);
 312                         kfree_rcu(hook, rcu);
 313                 }
 314         }
 315 }
 316
 317 static int nf_tables_register_hook(struct net *net,
 318                                    const struct nft_table *table,
 319                                    struct nft_chain *chain)
 320 {
 321         struct nft_base_chain *basechain;
 322         const struct nf_hook_ops *ops;
 323
 324         if (table->flags & NFT_TABLE_F_DORMANT ||
 325             !nft_is_base_chain(chain))
 326                 return 0;
 327
 328         basechain = nft_base_chain(chain);
 329         ops = &basechain->ops;
 330
 331         if (basechain->type->ops_register)
 332                 return basechain->type->ops_register(net, ops);
 333
 334         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 335                 return nft_netdev_register_hooks(net, &basechain->hook_list);
 336
 337         return nf_register_net_hook(net, &basechain->ops);
 338 }
 339
 340 static void __nf_tables_unregister_hook(struct net *net,
 341                                         const struct nft_table *table,
 342                                         struct nft_chain *chain,
 343                                         bool release_netdev)
 344 {
 345         struct nft_base_chain *basechain;
 346         const struct nf_hook_ops *ops;
 347
 348         if (table->flags & NFT_TABLE_F_DORMANT ||
 349             !nft_is_base_chain(chain))
 350                 return;
 351         basechain = nft_base_chain(chain);
 352         ops = &basechain->ops;
 353
 354         if (basechain->type->ops_unregister)
 355                 return basechain->type->ops_unregister(net, ops);
 356
 357         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 358                 nft_netdev_unregister_hooks(net, &basechain->hook_list,
 359                                             release_netdev);
 360         else
 361                 nf_unregister_net_hook(net, &basechain->ops);
 362 }
 363
 364 static void nf_tables_unregister_hook(struct net *net,
 365                                       const struct nft_table *table,
 366                                       struct nft_chain *chain)
 367 {
 368         return __nf_tables_unregister_hook(net, table, chain, false);
 369 }
 370
 371 static void nft_trans_commit_list_add_tail(struct net *net, struct nft_trans *trans)
 372 {
 373         struct nftables_pernet *nft_net = nft_pernet(net);
 374
 375         switch (trans->msg_type) {
 376         case NFT_MSG_NEWSET:
 377                 if (!nft_trans_set_update(trans) &&
 378                     nft_set_is_anonymous(nft_trans_set(trans)))
 379                         list_add_tail(&trans->binding_list, &nft_net->binding_list);
 380                 break;
 381         case NFT_MSG_NEWCHAIN:
 382                 if (!nft_trans_chain_update(trans) &&
 383                     nft_chain_binding(nft_trans_chain(trans)))
 384                         list_add_tail(&trans->binding_list, &nft_net->binding_list);
 385                 break;
 386         }
 387
 388         list_add_tail(&trans->list, &nft_net->commit_list);
 389 }
 390
 391 static int nft_trans_table_add(struct nft_ctx *ctx, int msg_type)
 392 {
 393         struct nft_trans *trans;
 394
 395         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_table));
 396         if (trans == NULL)
 397                 return -ENOMEM;
 398
 399         if (msg_type == NFT_MSG_NEWTABLE)
 400                 nft_activate_next(ctx->net, ctx->table);
 401
 402         nft_trans_commit_list_add_tail(ctx->net, trans);
 403         return 0;
 404 }
 405
 406 static int nft_deltable(struct nft_ctx *ctx)
 407 {
 408         int err;
 409
 410         err = nft_trans_table_add(ctx, NFT_MSG_DELTABLE);
 411         if (err < 0)
 412                 return err;
 413
 414         nft_deactivate_next(ctx->net, ctx->table);
 415         return err;
 416 }
 417
 418 static struct nft_trans *nft_trans_chain_add(struct nft_ctx *ctx, int msg_type)
 419 {
 420         struct nft_trans *trans;
 421
 422         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_chain));
 423         if (trans == NULL)
 424                 return ERR_PTR(-ENOMEM);
 425
 426         if (msg_type == NFT_MSG_NEWCHAIN) {
 427                 nft_activate_next(ctx->net, ctx->chain);
 428
 429                 if (ctx->nla[NFTA_CHAIN_ID]) {
 430                         nft_trans_chain_id(trans) =
 431                                 ntohl(nla_get_be32(ctx->nla[NFTA_CHAIN_ID]));
 432                 }
 433         }
 434         nft_trans_chain(trans) = ctx->chain;
 435         nft_trans_commit_list_add_tail(ctx->net, trans);
 436
 437         return trans;
 438 }
 439
 440 static int nft_delchain(struct nft_ctx *ctx)
 441 {
 442         struct nft_trans *trans;
 443
 444         trans = nft_trans_chain_add(ctx, NFT_MSG_DELCHAIN);
 445         if (IS_ERR(trans))
 446                 return PTR_ERR(trans);
 447
 448         nft_use_dec(&ctx->table->use);
 449         nft_deactivate_next(ctx->net, ctx->chain);
 450
 451         return 0;
 452 }
 453
 454 void nft_rule_expr_activate(const struct nft_ctx *ctx, struct nft_rule *rule)
 455 {
 456         struct nft_expr *expr;
 457
 458         expr = nft_expr_first(rule);
 459         while (nft_expr_more(rule, expr)) {
 460                 if (expr->ops->activate)
 461                         expr->ops->activate(ctx, expr);
 462
 463                 expr = nft_expr_next(expr);
 464         }
 465 }
 466
 467 void nft_rule_expr_deactivate(const struct nft_ctx *ctx, struct nft_rule *rule,
 468                               enum nft_trans_phase phase)
 469 {
 470         struct nft_expr *expr;
 471
 472         expr = nft_expr_first(rule);
 473         while (nft_expr_more(rule, expr)) {
 474                 if (expr->ops->deactivate)
 475                         expr->ops->deactivate(ctx, expr, phase);
 476
 477                 expr = nft_expr_next(expr);
 478         }
 479 }
 480
 481 static int
 482 nf_tables_delrule_deactivate(struct nft_ctx *ctx, struct nft_rule *rule)
 483 {
 484         /* You cannot delete the same rule twice */
 485         if (nft_is_active_next(ctx->net, rule)) {
 486                 nft_deactivate_next(ctx->net, rule);
 487                 nft_use_dec(&ctx->chain->use);
 488                 return 0;
 489         }
 490         return -ENOENT;
 491 }
 492
 493 static struct nft_trans *nft_trans_rule_add(struct nft_ctx *ctx, int msg_type,
 494                                             struct nft_rule *rule)
 495 {
 496         struct nft_trans *trans;
 497
 498         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_rule));
 499         if (trans == NULL)
 500                 return NULL;
 501
 502         if (msg_type == NFT_MSG_NEWRULE && ctx->nla[NFTA_RULE_ID] != NULL) {
 503                 nft_trans_rule_id(trans) =
 504                         ntohl(nla_get_be32(ctx->nla[NFTA_RULE_ID]));
 505         }
 506         nft_trans_rule(trans) = rule;
 507         nft_trans_commit_list_add_tail(ctx->net, trans);
 508
 509         return trans;
 510 }
 511
 512 static int nft_delrule(struct nft_ctx *ctx, struct nft_rule *rule)
 513 {
 514         struct nft_flow_rule *flow;
 515         struct nft_trans *trans;
 516         int err;
 517
 518         trans = nft_trans_rule_add(ctx, NFT_MSG_DELRULE, rule);
 519         if (trans == NULL)
 520                 return -ENOMEM;
 521
 522         if (ctx->chain->flags & NFT_CHAIN_HW_OFFLOAD) {
 523                 flow = nft_flow_rule_create(ctx->net, rule);
 524                 if (IS_ERR(flow)) {
 525                         nft_trans_destroy(trans);
 526                         return PTR_ERR(flow);
 527                 }
 528
 529                 nft_trans_flow_rule(trans) = flow;
 530         }
 531
 532         err = nf_tables_delrule_deactivate(ctx, rule);
 533         if (err < 0) {
 534                 nft_trans_destroy(trans);
 535                 return err;
 536         }
 537         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_PREPARE);
 538
 539         return 0;
 540 }
 541
 542 static int nft_delrule_by_chain(struct nft_ctx *ctx)
 543 {
 544         struct nft_rule *rule;
 545         int err;
 546
 547         list_for_each_entry(rule, &ctx->chain->rules, list) {
 548                 if (!nft_is_active_next(ctx->net, rule))
 549                         continue;
 550
 551                 err = nft_delrule(ctx, rule);
 552                 if (err < 0)
 553                         return err;
 554         }
 555         return 0;
 556 }
 557
 558 static int __nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
 559                                struct nft_set *set,
 560                                const struct nft_set_desc *desc)
 561 {
 562         struct nft_trans *trans;
 563
 564         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_set));
 565         if (trans == NULL)
 566                 return -ENOMEM;
 567
 568         if (msg_type == NFT_MSG_NEWSET && ctx->nla[NFTA_SET_ID] && !desc) {
 569                 nft_trans_set_id(trans) =
 570                         ntohl(nla_get_be32(ctx->nla[NFTA_SET_ID]));
 571                 nft_activate_next(ctx->net, set);
 572         }
 573         nft_trans_set(trans) = set;
 574         if (desc) {
 575                 nft_trans_set_update(trans) = true;
 576                 nft_trans_set_gc_int(trans) = desc->gc_int;
 577                 nft_trans_set_timeout(trans) = desc->timeout;
 578                 nft_trans_set_size(trans) = desc->size;
 579         }
 580         nft_trans_commit_list_add_tail(ctx->net, trans);
 581
 582         return 0;
 583 }
 584
 585 static int nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
 586                              struct nft_set *set)
 587 {
 588         return __nft_trans_set_add(ctx, msg_type, set, NULL);
 589 }
 590
 591 static int nft_mapelem_deactivate(const struct nft_ctx *ctx,
 592                                   struct nft_set *set,
 593                                   const struct nft_set_iter *iter,
 594                                   struct nft_set_elem *elem)
 595 {
 596         nft_setelem_data_deactivate(ctx->net, set, elem);
 597
 598         return 0;
 599 }
 600
 601 struct nft_set_elem_catchall {
 602         struct list_head        list;
 603         struct rcu_head         rcu;
 604         void                    *elem;
 605 };
 606
 607 static void nft_map_catchall_deactivate(const struct nft_ctx *ctx,
 608                                         struct nft_set *set)
 609 {
 610         u8 genmask = nft_genmask_next(ctx->net);
 611         struct nft_set_elem_catchall *catchall;
 612         struct nft_set_elem elem;
 613         struct nft_set_ext *ext;
 614
 615         list_for_each_entry(catchall, &set->catchall_list, list) {
 616                 ext = nft_set_elem_ext(set, catchall->elem);
 617                 if (!nft_set_elem_active(ext, genmask))
 618                         continue;
 619
 620                 elem.priv = catchall->elem;
 621                 nft_setelem_data_deactivate(ctx->net, set, &elem);
 622                 break;
 623         }
 624 }
 625
 626 static void nft_map_deactivate(const struct nft_ctx *ctx, struct nft_set *set)
 627 {
 628         struct nft_set_iter iter = {
 629                 .genmask        = nft_genmask_next(ctx->net),
 630                 .fn             = nft_mapelem_deactivate,
 631         };
 632
 633         set->ops->walk(ctx, set, &iter);
 634         WARN_ON_ONCE(iter.err);
 635
 636         nft_map_catchall_deactivate(ctx, set);
 637 }
 638
 639 static int nft_delset(const struct nft_ctx *ctx, struct nft_set *set)
 640 {
 641         int err;
 642
 643         err = nft_trans_set_add(ctx, NFT_MSG_DELSET, set);
 644         if (err < 0)
 645                 return err;
 646
 647         if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
 648                 nft_map_deactivate(ctx, set);
 649
 650         nft_deactivate_next(ctx->net, set);
 651         nft_use_dec(&ctx->table->use);
 652
 653         return err;
 654 }
 655
 656 static int nft_trans_obj_add(struct nft_ctx *ctx, int msg_type,
 657                              struct nft_object *obj)
 658 {
 659         struct nft_trans *trans;
 660
 661         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_obj));
 662         if (trans == NULL)
 663                 return -ENOMEM;
 664
 665         if (msg_type == NFT_MSG_NEWOBJ)
 666                 nft_activate_next(ctx->net, obj);
 667
 668         nft_trans_obj(trans) = obj;
 669         nft_trans_commit_list_add_tail(ctx->net, trans);
 670
 671         return 0;
 672 }
 673
 674 static int nft_delobj(struct nft_ctx *ctx, struct nft_object *obj)
 675 {
 676         int err;
 677
 678         err = nft_trans_obj_add(ctx, NFT_MSG_DELOBJ, obj);
 679         if (err < 0)
 680                 return err;
 681
 682         nft_deactivate_next(ctx->net, obj);
 683         nft_use_dec(&ctx->table->use);
 684
 685         return err;
 686 }
 687
 688 static int nft_trans_flowtable_add(struct nft_ctx *ctx, int msg_type,
 689                                    struct nft_flowtable *flowtable)
 690 {
 691         struct nft_trans *trans;
 692
 693         trans = nft_trans_alloc(ctx, msg_type,
 694                                 sizeof(struct nft_trans_flowtable));
 695         if (trans == NULL)
 696                 return -ENOMEM;
 697
 698         if (msg_type == NFT_MSG_NEWFLOWTABLE)
 699                 nft_activate_next(ctx->net, flowtable);
 700
 701         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
 702         nft_trans_flowtable(trans) = flowtable;
 703         nft_trans_commit_list_add_tail(ctx->net, trans);
 704
 705         return 0;
 706 }
 707
 708 static int nft_delflowtable(struct nft_ctx *ctx,
 709                             struct nft_flowtable *flowtable)
 710 {
 711         int err;
 712
 713         err = nft_trans_flowtable_add(ctx, NFT_MSG_DELFLOWTABLE, flowtable);
 714         if (err < 0)
 715                 return err;
 716
 717         nft_deactivate_next(ctx->net, flowtable);
 718         nft_use_dec(&ctx->table->use);
 719
 720         return err;
 721 }
 722
 723 static void __nft_reg_track_clobber(struct nft_regs_track *track, u8 dreg)
 724 {
 725         int i;
 726
 727         for (i = track->regs[dreg].num_reg; i > 0; i--)
 728                 __nft_reg_track_cancel(track, dreg - i);
 729 }
 730
 731 static void __nft_reg_track_update(struct nft_regs_track *track,
 732                                    const struct nft_expr *expr,
 733                                    u8 dreg, u8 num_reg)
 734 {
 735         track->regs[dreg].selector = expr;
 736         track->regs[dreg].bitwise = NULL;
 737         track->regs[dreg].num_reg = num_reg;
 738 }
 739
 740 void nft_reg_track_update(struct nft_regs_track *track,
 741                           const struct nft_expr *expr, u8 dreg, u8 len)
 742 {
 743         unsigned int regcount;
 744         int i;
 745
 746         __nft_reg_track_clobber(track, dreg);
 747
 748         regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
 749         for (i = 0; i < regcount; i++, dreg++)
 750                 __nft_reg_track_update(track, expr, dreg, i);
 751 }
 752 EXPORT_SYMBOL_GPL(nft_reg_track_update);
 753
 754 void nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg, u8 len)
 755 {
 756         unsigned int regcount;
 757         int i;
 758
 759         __nft_reg_track_clobber(track, dreg);
 760
 761         regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
 762         for (i = 0; i < regcount; i++, dreg++)
 763                 __nft_reg_track_cancel(track, dreg);
 764 }
 765 EXPORT_SYMBOL_GPL(nft_reg_track_cancel);
 766
 767 void __nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg)
 768 {
 769         track->regs[dreg].selector = NULL;
 770         track->regs[dreg].bitwise = NULL;
 771         track->regs[dreg].num_reg = 0;
 772 }
 773 EXPORT_SYMBOL_GPL(__nft_reg_track_cancel);
 774
 775 /*
 776  * Tables
 777  */
 778
 779 static struct nft_table *nft_table_lookup(const struct net *net,
 780                                           const struct nlattr *nla,
 781                                           u8 family, u8 genmask, u32 nlpid)
 782 {
 783         struct nftables_pernet *nft_net;
 784         struct nft_table *table;
 785
 786         if (nla == NULL)
 787                 return ERR_PTR(-EINVAL);
 788
 789         nft_net = nft_pernet(net);
 790         list_for_each_entry_rcu(table, &nft_net->tables, list,
 791                                 lockdep_is_held(&nft_net->commit_mutex)) {
 792                 if (!nla_strcmp(nla, table->name) &&
 793                     table->family == family &&
 794                     nft_active_genmask(table, genmask)) {
 795                         if (nft_table_has_owner(table) &&
 796                             nlpid && table->nlpid != nlpid)
 797                                 return ERR_PTR(-EPERM);
 798
 799                         return table;
 800                 }
 801         }
 802
 803         return ERR_PTR(-ENOENT);
 804 }
 805
 806 static struct nft_table *nft_table_lookup_byhandle(const struct net *net,
 807                                                    const struct nlattr *nla,
 808                                                    u8 genmask, u32 nlpid)
 809 {
 810         struct nftables_pernet *nft_net;
 811         struct nft_table *table;
 812
 813         nft_net = nft_pernet(net);
 814         list_for_each_entry(table, &nft_net->tables, list) {
 815                 if (be64_to_cpu(nla_get_be64(nla)) == table->handle &&
 816                     nft_active_genmask(table, genmask)) {
 817                         if (nft_table_has_owner(table) &&
 818                             nlpid && table->nlpid != nlpid)
 819                                 return ERR_PTR(-EPERM);
 820
 821                         return table;
 822                 }
 823         }
 824
 825         return ERR_PTR(-ENOENT);
 826 }
 827
 828 static inline u64 nf_tables_alloc_handle(struct nft_table *table)
 829 {
 830         return ++table->hgenerator;
 831 }
 832
 833 static const struct nft_chain_type *chain_type[NFPROTO_NUMPROTO][NFT_CHAIN_T_MAX];
 834
 835 static const struct nft_chain_type *
 836 __nft_chain_type_get(u8 family, enum nft_chain_types type)
 837 {
 838         if (family >= NFPROTO_NUMPROTO ||
 839             type >= NFT_CHAIN_T_MAX)
 840                 return NULL;
 841
 842         return chain_type[family][type];
 843 }
 844
 845 static const struct nft_chain_type *
 846 __nf_tables_chain_type_lookup(const struct nlattr *nla, u8 family)
 847 {
 848         const struct nft_chain_type *type;
 849         int i;
 850
 851         for (i = 0; i < NFT_CHAIN_T_MAX; i++) {
 852                 type = __nft_chain_type_get(family, i);
 853                 if (!type)
 854                         continue;
 855                 if (!nla_strcmp(nla, type->name))
 856                         return type;
 857         }
 858         return NULL;
 859 }
 860
 861 struct nft_module_request {
 862         struct list_head        list;
 863         char                    module[MODULE_NAME_LEN];
 864         bool                    done;
 865 };
 866
 867 #ifdef CONFIG_MODULES
 868 __printf(2, 3) int nft_request_module(struct net *net, const char *fmt,
 869                                       ...)
 870 {
 871         char module_name[MODULE_NAME_LEN];
 872         struct nftables_pernet *nft_net;
 873         struct nft_module_request *req;
 874         va_list args;
 875         int ret;
 876
 877         va_start(args, fmt);
 878         ret = vsnprintf(module_name, MODULE_NAME_LEN, fmt, args);
 879         va_end(args);
 880         if (ret >= MODULE_NAME_LEN)
 881                 return 0;
 882
 883         nft_net = nft_pernet(net);
 884         list_for_each_entry(req, &nft_net->module_list, list) {
 885                 if (!strcmp(req->module, module_name)) {
 886                         if (req->done)
 887                                 return 0;
 888
 889                         /* A request to load this module already exists. */
 890                         return -EAGAIN;
 891                 }
 892         }
 893
 894         req = kmalloc(sizeof(*req), GFP_KERNEL);
 895         if (!req)
 896                 return -ENOMEM;
 897
 898         req->done = false;
 899         strscpy(req->module, module_name, MODULE_NAME_LEN);
 900         list_add_tail(&req->list, &nft_net->module_list);
 901
 902         return -EAGAIN;
 903 }
 904 EXPORT_SYMBOL_GPL(nft_request_module);
 905 #endif
 906
 907 static void lockdep_nfnl_nft_mutex_not_held(void)
 908 {
 909 #ifdef CONFIG_PROVE_LOCKING
 910         if (debug_locks)
 911                 WARN_ON_ONCE(lockdep_nfnl_is_held(NFNL_SUBSYS_NFTABLES));
 912 #endif
 913 }
 914
 915 static const struct nft_chain_type *
 916 nf_tables_chain_type_lookup(struct net *net, const struct nlattr *nla,
 917                             u8 family, bool autoload)
 918 {
 919         const struct nft_chain_type *type;
 920
 921         type = __nf_tables_chain_type_lookup(nla, family);
 922         if (type != NULL)
 923                 return type;
 924
 925         lockdep_nfnl_nft_mutex_not_held();
 926 #ifdef CONFIG_MODULES
 927         if (autoload) {
 928                 if (nft_request_module(net, "nft-chain-%u-%.*s", family,
 929                                        nla_len(nla),
 930                                        (const char *)nla_data(nla)) == -EAGAIN)
 931                         return ERR_PTR(-EAGAIN);
 932         }
 933 #endif
 934         return ERR_PTR(-ENOENT);
 935 }
 936
 937 static __be16 nft_base_seq(const struct net *net)
 938 {
 939         struct nftables_pernet *nft_net = nft_pernet(net);
 940
 941         return htons(nft_net->base_seq & 0xffff);
 942 }
 943
 944 static const struct nla_policy nft_table_policy[NFTA_TABLE_MAX + 1] = {
 945         [NFTA_TABLE_NAME]       = { .type = NLA_STRING,
 946                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
 947         [NFTA_TABLE_FLAGS]      = { .type = NLA_U32 },
 948         [NFTA_TABLE_HANDLE]     = { .type = NLA_U64 },
 949         [NFTA_TABLE_USERDATA]   = { .type = NLA_BINARY,
 950                                     .len = NFT_USERDATA_MAXLEN }
 951 };
 952
 953 static int nf_tables_fill_table_info(struct sk_buff *skb, struct net *net,
 954                                      u32 portid, u32 seq, int event, u32 flags,
 955                                      int family, const struct nft_table *table)
 956 {
 957         struct nlmsghdr *nlh;
 958
 959         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
 960         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
 961                            NFNETLINK_V0, nft_base_seq(net));
 962         if (!nlh)
 963                 goto nla_put_failure;
 964
 965         if (nla_put_string(skb, NFTA_TABLE_NAME, table->name) ||
 966             nla_put_be32(skb, NFTA_TABLE_USE, htonl(table->use)) ||
 967             nla_put_be64(skb, NFTA_TABLE_HANDLE, cpu_to_be64(table->handle),
 968                          NFTA_TABLE_PAD))
 969                 goto nla_put_failure;
 970
 971         if (event == NFT_MSG_DELTABLE) {
 972                 nlmsg_end(skb, nlh);
 973                 return 0;
 974         }
 975
 976         if (nla_put_be32(skb, NFTA_TABLE_FLAGS,
 977                          htonl(table->flags & NFT_TABLE_F_MASK)))
 978                 goto nla_put_failure;
 979
 980         if (nft_table_has_owner(table) &&
 981             nla_put_be32(skb, NFTA_TABLE_OWNER, htonl(table->nlpid)))
 982                 goto nla_put_failure;
 983
 984         if (table->udata) {
 985                 if (nla_put(skb, NFTA_TABLE_USERDATA, table->udlen, table->udata))
 986                         goto nla_put_failure;
 987         }
 988
 989         nlmsg_end(skb, nlh);
 990         return 0;
 991
 992 nla_put_failure:
 993         nlmsg_trim(skb, nlh);
 994         return -1;
 995 }
 996
 997 struct nftnl_skb_parms {
 998         bool report;
 999 };
1000 #define NFT_CB(skb)     (*(struct nftnl_skb_parms*)&((skb)->cb))
1001
1002 static void nft_notify_enqueue(struct sk_buff *skb, bool report,
1003                                struct list_head *notify_list)
1004 {
1005         NFT_CB(skb).report = report;
1006         list_add_tail(&skb->list, notify_list);
1007 }
1008
1009 static void nf_tables_table_notify(const struct nft_ctx *ctx, int event)
1010 {
1011         struct nftables_pernet *nft_net;
1012         struct sk_buff *skb;
1013         u16 flags = 0;
1014         int err;
1015
1016         if (!ctx->report &&
1017             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
1018                 return;
1019
1020         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
1021         if (skb == NULL)
1022                 goto err;
1023
1024         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
1025                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
1026
1027         err = nf_tables_fill_table_info(skb, ctx->net, ctx->portid, ctx->seq,
1028                                         event, flags, ctx->family, ctx->table);
1029         if (err < 0) {
1030                 kfree_skb(skb);
1031                 goto err;
1032         }
1033
1034         nft_net = nft_pernet(ctx->net);
1035         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
1036         return;
1037 err:
1038         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
1039 }
1040
1041 static int nf_tables_dump_tables(struct sk_buff *skb,
1042                                  struct netlink_callback *cb)
1043 {
1044         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
1045         struct nftables_pernet *nft_net;
1046         const struct nft_table *table;
1047         unsigned int idx = 0, s_idx = cb->args[0];
1048         struct net *net = sock_net(skb->sk);
1049         int family = nfmsg->nfgen_family;
1050
1051         rcu_read_lock();
1052         nft_net = nft_pernet(net);
1053         cb->seq = READ_ONCE(nft_net->base_seq);
1054
1055         list_for_each_entry_rcu(table, &nft_net->tables, list) {
1056                 if (family != NFPROTO_UNSPEC && family != table->family)
1057                         continue;
1058
1059                 if (idx < s_idx)
1060                         goto cont;
1061                 if (idx > s_idx)
1062                         memset(&cb->args[1], 0,
1063                                sizeof(cb->args) - sizeof(cb->args[0]));
1064                 if (!nft_is_active(net, table))
1065                         continue;
1066                 if (nf_tables_fill_table_info(skb, net,
1067                                               NETLINK_CB(cb->skb).portid,
1068                                               cb->nlh->nlmsg_seq,
1069                                               NFT_MSG_NEWTABLE, NLM_F_MULTI,
1070                                               table->family, table) < 0)
1071                         goto done;
1072
1073                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
1074 cont:
1075                 idx++;
1076         }
1077 done:
1078         rcu_read_unlock();
1079         cb->args[0] = idx;
1080         return skb->len;
1081 }
1082
1083 static int nft_netlink_dump_start_rcu(struct sock *nlsk, struct sk_buff *skb,
1084                                       const struct nlmsghdr *nlh,
1085                                       struct netlink_dump_control *c)
1086 {
1087         int err;
1088
1089         if (!try_module_get(THIS_MODULE))
1090                 return -EINVAL;
1091
1092         rcu_read_unlock();
1093         err = netlink_dump_start(nlsk, skb, nlh, c);
1094         rcu_read_lock();
1095         module_put(THIS_MODULE);
1096
1097         return err;
1098 }
1099
1100 /* called with rcu_read_lock held */
1101 static int nf_tables_gettable(struct sk_buff *skb, const struct nfnl_info *info,
1102                               const struct nlattr * const nla[])
1103 {
1104         struct netlink_ext_ack *extack = info->extack;
1105         u8 genmask = nft_genmask_cur(info->net);
1106         u8 family = info->nfmsg->nfgen_family;
1107         const struct nft_table *table;
1108         struct net *net = info->net;
1109         struct sk_buff *skb2;
1110         int err;
1111
1112         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
1113                 struct netlink_dump_control c = {
1114                         .dump = nf_tables_dump_tables,
1115                         .module = THIS_MODULE,
1116                 };
1117
1118                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
1119         }
1120
1121         table = nft_table_lookup(net, nla[NFTA_TABLE_NAME], family, genmask, 0);
1122         if (IS_ERR(table)) {
1123                 NL_SET_BAD_ATTR(extack, nla[NFTA_TABLE_NAME]);
1124                 return PTR_ERR(table);
1125         }
1126
1127         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
1128         if (!skb2)
1129                 return -ENOMEM;
1130
1131         err = nf_tables_fill_table_info(skb2, net, NETLINK_CB(skb).portid,
1132                                         info->nlh->nlmsg_seq, NFT_MSG_NEWTABLE,
1133                                         0, family, table);
1134         if (err < 0)
1135                 goto err_fill_table_info;
1136
1137         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
1138
1139 err_fill_table_info:
1140         kfree_skb(skb2);
1141         return err;
1142 }
1143
1144 static void nft_table_disable(struct net *net, struct nft_table *table, u32 cnt)
1145 {
1146         struct nft_chain *chain;
1147         u32 i = 0;
1148
1149         list_for_each_entry(chain, &table->chains, list) {
1150                 if (!nft_is_active_next(net, chain))
1151                         continue;
1152                 if (!nft_is_base_chain(chain))
1153                         continue;
1154
1155                 if (cnt && i++ == cnt)
1156                         break;
1157
1158                 nf_tables_unregister_hook(net, table, chain);
1159         }
1160 }
1161
1162 static int nf_tables_table_enable(struct net *net, struct nft_table *table)
1163 {
1164         struct nft_chain *chain;
1165         int err, i = 0;
1166
1167         list_for_each_entry(chain, &table->chains, list) {
1168                 if (!nft_is_active_next(net, chain))
1169                         continue;
1170                 if (!nft_is_base_chain(chain))
1171                         continue;
1172
1173                 err = nf_tables_register_hook(net, table, chain);
1174                 if (err < 0)
1175                         goto err_register_hooks;
1176
1177                 i++;
1178         }
1179         return 0;
1180
1181 err_register_hooks:
1182         if (i)
1183                 nft_table_disable(net, table, i);
1184         return err;
1185 }
1186
1187 static void nf_tables_table_disable(struct net *net, struct nft_table *table)
1188 {
1189         table->flags &= ~NFT_TABLE_F_DORMANT;
1190         nft_table_disable(net, table, 0);
1191         table->flags |= NFT_TABLE_F_DORMANT;
1192 }
1193
1194 #define __NFT_TABLE_F_INTERNAL          (NFT_TABLE_F_MASK + 1)
1195 #define __NFT_TABLE_F_WAS_DORMANT       (__NFT_TABLE_F_INTERNAL << 0)
1196 #define __NFT_TABLE_F_WAS_AWAKEN        (__NFT_TABLE_F_INTERNAL << 1)
1197 #define __NFT_TABLE_F_UPDATE            (__NFT_TABLE_F_WAS_DORMANT | \
1198                                          __NFT_TABLE_F_WAS_AWAKEN)
1199
1200 static int nf_tables_updtable(struct nft_ctx *ctx)
1201 {
1202         struct nft_trans *trans;
1203         u32 flags;
1204         int ret;
1205
1206         if (!ctx->nla[NFTA_TABLE_FLAGS])
1207                 return 0;
1208
1209         flags = ntohl(nla_get_be32(ctx->nla[NFTA_TABLE_FLAGS]));
1210         if (flags & ~NFT_TABLE_F_MASK)
1211                 return -EOPNOTSUPP;
1212
1213         if (flags == ctx->table->flags)
1214                 return 0;
1215
1216         if ((nft_table_has_owner(ctx->table) &&
1217              !(flags & NFT_TABLE_F_OWNER)) ||
1218             (!nft_table_has_owner(ctx->table) &&
1219              flags & NFT_TABLE_F_OWNER))
1220                 return -EOPNOTSUPP;
1221
1222         trans = nft_trans_alloc(ctx, NFT_MSG_NEWTABLE,
1223                                 sizeof(struct nft_trans_table));
1224         if (trans == NULL)
1225                 return -ENOMEM;
1226
1227         if ((flags & NFT_TABLE_F_DORMANT) &&
1228             !(ctx->table->flags & NFT_TABLE_F_DORMANT)) {
1229                 ctx->table->flags |= NFT_TABLE_F_DORMANT;
1230                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE))
1231                         ctx->table->flags |= __NFT_TABLE_F_WAS_AWAKEN;
1232         } else if (!(flags & NFT_TABLE_F_DORMANT) &&
1233                    ctx->table->flags & NFT_TABLE_F_DORMANT) {
1234                 ctx->table->flags &= ~NFT_TABLE_F_DORMANT;
1235                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE)) {
1236                         ret = nf_tables_table_enable(ctx->net, ctx->table);
1237                         if (ret < 0)
1238                                 goto err_register_hooks;
1239
1240                         ctx->table->flags |= __NFT_TABLE_F_WAS_DORMANT;
1241                 }
1242         }
1243
1244         nft_trans_table_update(trans) = true;
1245         nft_trans_commit_list_add_tail(ctx->net, trans);
1246
1247         return 0;
1248
1249 err_register_hooks:
1250         nft_trans_destroy(trans);
1251         return ret;
1252 }
1253
1254 static u32 nft_chain_hash(const void *data, u32 len, u32 seed)
1255 {
1256         const char *name = data;
1257
1258         return jhash(name, strlen(name), seed);
1259 }
1260
1261 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed)
1262 {
1263         const struct nft_chain *chain = data;
1264
1265         return nft_chain_hash(chain->name, 0, seed);
1266 }
1267
1268 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *arg,
1269                               const void *ptr)
1270 {
1271         const struct nft_chain *chain = ptr;
1272         const char *name = arg->key;
1273
1274         return strcmp(chain->name, name);
1275 }
1276
1277 static u32 nft_objname_hash(const void *data, u32 len, u32 seed)
1278 {
1279         const struct nft_object_hash_key *k = data;
1280
1281         seed ^= hash_ptr(k->table, 32);
1282
1283         return jhash(k->name, strlen(k->name), seed);
1284 }
1285
1286 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed)
1287 {
1288         const struct nft_object *obj = data;
1289
1290         return nft_objname_hash(&obj->key, 0, seed);
1291 }
1292
1293 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *arg,
1294                                 const void *ptr)
1295 {
1296         const struct nft_object_hash_key *k = arg->key;
1297         const struct nft_object *obj = ptr;
1298
1299         if (obj->key.table != k->table)
1300                 return -1;
1301
1302         return strcmp(obj->key.name, k->name);
1303 }
1304
1305 static bool nft_supported_family(u8 family)
1306 {
1307         return false
1308 #ifdef CONFIG_NF_TABLES_INET
1309                 || family == NFPROTO_INET
1310 #endif
1311 #ifdef CONFIG_NF_TABLES_IPV4
1312                 || family == NFPROTO_IPV4
1313 #endif
1314 #ifdef CONFIG_NF_TABLES_ARP
1315                 || family == NFPROTO_ARP
1316 #endif
1317 #ifdef CONFIG_NF_TABLES_NETDEV
1318                 || family == NFPROTO_NETDEV
1319 #endif
1320 #if IS_ENABLED(CONFIG_NF_TABLES_BRIDGE)
1321                 || family == NFPROTO_BRIDGE
1322 #endif
1323 #ifdef CONFIG_NF_TABLES_IPV6
1324                 || family == NFPROTO_IPV6
1325 #endif
1326                 ;
1327 }
1328
1329 static int nf_tables_newtable(struct sk_buff *skb, const struct nfnl_info *info,
1330                               const struct nlattr * const nla[])
1331 {
1332         struct nftables_pernet *nft_net = nft_pernet(info->net);
1333         struct netlink_ext_ack *extack = info->extack;
1334         u8 genmask = nft_genmask_next(info->net);
1335         u8 family = info->nfmsg->nfgen_family;
1336         struct net *net = info->net;
1337         const struct nlattr *attr;
1338         struct nft_table *table;
1339         struct nft_ctx ctx;
1340         u32 flags = 0;
1341         int err;
1342
1343         if (!nft_supported_family(family))
1344                 return -EOPNOTSUPP;
1345
1346         lockdep_assert_held(&nft_net->commit_mutex);
1347         attr = nla[NFTA_TABLE_NAME];
1348         table = nft_table_lookup(net, attr, family, genmask,
1349                                  NETLINK_CB(skb).portid);
1350         if (IS_ERR(table)) {
1351                 if (PTR_ERR(table) != -ENOENT)
1352                         return PTR_ERR(table);
1353         } else {
1354                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
1355                         NL_SET_BAD_ATTR(extack, attr);
1356                         return -EEXIST;
1357                 }
1358                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
1359                         return -EOPNOTSUPP;
1360
1361                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1362
1363                 return nf_tables_updtable(&ctx);
1364         }
1365
1366         if (nla[NFTA_TABLE_FLAGS]) {
1367                 flags = ntohl(nla_get_be32(nla[NFTA_TABLE_FLAGS]));
1368                 if (flags & ~NFT_TABLE_F_MASK)
1369                         return -EOPNOTSUPP;
1370         }
1371
1372         err = -ENOMEM;
1373         table = kzalloc(sizeof(*table), GFP_KERNEL_ACCOUNT);
1374         if (table == NULL)
1375                 goto err_kzalloc;
1376
1377         table->validate_state = nft_net->validate_state;
1378         table->name = nla_strdup(attr, GFP_KERNEL_ACCOUNT);
1379         if (table->name == NULL)
1380                 goto err_strdup;
1381
1382         if (nla[NFTA_TABLE_USERDATA]) {
1383                 table->udata = nla_memdup(nla[NFTA_TABLE_USERDATA], GFP_KERNEL_ACCOUNT);
1384                 if (table->udata == NULL)
1385                         goto err_table_udata;
1386
1387                 table->udlen = nla_len(nla[NFTA_TABLE_USERDATA]);
1388         }
1389
1390         err = rhltable_init(&table->chains_ht, &nft_chain_ht_params);
1391         if (err)
1392                 goto err_chain_ht;
1393
1394         INIT_LIST_HEAD(&table->chains);
1395         INIT_LIST_HEAD(&table->sets);
1396         INIT_LIST_HEAD(&table->objects);
1397         INIT_LIST_HEAD(&table->flowtables);
1398         table->family = family;
1399         table->flags = flags;
1400         table->handle = ++nft_net->table_handle;
1401         if (table->flags & NFT_TABLE_F_OWNER)
1402                 table->nlpid = NETLINK_CB(skb).portid;
1403
1404         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1405         err = nft_trans_table_add(&ctx, NFT_MSG_NEWTABLE);
1406         if (err < 0)
1407                 goto err_trans;
1408
1409         list_add_tail_rcu(&table->list, &nft_net->tables);
1410         return 0;
1411 err_trans:
1412         rhltable_destroy(&table->chains_ht);
1413 err_chain_ht:
1414         kfree(table->udata);
1415 err_table_udata:
1416         kfree(table->name);
1417 err_strdup:
1418         kfree(table);
1419 err_kzalloc:
1420         return err;
1421 }
1422
1423 static int nft_flush_table(struct nft_ctx *ctx)
1424 {
1425         struct nft_flowtable *flowtable, *nft;
1426         struct nft_chain *chain, *nc;
1427         struct nft_object *obj, *ne;
1428         struct nft_set *set, *ns;
1429         int err;
1430
1431         list_for_each_entry(chain, &ctx->table->chains, list) {
1432                 if (!nft_is_active_next(ctx->net, chain))
1433                         continue;
1434
1435                 if (nft_chain_is_bound(chain))
1436                         continue;
1437
1438                 ctx->chain = chain;
1439
1440                 err = nft_delrule_by_chain(ctx);
1441                 if (err < 0)
1442                         goto out;
1443         }
1444
1445         list_for_each_entry_safe(set, ns, &ctx->table->sets, list) {
1446                 if (!nft_is_active_next(ctx->net, set))
1447                         continue;
1448
1449                 if (nft_set_is_anonymous(set) &&
1450                     !list_empty(&set->bindings))
1451                         continue;
1452
1453                 err = nft_delset(ctx, set);
1454                 if (err < 0)
1455                         goto out;
1456         }
1457
1458         list_for_each_entry_safe(flowtable, nft, &ctx->table->flowtables, list) {
1459                 if (!nft_is_active_next(ctx->net, flowtable))
1460                         continue;
1461
1462                 err = nft_delflowtable(ctx, flowtable);
1463                 if (err < 0)
1464                         goto out;
1465         }
1466
1467         list_for_each_entry_safe(obj, ne, &ctx->table->objects, list) {
1468                 if (!nft_is_active_next(ctx->net, obj))
1469                         continue;
1470
1471                 err = nft_delobj(ctx, obj);
1472                 if (err < 0)
1473                         goto out;
1474         }
1475
1476         list_for_each_entry_safe(chain, nc, &ctx->table->chains, list) {
1477                 if (!nft_is_active_next(ctx->net, chain))
1478                         continue;
1479
1480                 if (nft_chain_is_bound(chain))
1481                         continue;
1482
1483                 ctx->chain = chain;
1484
1485                 err = nft_delchain(ctx);
1486                 if (err < 0)
1487                         goto out;
1488         }
1489
1490         err = nft_deltable(ctx);
1491 out:
1492         return err;
1493 }
1494
1495 static int nft_flush(struct nft_ctx *ctx, int family)
1496 {
1497         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
1498         const struct nlattr * const *nla = ctx->nla;
1499         struct nft_table *table, *nt;
1500         int err = 0;
1501
1502         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
1503                 if (family != AF_UNSPEC && table->family != family)
1504                         continue;
1505
1506                 ctx->family = table->family;
1507
1508                 if (!nft_is_active_next(ctx->net, table))
1509                         continue;
1510
1511                 if (nft_table_has_owner(table) && table->nlpid != ctx->portid)
1512                         continue;
1513
1514                 if (nla[NFTA_TABLE_NAME] &&
1515                     nla_strcmp(nla[NFTA_TABLE_NAME], table->name) != 0)
1516                         continue;
1517
1518                 ctx->table = table;
1519
1520                 err = nft_flush_table(ctx);
1521                 if (err < 0)
1522                         goto out;
1523         }
1524 out:
1525         return err;
1526 }
1527
1528 static int nf_tables_deltable(struct sk_buff *skb, const struct nfnl_info *info,
1529                               const struct nlattr * const nla[])
1530 {
1531         struct netlink_ext_ack *extack = info->extack;
1532         u8 genmask = nft_genmask_next(info->net);
1533         u8 family = info->nfmsg->nfgen_family;
1534         struct net *net = info->net;
1535         const struct nlattr *attr;
1536         struct nft_table *table;
1537         struct nft_ctx ctx;
1538
1539         nft_ctx_init(&ctx, net, skb, info->nlh, 0, NULL, NULL, nla);
1540         if (family == AF_UNSPEC ||
1541             (!nla[NFTA_TABLE_NAME] && !nla[NFTA_TABLE_HANDLE]))
1542                 return nft_flush(&ctx, family);
1543
1544         if (nla[NFTA_TABLE_HANDLE]) {
1545                 attr = nla[NFTA_TABLE_HANDLE];
1546                 table = nft_table_lookup_byhandle(net, attr, genmask,
1547                                                   NETLINK_CB(skb).portid);
1548         } else {
1549                 attr = nla[NFTA_TABLE_NAME];
1550                 table = nft_table_lookup(net, attr, family, genmask,
1551                                          NETLINK_CB(skb).portid);
1552         }
1553
1554         if (IS_ERR(table)) {
1555                 if (PTR_ERR(table) == -ENOENT &&
1556                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYTABLE)
1557                         return 0;
1558
1559                 NL_SET_BAD_ATTR(extack, attr);
1560                 return PTR_ERR(table);
1561         }
1562
1563         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
1564             table->use > 0)
1565                 return -EBUSY;
1566
1567         ctx.family = family;
1568         ctx.table = table;
1569
1570         return nft_flush_table(&ctx);
1571 }
1572
1573 static void nf_tables_table_destroy(struct nft_ctx *ctx)
1574 {
1575         if (WARN_ON(ctx->table->use > 0))
1576                 return;
1577
1578         rhltable_destroy(&ctx->table->chains_ht);
1579         kfree(ctx->table->name);
1580         kfree(ctx->table->udata);
1581         kfree(ctx->table);
1582 }
1583
1584 void nft_register_chain_type(const struct nft_chain_type *ctype)
1585 {
1586         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1587         if (WARN_ON(__nft_chain_type_get(ctype->family, ctype->type))) {
1588                 nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1589                 return;
1590         }
1591         chain_type[ctype->family][ctype->type] = ctype;
1592         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1593 }
1594 EXPORT_SYMBOL_GPL(nft_register_chain_type);
1595
1596 void nft_unregister_chain_type(const struct nft_chain_type *ctype)
1597 {
1598         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1599         chain_type[ctype->family][ctype->type] = NULL;
1600         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1601 }
1602 EXPORT_SYMBOL_GPL(nft_unregister_chain_type);
1603
1604 /*
1605  * Chains
1606  */
1607
1608 static struct nft_chain *
1609 nft_chain_lookup_byhandle(const struct nft_table *table, u64 handle, u8 genmask)
1610 {
1611         struct nft_chain *chain;
1612
1613         list_for_each_entry(chain, &table->chains, list) {
1614                 if (chain->handle == handle &&
1615                     nft_active_genmask(chain, genmask))
1616                         return chain;
1617         }
1618
1619         return ERR_PTR(-ENOENT);
1620 }
1621
1622 static bool lockdep_commit_lock_is_held(const struct net *net)
1623 {
1624 #ifdef CONFIG_PROVE_LOCKING
1625         struct nftables_pernet *nft_net = nft_pernet(net);
1626
1627         return lockdep_is_held(&nft_net->commit_mutex);
1628 #else
1629         return true;
1630 #endif
1631 }
1632
1633 static struct nft_chain *nft_chain_lookup(struct net *net,
1634                                           struct nft_table *table,
1635                                           const struct nlattr *nla, u8 genmask)
1636 {
1637         char search[NFT_CHAIN_MAXNAMELEN + 1];
1638         struct rhlist_head *tmp, *list;
1639         struct nft_chain *chain;
1640
1641         if (nla == NULL)
1642                 return ERR_PTR(-EINVAL);
1643
1644         nla_strscpy(search, nla, sizeof(search));
1645
1646         WARN_ON(!rcu_read_lock_held() &&
1647                 !lockdep_commit_lock_is_held(net));
1648
1649         chain = ERR_PTR(-ENOENT);
1650         rcu_read_lock();
1651         list = rhltable_lookup(&table->chains_ht, search, nft_chain_ht_params);
1652         if (!list)
1653                 goto out_unlock;
1654
1655         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
1656                 if (nft_active_genmask(chain, genmask))
1657                         goto out_unlock;
1658         }
1659         chain = ERR_PTR(-ENOENT);
1660 out_unlock:
1661         rcu_read_unlock();
1662         return chain;
1663 }
1664
1665 static const struct nla_policy nft_chain_policy[NFTA_CHAIN_MAX + 1] = {
1666         [NFTA_CHAIN_TABLE]      = { .type = NLA_STRING,
1667                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
1668         [NFTA_CHAIN_HANDLE]     = { .type = NLA_U64 },
1669         [NFTA_CHAIN_NAME]       = { .type = NLA_STRING,
1670                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
1671         [NFTA_CHAIN_HOOK]       = { .type = NLA_NESTED },
1672         [NFTA_CHAIN_POLICY]     = { .type = NLA_U32 },
1673         [NFTA_CHAIN_TYPE]       = { .type = NLA_STRING,
1674                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
1675         [NFTA_CHAIN_COUNTERS]   = { .type = NLA_NESTED },
1676         [NFTA_CHAIN_FLAGS]      = { .type = NLA_U32 },
1677         [NFTA_CHAIN_ID]         = { .type = NLA_U32 },
1678         [NFTA_CHAIN_USERDATA]   = { .type = NLA_BINARY,
1679                                     .len = NFT_USERDATA_MAXLEN },
1680 };
1681
1682 static const struct nla_policy nft_hook_policy[NFTA_HOOK_MAX + 1] = {
1683         [NFTA_HOOK_HOOKNUM]     = { .type = NLA_U32 },
1684         [NFTA_HOOK_PRIORITY]    = { .type = NLA_U32 },
1685         [NFTA_HOOK_DEV]         = { .type = NLA_STRING,
1686                                     .len = IFNAMSIZ - 1 },
1687 };
1688
1689 static int nft_dump_stats(struct sk_buff *skb, struct nft_stats __percpu *stats)
1690 {
1691         struct nft_stats *cpu_stats, total;
1692         struct nlattr *nest;
1693         unsigned int seq;
1694         u64 pkts, bytes;
1695         int cpu;
1696
1697         if (!stats)
1698                 return 0;
1699
1700         memset(&total, 0, sizeof(total));
1701         for_each_possible_cpu(cpu) {
1702                 cpu_stats = per_cpu_ptr(stats, cpu);
1703                 do {
1704                         seq = u64_stats_fetch_begin(&cpu_stats->syncp);
1705                         pkts = cpu_stats->pkts;
1706                         bytes = cpu_stats->bytes;
1707                 } while (u64_stats_fetch_retry(&cpu_stats->syncp, seq));
1708                 total.pkts += pkts;
1709                 total.bytes += bytes;
1710         }
1711         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_COUNTERS);
1712         if (nest == NULL)
1713                 goto nla_put_failure;
1714
1715         if (nla_put_be64(skb, NFTA_COUNTER_PACKETS, cpu_to_be64(total.pkts),
1716                          NFTA_COUNTER_PAD) ||
1717             nla_put_be64(skb, NFTA_COUNTER_BYTES, cpu_to_be64(total.bytes),
1718                          NFTA_COUNTER_PAD))
1719                 goto nla_put_failure;
1720
1721         nla_nest_end(skb, nest);
1722         return 0;
1723
1724 nla_put_failure:
1725         return -ENOSPC;
1726 }
1727
1728 static int nft_dump_basechain_hook(struct sk_buff *skb, int family,
1729                                    const struct nft_base_chain *basechain,
1730                                    const struct list_head *hook_list)
1731 {
1732         const struct nf_hook_ops *ops = &basechain->ops;
1733         struct nft_hook *hook, *first = NULL;
1734         struct nlattr *nest, *nest_devs;
1735         int n = 0;
1736
1737         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_HOOK);
1738         if (nest == NULL)
1739                 goto nla_put_failure;
1740         if (nla_put_be32(skb, NFTA_HOOK_HOOKNUM, htonl(ops->hooknum)))
1741                 goto nla_put_failure;
1742         if (nla_put_be32(skb, NFTA_HOOK_PRIORITY, htonl(ops->priority)))
1743                 goto nla_put_failure;
1744
1745         if (nft_base_chain_netdev(family, ops->hooknum)) {
1746                 nest_devs = nla_nest_start_noflag(skb, NFTA_HOOK_DEVS);
1747                 if (!nest_devs)
1748                         goto nla_put_failure;
1749
1750                 if (!hook_list)
1751                         hook_list = &basechain->hook_list;
1752
1753                 list_for_each_entry(hook, hook_list, list) {
1754                         if (!first)
1755                                 first = hook;
1756
1757                         if (nla_put_string(skb, NFTA_DEVICE_NAME,
1758                                            hook->ops.dev->name))
1759                                 goto nla_put_failure;
1760                         n++;
1761                 }
1762                 nla_nest_end(skb, nest_devs);
1763
1764                 if (n == 1 &&
1765                     nla_put_string(skb, NFTA_HOOK_DEV, first->ops.dev->name))
1766                         goto nla_put_failure;
1767         }
1768         nla_nest_end(skb, nest);
1769
1770         return 0;
1771 nla_put_failure:
1772         return -1;
1773 }
1774
1775 static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net,
1776                                      u32 portid, u32 seq, int event, u32 flags,
1777                                      int family, const struct nft_table *table,
1778                                      const struct nft_chain *chain,
1779                                      const struct list_head *hook_list)
1780 {
1781         struct nlmsghdr *nlh;
1782
1783         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
1784         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
1785                            NFNETLINK_V0, nft_base_seq(net));
1786         if (!nlh)
1787                 goto nla_put_failure;
1788
1789         if (nla_put_string(skb, NFTA_CHAIN_TABLE, table->name) ||
1790             nla_put_string(skb, NFTA_CHAIN_NAME, chain->name) ||
1791             nla_put_be64(skb, NFTA_CHAIN_HANDLE, cpu_to_be64(chain->handle),
1792                          NFTA_CHAIN_PAD))
1793                 goto nla_put_failure;
1794
1795         if (event == NFT_MSG_DELCHAIN && !hook_list) {
1796                 nlmsg_end(skb, nlh);
1797                 return 0;
1798         }
1799
1800         if (nft_is_base_chain(chain)) {
1801                 const struct nft_base_chain *basechain = nft_base_chain(chain);
1802                 struct nft_stats __percpu *stats;
1803
1804                 if (nft_dump_basechain_hook(skb, family, basechain, hook_list))
1805                         goto nla_put_failure;
1806
1807                 if (nla_put_be32(skb, NFTA_CHAIN_POLICY,
1808                                  htonl(basechain->policy)))
1809                         goto nla_put_failure;
1810
1811                 if (nla_put_string(skb, NFTA_CHAIN_TYPE, basechain->type->name))
1812                         goto nla_put_failure;
1813
1814                 stats = rcu_dereference_check(basechain->stats,
1815                                               lockdep_commit_lock_is_held(net));
1816                 if (nft_dump_stats(skb, stats))
1817                         goto nla_put_failure;
1818         }
1819
1820         if (chain->flags &&
1821             nla_put_be32(skb, NFTA_CHAIN_FLAGS, htonl(chain->flags)))
1822                 goto nla_put_failure;
1823
1824         if (nla_put_be32(skb, NFTA_CHAIN_USE, htonl(chain->use)))
1825                 goto nla_put_failure;
1826
1827         if (chain->udata &&
1828             nla_put(skb, NFTA_CHAIN_USERDATA, chain->udlen, chain->udata))
1829                 goto nla_put_failure;
1830
1831         nlmsg_end(skb, nlh);
1832         return 0;
1833
1834 nla_put_failure:
1835         nlmsg_trim(skb, nlh);
1836         return -1;
1837 }
1838
1839 static void nf_tables_chain_notify(const struct nft_ctx *ctx, int event,
1840                                    const struct list_head *hook_list)
1841 {
1842         struct nftables_pernet *nft_net;
1843         struct sk_buff *skb;
1844         u16 flags = 0;
1845         int err;
1846
1847         if (!ctx->report &&
1848             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
1849                 return;
1850
1851         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
1852         if (skb == NULL)
1853                 goto err;
1854
1855         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
1856                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
1857
1858         err = nf_tables_fill_chain_info(skb, ctx->net, ctx->portid, ctx->seq,
1859                                         event, flags, ctx->family, ctx->table,
1860                                         ctx->chain, hook_list);
1861         if (err < 0) {
1862                 kfree_skb(skb);
1863                 goto err;
1864         }
1865
1866         nft_net = nft_pernet(ctx->net);
1867         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
1868         return;
1869 err:
1870         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
1871 }
1872
1873 static int nf_tables_dump_chains(struct sk_buff *skb,
1874                                  struct netlink_callback *cb)
1875 {
1876         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
1877         unsigned int idx = 0, s_idx = cb->args[0];
1878         struct net *net = sock_net(skb->sk);
1879         int family = nfmsg->nfgen_family;
1880         struct nftables_pernet *nft_net;
1881         const struct nft_table *table;
1882         const struct nft_chain *chain;
1883
1884         rcu_read_lock();
1885         nft_net = nft_pernet(net);
1886         cb->seq = READ_ONCE(nft_net->base_seq);
1887
1888         list_for_each_entry_rcu(table, &nft_net->tables, list) {
1889                 if (family != NFPROTO_UNSPEC && family != table->family)
1890                         continue;
1891
1892                 list_for_each_entry_rcu(chain, &table->chains, list) {
1893                         if (idx < s_idx)
1894                                 goto cont;
1895                         if (idx > s_idx)
1896                                 memset(&cb->args[1], 0,
1897                                        sizeof(cb->args) - sizeof(cb->args[0]));
1898                         if (!nft_is_active(net, chain))
1899                                 continue;
1900                         if (nf_tables_fill_chain_info(skb, net,
1901                                                       NETLINK_CB(cb->skb).portid,
1902                                                       cb->nlh->nlmsg_seq,
1903                                                       NFT_MSG_NEWCHAIN,
1904                                                       NLM_F_MULTI,
1905                                                       table->family, table,
1906                                                       chain, NULL) < 0)
1907                                 goto done;
1908
1909                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
1910 cont:
1911                         idx++;
1912                 }
1913         }
1914 done:
1915         rcu_read_unlock();
1916         cb->args[0] = idx;
1917         return skb->len;
1918 }
1919
1920 /* called with rcu_read_lock held */
1921 static int nf_tables_getchain(struct sk_buff *skb, const struct nfnl_info *info,
1922                               const struct nlattr * const nla[])
1923 {
1924         struct netlink_ext_ack *extack = info->extack;
1925         u8 genmask = nft_genmask_cur(info->net);
1926         u8 family = info->nfmsg->nfgen_family;
1927         const struct nft_chain *chain;
1928         struct net *net = info->net;
1929         struct nft_table *table;
1930         struct sk_buff *skb2;
1931         int err;
1932
1933         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
1934                 struct netlink_dump_control c = {
1935                         .dump = nf_tables_dump_chains,
1936                         .module = THIS_MODULE,
1937                 };
1938
1939                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
1940         }
1941
1942         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask, 0);
1943         if (IS_ERR(table)) {
1944                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
1945                 return PTR_ERR(table);
1946         }
1947
1948         chain = nft_chain_lookup(net, table, nla[NFTA_CHAIN_NAME], genmask);
1949         if (IS_ERR(chain)) {
1950                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
1951                 return PTR_ERR(chain);
1952         }
1953
1954         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
1955         if (!skb2)
1956                 return -ENOMEM;
1957
1958         err = nf_tables_fill_chain_info(skb2, net, NETLINK_CB(skb).portid,
1959                                         info->nlh->nlmsg_seq, NFT_MSG_NEWCHAIN,
1960                                         0, family, table, chain, NULL);
1961         if (err < 0)
1962                 goto err_fill_chain_info;
1963
1964         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
1965
1966 err_fill_chain_info:
1967         kfree_skb(skb2);
1968         return err;
1969 }
1970
1971 static const struct nla_policy nft_counter_policy[NFTA_COUNTER_MAX + 1] = {
1972         [NFTA_COUNTER_PACKETS]  = { .type = NLA_U64 },
1973         [NFTA_COUNTER_BYTES]    = { .type = NLA_U64 },
1974 };
1975
1976 static struct nft_stats __percpu *nft_stats_alloc(const struct nlattr *attr)
1977 {
1978         struct nlattr *tb[NFTA_COUNTER_MAX+1];
1979         struct nft_stats __percpu *newstats;
1980         struct nft_stats *stats;
1981         int err;
1982
1983         err = nla_parse_nested_deprecated(tb, NFTA_COUNTER_MAX, attr,
1984                                           nft_counter_policy, NULL);
1985         if (err < 0)
1986                 return ERR_PTR(err);
1987
1988         if (!tb[NFTA_COUNTER_BYTES] || !tb[NFTA_COUNTER_PACKETS])
1989                 return ERR_PTR(-EINVAL);
1990
1991         newstats = netdev_alloc_pcpu_stats(struct nft_stats);
1992         if (newstats == NULL)
1993                 return ERR_PTR(-ENOMEM);
1994
1995         /* Restore old counters on this cpu, no problem. Per-cpu statistics
1996          * are not exposed to userspace.
1997          */
1998         preempt_disable();
1999         stats = this_cpu_ptr(newstats);
2000         stats->bytes = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_BYTES]));
2001         stats->pkts = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_PACKETS]));
2002         preempt_enable();
2003
2004         return newstats;
2005 }
2006
2007 static void nft_chain_stats_replace(struct nft_trans *trans)
2008 {
2009         struct nft_base_chain *chain = nft_base_chain(trans->ctx.chain);
2010
2011         if (!nft_trans_chain_stats(trans))
2012                 return;
2013
2014         nft_trans_chain_stats(trans) =
2015                 rcu_replace_pointer(chain->stats, nft_trans_chain_stats(trans),
2016                                     lockdep_commit_lock_is_held(trans->ctx.net));
2017
2018         if (!nft_trans_chain_stats(trans))
2019                 static_branch_inc(&nft_counters_enabled);
2020 }
2021
2022 static void nf_tables_chain_free_chain_rules(struct nft_chain *chain)
2023 {
2024         struct nft_rule_blob *g0 = rcu_dereference_raw(chain->blob_gen_0);
2025         struct nft_rule_blob *g1 = rcu_dereference_raw(chain->blob_gen_1);
2026
2027         if (g0 != g1)
2028                 kvfree(g1);
2029         kvfree(g0);
2030
2031         /* should be NULL either via abort or via successful commit */
2032         WARN_ON_ONCE(chain->blob_next);
2033         kvfree(chain->blob_next);
2034 }
2035
2036 void nf_tables_chain_destroy(struct nft_ctx *ctx)
2037 {
2038         struct nft_chain *chain = ctx->chain;
2039         struct nft_hook *hook, *next;
2040
2041         if (WARN_ON(chain->use > 0))
2042                 return;
2043
2044         /* no concurrent access possible anymore */
2045         nf_tables_chain_free_chain_rules(chain);
2046
2047         if (nft_is_base_chain(chain)) {
2048                 struct nft_base_chain *basechain = nft_base_chain(chain);
2049
2050                 if (nft_base_chain_netdev(ctx->family, basechain->ops.hooknum)) {
2051                         list_for_each_entry_safe(hook, next,
2052                                                  &basechain->hook_list, list) {
2053                                 list_del_rcu(&hook->list);
2054                                 kfree_rcu(hook, rcu);
2055                         }
2056                 }
2057                 module_put(basechain->type->owner);
2058                 if (rcu_access_pointer(basechain->stats)) {
2059                         static_branch_dec(&nft_counters_enabled);
2060                         free_percpu(rcu_dereference_raw(basechain->stats));
2061                 }
2062                 kfree(chain->name);
2063                 kfree(chain->udata);
2064                 kfree(basechain);
2065         } else {
2066                 kfree(chain->name);
2067                 kfree(chain->udata);
2068                 kfree(chain);
2069         }
2070 }
2071
2072 static struct nft_hook *nft_netdev_hook_alloc(struct net *net,
2073                                               const struct nlattr *attr)
2074 {
2075         struct net_device *dev;
2076         char ifname[IFNAMSIZ];
2077         struct nft_hook *hook;
2078         int err;
2079
2080         hook = kmalloc(sizeof(struct nft_hook), GFP_KERNEL_ACCOUNT);
2081         if (!hook) {
2082                 err = -ENOMEM;
2083                 goto err_hook_alloc;
2084         }
2085
2086         nla_strscpy(ifname, attr, IFNAMSIZ);
2087         /* nf_tables_netdev_event() is called under rtnl_mutex, this is
2088          * indirectly serializing all the other holders of the commit_mutex with
2089          * the rtnl_mutex.
2090          */
2091         dev = __dev_get_by_name(net, ifname);
2092         if (!dev) {
2093                 err = -ENOENT;
2094                 goto err_hook_dev;
2095         }
2096         hook->ops.dev = dev;
2097
2098         return hook;
2099
2100 err_hook_dev:
2101         kfree(hook);
2102 err_hook_alloc:
2103         return ERR_PTR(err);
2104 }
2105
2106 static struct nft_hook *nft_hook_list_find(struct list_head *hook_list,
2107                                            const struct nft_hook *this)
2108 {
2109         struct nft_hook *hook;
2110
2111         list_for_each_entry(hook, hook_list, list) {
2112                 if (this->ops.dev == hook->ops.dev)
2113                         return hook;
2114         }
2115
2116         return NULL;
2117 }
2118
2119 static int nf_tables_parse_netdev_hooks(struct net *net,
2120                                         const struct nlattr *attr,
2121                                         struct list_head *hook_list,
2122                                         struct netlink_ext_ack *extack)
2123 {
2124         struct nft_hook *hook, *next;
2125         const struct nlattr *tmp;
2126         int rem, n = 0, err;
2127
2128         nla_for_each_nested(tmp, attr, rem) {
2129                 if (nla_type(tmp) != NFTA_DEVICE_NAME) {
2130                         err = -EINVAL;
2131                         goto err_hook;
2132                 }
2133
2134                 hook = nft_netdev_hook_alloc(net, tmp);
2135                 if (IS_ERR(hook)) {
2136                         NL_SET_BAD_ATTR(extack, tmp);
2137                         err = PTR_ERR(hook);
2138                         goto err_hook;
2139                 }
2140                 if (nft_hook_list_find(hook_list, hook)) {
2141                         NL_SET_BAD_ATTR(extack, tmp);
2142                         kfree(hook);
2143                         err = -EEXIST;
2144                         goto err_hook;
2145                 }
2146                 list_add_tail(&hook->list, hook_list);
2147                 n++;
2148
2149                 if (n == NFT_NETDEVICE_MAX) {
2150                         err = -EFBIG;
2151                         goto err_hook;
2152                 }
2153         }
2154
2155         return 0;
2156
2157 err_hook:
2158         list_for_each_entry_safe(hook, next, hook_list, list) {
2159                 list_del(&hook->list);
2160                 kfree(hook);
2161         }
2162         return err;
2163 }
2164
2165 struct nft_chain_hook {
2166         u32                             num;
2167         s32                             priority;
2168         const struct nft_chain_type     *type;
2169         struct list_head                list;
2170 };
2171
2172 static int nft_chain_parse_netdev(struct net *net, struct nlattr *tb[],
2173                                   struct list_head *hook_list,
2174                                   struct netlink_ext_ack *extack, u32 flags)
2175 {
2176         struct nft_hook *hook;
2177         int err;
2178
2179         if (tb[NFTA_HOOK_DEV]) {
2180                 hook = nft_netdev_hook_alloc(net, tb[NFTA_HOOK_DEV]);
2181                 if (IS_ERR(hook)) {
2182                         NL_SET_BAD_ATTR(extack, tb[NFTA_HOOK_DEV]);
2183                         return PTR_ERR(hook);
2184                 }
2185
2186                 list_add_tail(&hook->list, hook_list);
2187         } else if (tb[NFTA_HOOK_DEVS]) {
2188                 err = nf_tables_parse_netdev_hooks(net, tb[NFTA_HOOK_DEVS],
2189                                                    hook_list, extack);
2190                 if (err < 0)
2191                         return err;
2192
2193         }
2194
2195         if (flags & NFT_CHAIN_HW_OFFLOAD &&
2196             list_empty(hook_list))
2197                 return -EINVAL;
2198
2199         return 0;
2200 }
2201
2202 static int nft_chain_parse_hook(struct net *net,
2203                                 struct nft_base_chain *basechain,
2204                                 const struct nlattr * const nla[],
2205                                 struct nft_chain_hook *hook, u8 family,
2206                                 u32 flags, struct netlink_ext_ack *extack)
2207 {
2208         struct nftables_pernet *nft_net = nft_pernet(net);
2209         struct nlattr *ha[NFTA_HOOK_MAX + 1];
2210         const struct nft_chain_type *type;
2211         int err;
2212
2213         lockdep_assert_held(&nft_net->commit_mutex);
2214         lockdep_nfnl_nft_mutex_not_held();
2215
2216         err = nla_parse_nested_deprecated(ha, NFTA_HOOK_MAX,
2217                                           nla[NFTA_CHAIN_HOOK],
2218                                           nft_hook_policy, NULL);
2219         if (err < 0)
2220                 return err;
2221
2222         if (!basechain) {
2223                 if (!ha[NFTA_HOOK_HOOKNUM] ||
2224                     !ha[NFTA_HOOK_PRIORITY]) {
2225                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2226                         return -ENOENT;
2227                 }
2228
2229                 hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
2230                 hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
2231
2232                 type = __nft_chain_type_get(family, NFT_CHAIN_T_DEFAULT);
2233                 if (!type)
2234                         return -EOPNOTSUPP;
2235
2236                 if (nla[NFTA_CHAIN_TYPE]) {
2237                         type = nf_tables_chain_type_lookup(net, nla[NFTA_CHAIN_TYPE],
2238                                                            family, true);
2239                         if (IS_ERR(type)) {
2240                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2241                                 return PTR_ERR(type);
2242                         }
2243                 }
2244                 if (hook->num >= NFT_MAX_HOOKS || !(type->hook_mask & (1 << hook->num)))
2245                         return -EOPNOTSUPP;
2246
2247                 if (type->type == NFT_CHAIN_T_NAT &&
2248                     hook->priority <= NF_IP_PRI_CONNTRACK)
2249                         return -EOPNOTSUPP;
2250         } else {
2251                 if (ha[NFTA_HOOK_HOOKNUM]) {
2252                         hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
2253                         if (hook->num != basechain->ops.hooknum)
2254                                 return -EOPNOTSUPP;
2255                 }
2256                 if (ha[NFTA_HOOK_PRIORITY]) {
2257                         hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
2258                         if (hook->priority != basechain->ops.priority)
2259                                 return -EOPNOTSUPP;
2260                 }
2261
2262                 type = basechain->type;
2263         }
2264
2265         if (!try_module_get(type->owner)) {
2266                 if (nla[NFTA_CHAIN_TYPE])
2267                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2268                 return -ENOENT;
2269         }
2270
2271         hook->type = type;
2272
2273         INIT_LIST_HEAD(&hook->list);
2274         if (nft_base_chain_netdev(family, hook->num)) {
2275                 err = nft_chain_parse_netdev(net, ha, &hook->list, extack, flags);
2276                 if (err < 0) {
2277                         module_put(type->owner);
2278                         return err;
2279                 }
2280         } else if (ha[NFTA_HOOK_DEV] || ha[NFTA_HOOK_DEVS]) {
2281                 module_put(type->owner);
2282                 return -EOPNOTSUPP;
2283         }
2284
2285         return 0;
2286 }
2287
2288 static void nft_chain_release_hook(struct nft_chain_hook *hook)
2289 {
2290         struct nft_hook *h, *next;
2291
2292         list_for_each_entry_safe(h, next, &hook->list, list) {
2293                 list_del(&h->list);
2294                 kfree(h);
2295         }
2296         module_put(hook->type->owner);
2297 }
2298
2299 static void nft_last_rule(const struct nft_chain *chain, const void *ptr)
2300 {
2301         struct nft_rule_dp_last *lrule;
2302
2303         BUILD_BUG_ON(offsetof(struct nft_rule_dp_last, end) != 0);
2304
2305         lrule = (struct nft_rule_dp_last *)ptr;
2306         lrule->end.is_last = 1;
2307         lrule->chain = chain;
2308         /* blob size does not include the trailer rule */
2309 }
2310
2311 static struct nft_rule_blob *nf_tables_chain_alloc_rules(const struct nft_chain *chain,
2312                                                          unsigned int size)
2313 {
2314         struct nft_rule_blob *blob;
2315
2316         if (size > INT_MAX)
2317                 return NULL;
2318
2319         size += sizeof(struct nft_rule_blob) + sizeof(struct nft_rule_dp_last);
2320
2321         blob = kvmalloc(size, GFP_KERNEL_ACCOUNT);
2322         if (!blob)
2323                 return NULL;
2324
2325         blob->size = 0;
2326         nft_last_rule(chain, blob->data);
2327
2328         return blob;
2329 }
2330
2331 static void nft_basechain_hook_init(struct nf_hook_ops *ops, u8 family,
2332                                     const struct nft_chain_hook *hook,
2333                                     struct nft_chain *chain)
2334 {
2335         ops->pf                 = family;
2336         ops->hooknum            = hook->num;
2337         ops->priority           = hook->priority;
2338         ops->priv               = chain;
2339         ops->hook               = hook->type->hooks[ops->hooknum];
2340         ops->hook_ops_type      = NF_HOOK_OP_NF_TABLES;
2341 }
2342
2343 static int nft_basechain_init(struct nft_base_chain *basechain, u8 family,
2344                               struct nft_chain_hook *hook, u32 flags)
2345 {
2346         struct nft_chain *chain;
2347         struct nft_hook *h;
2348
2349         basechain->type = hook->type;
2350         INIT_LIST_HEAD(&basechain->hook_list);
2351         chain = &basechain->chain;
2352
2353         if (nft_base_chain_netdev(family, hook->num)) {
2354                 list_splice_init(&hook->list, &basechain->hook_list);
2355                 list_for_each_entry(h, &basechain->hook_list, list)
2356                         nft_basechain_hook_init(&h->ops, family, hook, chain);
2357         }
2358         nft_basechain_hook_init(&basechain->ops, family, hook, chain);
2359
2360         chain->flags |= NFT_CHAIN_BASE | flags;
2361         basechain->policy = NF_ACCEPT;
2362         if (chain->flags & NFT_CHAIN_HW_OFFLOAD &&
2363             !nft_chain_offload_support(basechain)) {
2364                 list_splice_init(&basechain->hook_list, &hook->list);
2365                 return -EOPNOTSUPP;
2366         }
2367
2368         flow_block_init(&basechain->flow_block);
2369
2370         return 0;
2371 }
2372
2373 int nft_chain_add(struct nft_table *table, struct nft_chain *chain)
2374 {
2375         int err;
2376
2377         err = rhltable_insert_key(&table->chains_ht, chain->name,
2378                                   &chain->rhlhead, nft_chain_ht_params);
2379         if (err)
2380                 return err;
2381
2382         list_add_tail_rcu(&chain->list, &table->chains);
2383
2384         return 0;
2385 }
2386
2387 static u64 chain_id;
2388
2389 static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 genmask,
2390                               u8 policy, u32 flags,
2391                               struct netlink_ext_ack *extack)
2392 {
2393         const struct nlattr * const *nla = ctx->nla;
2394         struct nft_table *table = ctx->table;
2395         struct nft_base_chain *basechain;
2396         struct net *net = ctx->net;
2397         char name[NFT_NAME_MAXLEN];
2398         struct nft_rule_blob *blob;
2399         struct nft_trans *trans;
2400         struct nft_chain *chain;
2401         int err;
2402
2403         if (nla[NFTA_CHAIN_HOOK]) {
2404                 struct nft_stats __percpu *stats = NULL;
2405                 struct nft_chain_hook hook = {};
2406
2407                 if (flags & NFT_CHAIN_BINDING)
2408                         return -EOPNOTSUPP;
2409
2410                 err = nft_chain_parse_hook(net, NULL, nla, &hook, family, flags,
2411                                            extack);
2412                 if (err < 0)
2413                         return err;
2414
2415                 basechain = kzalloc(sizeof(*basechain), GFP_KERNEL_ACCOUNT);
2416                 if (basechain == NULL) {
2417                         nft_chain_release_hook(&hook);
2418                         return -ENOMEM;
2419                 }
2420                 chain = &basechain->chain;
2421
2422                 if (nla[NFTA_CHAIN_COUNTERS]) {
2423                         stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2424                         if (IS_ERR(stats)) {
2425                                 nft_chain_release_hook(&hook);
2426                                 kfree(basechain);
2427                                 return PTR_ERR(stats);
2428                         }
2429                         rcu_assign_pointer(basechain->stats, stats);
2430                 }
2431
2432                 err = nft_basechain_init(basechain, family, &hook, flags);
2433                 if (err < 0) {
2434                         nft_chain_release_hook(&hook);
2435                         kfree(basechain);
2436                         free_percpu(stats);
2437                         return err;
2438                 }
2439                 if (stats)
2440                         static_branch_inc(&nft_counters_enabled);
2441         } else {
2442                 if (flags & NFT_CHAIN_BASE)
2443                         return -EINVAL;
2444                 if (flags & NFT_CHAIN_HW_OFFLOAD)
2445                         return -EOPNOTSUPP;
2446
2447                 chain = kzalloc(sizeof(*chain), GFP_KERNEL_ACCOUNT);
2448                 if (chain == NULL)
2449                         return -ENOMEM;
2450
2451                 chain->flags = flags;
2452         }
2453         ctx->chain = chain;
2454
2455         INIT_LIST_HEAD(&chain->rules);
2456         chain->handle = nf_tables_alloc_handle(table);
2457         chain->table = table;
2458
2459         if (nla[NFTA_CHAIN_NAME]) {
2460                 chain->name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
2461         } else {
2462                 if (!(flags & NFT_CHAIN_BINDING)) {
2463                         err = -EINVAL;
2464                         goto err_destroy_chain;
2465                 }
2466
2467                 snprintf(name, sizeof(name), "__chain%llu", ++chain_id);
2468                 chain->name = kstrdup(name, GFP_KERNEL_ACCOUNT);
2469         }
2470
2471         if (!chain->name) {
2472                 err = -ENOMEM;
2473                 goto err_destroy_chain;
2474         }
2475
2476         if (nla[NFTA_CHAIN_USERDATA]) {
2477                 chain->udata = nla_memdup(nla[NFTA_CHAIN_USERDATA], GFP_KERNEL_ACCOUNT);
2478                 if (chain->udata == NULL) {
2479                         err = -ENOMEM;
2480                         goto err_destroy_chain;
2481                 }
2482                 chain->udlen = nla_len(nla[NFTA_CHAIN_USERDATA]);
2483         }
2484
2485         blob = nf_tables_chain_alloc_rules(chain, 0);
2486         if (!blob) {
2487                 err = -ENOMEM;
2488                 goto err_destroy_chain;
2489         }
2490
2491         RCU_INIT_POINTER(chain->blob_gen_0, blob);
2492         RCU_INIT_POINTER(chain->blob_gen_1, blob);
2493
2494         err = nf_tables_register_hook(net, table, chain);
2495         if (err < 0)
2496                 goto err_destroy_chain;
2497
2498         if (!nft_use_inc(&table->use)) {
2499                 err = -EMFILE;
2500                 goto err_use;
2501         }
2502
2503         trans = nft_trans_chain_add(ctx, NFT_MSG_NEWCHAIN);
2504         if (IS_ERR(trans)) {
2505                 err = PTR_ERR(trans);
2506                 goto err_unregister_hook;
2507         }
2508
2509         nft_trans_chain_policy(trans) = NFT_CHAIN_POLICY_UNSET;
2510         if (nft_is_base_chain(chain))
2511                 nft_trans_chain_policy(trans) = policy;
2512
2513         err = nft_chain_add(table, chain);
2514         if (err < 0) {
2515                 nft_trans_destroy(trans);
2516                 goto err_unregister_hook;
2517         }
2518
2519         return 0;
2520
2521 err_unregister_hook:
2522         nft_use_dec_restore(&table->use);
2523 err_use:
2524         nf_tables_unregister_hook(net, table, chain);
2525 err_destroy_chain:
2526         nf_tables_chain_destroy(ctx);
2527
2528         return err;
2529 }
2530
2531 static int nf_tables_updchain(struct nft_ctx *ctx, u8 genmask, u8 policy,
2532                               u32 flags, const struct nlattr *attr,
2533                               struct netlink_ext_ack *extack)
2534 {
2535         const struct nlattr * const *nla = ctx->nla;
2536         struct nft_base_chain *basechain = NULL;
2537         struct nft_table *table = ctx->table;
2538         struct nft_chain *chain = ctx->chain;
2539         struct nft_chain_hook hook = {};
2540         struct nft_stats *stats = NULL;
2541         struct nft_hook *h, *next;
2542         struct nf_hook_ops *ops;
2543         struct nft_trans *trans;
2544         bool unregister = false;
2545         int err;
2546
2547         if (chain->flags ^ flags)
2548                 return -EOPNOTSUPP;
2549
2550         INIT_LIST_HEAD(&hook.list);
2551
2552         if (nla[NFTA_CHAIN_HOOK]) {
2553                 if (!nft_is_base_chain(chain)) {
2554                         NL_SET_BAD_ATTR(extack, attr);
2555                         return -EEXIST;
2556                 }
2557
2558                 basechain = nft_base_chain(chain);
2559                 err = nft_chain_parse_hook(ctx->net, basechain, nla, &hook,
2560                                            ctx->family, flags, extack);
2561                 if (err < 0)
2562                         return err;
2563
2564                 if (basechain->type != hook.type) {
2565                         nft_chain_release_hook(&hook);
2566                         NL_SET_BAD_ATTR(extack, attr);
2567                         return -EEXIST;
2568                 }
2569
2570                 if (nft_base_chain_netdev(ctx->family, basechain->ops.hooknum)) {
2571                         list_for_each_entry_safe(h, next, &hook.list, list) {
2572                                 h->ops.pf       = basechain->ops.pf;
2573                                 h->ops.hooknum  = basechain->ops.hooknum;
2574                                 h->ops.priority = basechain->ops.priority;
2575                                 h->ops.priv     = basechain->ops.priv;
2576                                 h->ops.hook     = basechain->ops.hook;
2577
2578                                 if (nft_hook_list_find(&basechain->hook_list, h)) {
2579                                         list_del(&h->list);
2580                                         kfree(h);
2581                                 }
2582                         }
2583                 } else {
2584                         ops = &basechain->ops;
2585                         if (ops->hooknum != hook.num ||
2586                             ops->priority != hook.priority) {
2587                                 nft_chain_release_hook(&hook);
2588                                 NL_SET_BAD_ATTR(extack, attr);
2589                                 return -EEXIST;
2590                         }
2591                 }
2592         }
2593
2594         if (nla[NFTA_CHAIN_HANDLE] &&
2595             nla[NFTA_CHAIN_NAME]) {
2596                 struct nft_chain *chain2;
2597
2598                 chain2 = nft_chain_lookup(ctx->net, table,
2599                                           nla[NFTA_CHAIN_NAME], genmask);
2600                 if (!IS_ERR(chain2)) {
2601                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2602                         err = -EEXIST;
2603                         goto err_hooks;
2604                 }
2605         }
2606
2607         if (nla[NFTA_CHAIN_COUNTERS]) {
2608                 if (!nft_is_base_chain(chain)) {
2609                         err = -EOPNOTSUPP;
2610                         goto err_hooks;
2611                 }
2612
2613                 stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2614                 if (IS_ERR(stats)) {
2615                         err = PTR_ERR(stats);
2616                         goto err_hooks;
2617                 }
2618         }
2619
2620         if (!(table->flags & NFT_TABLE_F_DORMANT) &&
2621             nft_is_base_chain(chain) &&
2622             !list_empty(&hook.list)) {
2623                 basechain = nft_base_chain(chain);
2624                 ops = &basechain->ops;
2625
2626                 if (nft_base_chain_netdev(table->family, basechain->ops.hooknum)) {
2627                         err = nft_netdev_register_hooks(ctx->net, &hook.list);
2628                         if (err < 0)
2629                                 goto err_hooks;
2630                 }
2631         }
2632
2633         unregister = true;
2634         err = -ENOMEM;
2635         trans = nft_trans_alloc(ctx, NFT_MSG_NEWCHAIN,
2636                                 sizeof(struct nft_trans_chain));
2637         if (trans == NULL)
2638                 goto err_trans;
2639
2640         nft_trans_chain_stats(trans) = stats;
2641         nft_trans_chain_update(trans) = true;
2642
2643         if (nla[NFTA_CHAIN_POLICY])
2644                 nft_trans_chain_policy(trans) = policy;
2645         else
2646                 nft_trans_chain_policy(trans) = -1;
2647
2648         if (nla[NFTA_CHAIN_HANDLE] &&
2649             nla[NFTA_CHAIN_NAME]) {
2650                 struct nftables_pernet *nft_net = nft_pernet(ctx->net);
2651                 struct nft_trans *tmp;
2652                 char *name;
2653
2654                 err = -ENOMEM;
2655                 name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
2656                 if (!name)
2657                         goto err_trans;
2658
2659                 err = -EEXIST;
2660                 list_for_each_entry(tmp, &nft_net->commit_list, list) {
2661                         if (tmp->msg_type == NFT_MSG_NEWCHAIN &&
2662                             tmp->ctx.table == table &&
2663                             nft_trans_chain_update(tmp) &&
2664                             nft_trans_chain_name(tmp) &&
2665                             strcmp(name, nft_trans_chain_name(tmp)) == 0) {
2666                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2667                                 kfree(name);
2668                                 goto err_trans;
2669                         }
2670                 }
2671
2672                 nft_trans_chain_name(trans) = name;
2673         }
2674
2675         nft_trans_basechain(trans) = basechain;
2676         INIT_LIST_HEAD(&nft_trans_chain_hooks(trans));
2677         list_splice(&hook.list, &nft_trans_chain_hooks(trans));
2678         if (nla[NFTA_CHAIN_HOOK])
2679                 module_put(hook.type->owner);
2680
2681         nft_trans_commit_list_add_tail(ctx->net, trans);
2682
2683         return 0;
2684
2685 err_trans:
2686         free_percpu(stats);
2687         kfree(trans);
2688 err_hooks:
2689         if (nla[NFTA_CHAIN_HOOK]) {
2690                 list_for_each_entry_safe(h, next, &hook.list, list) {
2691                         if (unregister)
2692                                 nf_unregister_net_hook(ctx->net, &h->ops);
2693                         list_del(&h->list);
2694                         kfree_rcu(h, rcu);
2695                 }
2696                 module_put(hook.type->owner);
2697         }
2698
2699         return err;
2700 }
2701
2702 static struct nft_chain *nft_chain_lookup_byid(const struct net *net,
2703                                                const struct nft_table *table,
2704                                                const struct nlattr *nla, u8 genmask)
2705 {
2706         struct nftables_pernet *nft_net = nft_pernet(net);
2707         u32 id = ntohl(nla_get_be32(nla));
2708         struct nft_trans *trans;
2709
2710         list_for_each_entry(trans, &nft_net->commit_list, list) {
2711                 struct nft_chain *chain = trans->ctx.chain;
2712
2713                 if (trans->msg_type == NFT_MSG_NEWCHAIN &&
2714                     chain->table == table &&
2715                     id == nft_trans_chain_id(trans) &&
2716                     nft_active_genmask(chain, genmask))
2717                         return chain;
2718         }
2719         return ERR_PTR(-ENOENT);
2720 }
2721
2722 static int nf_tables_newchain(struct sk_buff *skb, const struct nfnl_info *info,
2723                               const struct nlattr * const nla[])
2724 {
2725         struct nftables_pernet *nft_net = nft_pernet(info->net);
2726         struct netlink_ext_ack *extack = info->extack;
2727         u8 genmask = nft_genmask_next(info->net);
2728         u8 family = info->nfmsg->nfgen_family;
2729         struct nft_chain *chain = NULL;
2730         struct net *net = info->net;
2731         const struct nlattr *attr;
2732         struct nft_table *table;
2733         u8 policy = NF_ACCEPT;
2734         struct nft_ctx ctx;
2735         u64 handle = 0;
2736         u32 flags = 0;
2737
2738         lockdep_assert_held(&nft_net->commit_mutex);
2739
2740         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
2741                                  NETLINK_CB(skb).portid);
2742         if (IS_ERR(table)) {
2743                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2744                 return PTR_ERR(table);
2745         }
2746
2747         chain = NULL;
2748         attr = nla[NFTA_CHAIN_NAME];
2749
2750         if (nla[NFTA_CHAIN_HANDLE]) {
2751                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_CHAIN_HANDLE]));
2752                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2753                 if (IS_ERR(chain)) {
2754                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_HANDLE]);
2755                         return PTR_ERR(chain);
2756                 }
2757                 attr = nla[NFTA_CHAIN_HANDLE];
2758         } else if (nla[NFTA_CHAIN_NAME]) {
2759                 chain = nft_chain_lookup(net, table, attr, genmask);
2760                 if (IS_ERR(chain)) {
2761                         if (PTR_ERR(chain) != -ENOENT) {
2762                                 NL_SET_BAD_ATTR(extack, attr);
2763                                 return PTR_ERR(chain);
2764                         }
2765                         chain = NULL;
2766                 }
2767         } else if (!nla[NFTA_CHAIN_ID]) {
2768                 return -EINVAL;
2769         }
2770
2771         if (nla[NFTA_CHAIN_POLICY]) {
2772                 if (chain != NULL &&
2773                     !nft_is_base_chain(chain)) {
2774                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2775                         return -EOPNOTSUPP;
2776                 }
2777
2778                 if (chain == NULL &&
2779                     nla[NFTA_CHAIN_HOOK] == NULL) {
2780                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2781                         return -EOPNOTSUPP;
2782                 }
2783
2784                 policy = ntohl(nla_get_be32(nla[NFTA_CHAIN_POLICY]));
2785                 switch (policy) {
2786                 case NF_DROP:
2787                 case NF_ACCEPT:
2788                         break;
2789                 default:
2790                         return -EINVAL;
2791                 }
2792         }
2793
2794         if (nla[NFTA_CHAIN_FLAGS])
2795                 flags = ntohl(nla_get_be32(nla[NFTA_CHAIN_FLAGS]));
2796         else if (chain)
2797                 flags = chain->flags;
2798
2799         if (flags & ~NFT_CHAIN_FLAGS)
2800                 return -EOPNOTSUPP;
2801
2802         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
2803
2804         if (chain != NULL) {
2805                 if (chain->flags & NFT_CHAIN_BINDING)
2806                         return -EINVAL;
2807
2808                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
2809                         NL_SET_BAD_ATTR(extack, attr);
2810                         return -EEXIST;
2811                 }
2812                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
2813                         return -EOPNOTSUPP;
2814
2815                 flags |= chain->flags & NFT_CHAIN_BASE;
2816                 return nf_tables_updchain(&ctx, genmask, policy, flags, attr,
2817                                           extack);
2818         }
2819
2820         return nf_tables_addchain(&ctx, family, genmask, policy, flags, extack);
2821 }
2822
2823 static int nft_delchain_hook(struct nft_ctx *ctx,
2824                              struct nft_base_chain *basechain,
2825                              struct netlink_ext_ack *extack)
2826 {
2827         const struct nft_chain *chain = &basechain->chain;
2828         const struct nlattr * const *nla = ctx->nla;
2829         struct nft_chain_hook chain_hook = {};
2830         struct nft_hook *this, *hook;
2831         LIST_HEAD(chain_del_list);
2832         struct nft_trans *trans;
2833         int err;
2834
2835         err = nft_chain_parse_hook(ctx->net, basechain, nla, &chain_hook,
2836                                    ctx->family, chain->flags, extack);
2837         if (err < 0)
2838                 return err;
2839
2840         list_for_each_entry(this, &chain_hook.list, list) {
2841                 hook = nft_hook_list_find(&basechain->hook_list, this);
2842                 if (!hook) {
2843                         err = -ENOENT;
2844                         goto err_chain_del_hook;
2845                 }
2846                 list_move(&hook->list, &chain_del_list);
2847         }
2848
2849         trans = nft_trans_alloc(ctx, NFT_MSG_DELCHAIN,
2850                                 sizeof(struct nft_trans_chain));
2851         if (!trans) {
2852                 err = -ENOMEM;
2853                 goto err_chain_del_hook;
2854         }
2855
2856         nft_trans_basechain(trans) = basechain;
2857         nft_trans_chain_update(trans) = true;
2858         INIT_LIST_HEAD(&nft_trans_chain_hooks(trans));
2859         list_splice(&chain_del_list, &nft_trans_chain_hooks(trans));
2860         nft_chain_release_hook(&chain_hook);
2861
2862         nft_trans_commit_list_add_tail(ctx->net, trans);
2863
2864         return 0;
2865
2866 err_chain_del_hook:
2867         list_splice(&chain_del_list, &basechain->hook_list);
2868         nft_chain_release_hook(&chain_hook);
2869
2870         return err;
2871 }
2872
2873 static int nf_tables_delchain(struct sk_buff *skb, const struct nfnl_info *info,
2874                               const struct nlattr * const nla[])
2875 {
2876         struct netlink_ext_ack *extack = info->extack;
2877         u8 genmask = nft_genmask_next(info->net);
2878         u8 family = info->nfmsg->nfgen_family;
2879         struct net *net = info->net;
2880         const struct nlattr *attr;
2881         struct nft_table *table;
2882         struct nft_chain *chain;
2883         struct nft_rule *rule;
2884         struct nft_ctx ctx;
2885         u64 handle;
2886         u32 use;
2887         int err;
2888
2889         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
2890                                  NETLINK_CB(skb).portid);
2891         if (IS_ERR(table)) {
2892                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2893                 return PTR_ERR(table);
2894         }
2895
2896         if (nla[NFTA_CHAIN_HANDLE]) {
2897                 attr = nla[NFTA_CHAIN_HANDLE];
2898                 handle = be64_to_cpu(nla_get_be64(attr));
2899                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2900         } else {
2901                 attr = nla[NFTA_CHAIN_NAME];
2902                 chain = nft_chain_lookup(net, table, attr, genmask);
2903         }
2904         if (IS_ERR(chain)) {
2905                 if (PTR_ERR(chain) == -ENOENT &&
2906                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYCHAIN)
2907                         return 0;
2908
2909                 NL_SET_BAD_ATTR(extack, attr);
2910                 return PTR_ERR(chain);
2911         }
2912
2913         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
2914
2915         if (nla[NFTA_CHAIN_HOOK]) {
2916                 if (chain->flags & NFT_CHAIN_HW_OFFLOAD)
2917                         return -EOPNOTSUPP;
2918
2919                 if (nft_is_base_chain(chain)) {
2920                         struct nft_base_chain *basechain = nft_base_chain(chain);
2921
2922                         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
2923                                 return nft_delchain_hook(&ctx, basechain, extack);
2924                 }
2925         }
2926
2927         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
2928             chain->use > 0)
2929                 return -EBUSY;
2930
2931         use = chain->use;
2932         list_for_each_entry(rule, &chain->rules, list) {
2933                 if (!nft_is_active_next(net, rule))
2934                         continue;
2935                 use--;
2936
2937                 err = nft_delrule(&ctx, rule);
2938                 if (err < 0)
2939                         return err;
2940         }
2941
2942         /* There are rules and elements that are still holding references to us,
2943          * we cannot do a recursive removal in this case.
2944          */
2945         if (use > 0) {
2946                 NL_SET_BAD_ATTR(extack, attr);
2947                 return -EBUSY;
2948         }
2949
2950         return nft_delchain(&ctx);
2951 }
2952
2953 /*
2954  * Expressions
2955  */
2956
2957 /**
2958  *      nft_register_expr - register nf_tables expr type
2959  *      @type: expr type
2960  *
2961  *      Registers the expr type for use with nf_tables. Returns zero on
2962  *      success or a negative errno code otherwise.
2963  */
2964 int nft_register_expr(struct nft_expr_type *type)
2965 {
2966         nfnl_lock(NFNL_SUBSYS_NFTABLES);
2967         if (type->family == NFPROTO_UNSPEC)
2968                 list_add_tail_rcu(&type->list, &nf_tables_expressions);
2969         else
2970                 list_add_rcu(&type->list, &nf_tables_expressions);
2971         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
2972         return 0;
2973 }
2974 EXPORT_SYMBOL_GPL(nft_register_expr);
2975
2976 /**
2977  *      nft_unregister_expr - unregister nf_tables expr type
2978  *      @type: expr type
2979  *
2980  *      Unregisters the expr typefor use with nf_tables.
2981  */
2982 void nft_unregister_expr(struct nft_expr_type *type)
2983 {
2984         nfnl_lock(NFNL_SUBSYS_NFTABLES);
2985         list_del_rcu(&type->list);
2986         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
2987 }
2988 EXPORT_SYMBOL_GPL(nft_unregister_expr);
2989
2990 static const struct nft_expr_type *__nft_expr_type_get(u8 family,
2991                                                        struct nlattr *nla)
2992 {
2993         const struct nft_expr_type *type, *candidate = NULL;
2994
2995         list_for_each_entry(type, &nf_tables_expressions, list) {
2996                 if (!nla_strcmp(nla, type->name)) {
2997                         if (!type->family && !candidate)
2998                                 candidate = type;
2999                         else if (type->family == family)
3000                                 candidate = type;
3001                 }
3002         }
3003         return candidate;
3004 }
3005
3006 #ifdef CONFIG_MODULES
3007 static int nft_expr_type_request_module(struct net *net, u8 family,
3008                                         struct nlattr *nla)
3009 {
3010         if (nft_request_module(net, "nft-expr-%u-%.*s", family,
3011                                nla_len(nla), (char *)nla_data(nla)) == -EAGAIN)
3012                 return -EAGAIN;
3013
3014         return 0;
3015 }
3016 #endif
3017
3018 static const struct nft_expr_type *nft_expr_type_get(struct net *net,
3019                                                      u8 family,
3020                                                      struct nlattr *nla)
3021 {
3022         const struct nft_expr_type *type;
3023
3024         if (nla == NULL)
3025                 return ERR_PTR(-EINVAL);
3026
3027         type = __nft_expr_type_get(family, nla);
3028         if (type != NULL && try_module_get(type->owner))
3029                 return type;
3030
3031         lockdep_nfnl_nft_mutex_not_held();
3032 #ifdef CONFIG_MODULES
3033         if (type == NULL) {
3034                 if (nft_expr_type_request_module(net, family, nla) == -EAGAIN)
3035                         return ERR_PTR(-EAGAIN);
3036
3037                 if (nft_request_module(net, "nft-expr-%.*s",
3038                                        nla_len(nla),
3039                                        (char *)nla_data(nla)) == -EAGAIN)
3040                         return ERR_PTR(-EAGAIN);
3041         }
3042 #endif
3043         return ERR_PTR(-ENOENT);
3044 }
3045
3046 static const struct nla_policy nft_expr_policy[NFTA_EXPR_MAX + 1] = {
3047         [NFTA_EXPR_NAME]        = { .type = NLA_STRING,
3048                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
3049         [NFTA_EXPR_DATA]        = { .type = NLA_NESTED },
3050 };
3051
3052 static int nf_tables_fill_expr_info(struct sk_buff *skb,
3053                                     const struct nft_expr *expr, bool reset)
3054 {
3055         if (nla_put_string(skb, NFTA_EXPR_NAME, expr->ops->type->name))
3056                 goto nla_put_failure;
3057
3058         if (expr->ops->dump) {
3059                 struct nlattr *data = nla_nest_start_noflag(skb,
3060                                                             NFTA_EXPR_DATA);
3061                 if (data == NULL)
3062                         goto nla_put_failure;
3063                 if (expr->ops->dump(skb, expr, reset) < 0)
3064                         goto nla_put_failure;
3065                 nla_nest_end(skb, data);
3066         }
3067
3068         return skb->len;
3069
3070 nla_put_failure:
3071         return -1;
3072 };
3073
3074 int nft_expr_dump(struct sk_buff *skb, unsigned int attr,
3075                   const struct nft_expr *expr, bool reset)
3076 {
3077         struct nlattr *nest;
3078
3079         nest = nla_nest_start_noflag(skb, attr);
3080         if (!nest)
3081                 goto nla_put_failure;
3082         if (nf_tables_fill_expr_info(skb, expr, reset) < 0)
3083                 goto nla_put_failure;
3084         nla_nest_end(skb, nest);
3085         return 0;
3086
3087 nla_put_failure:
3088         return -1;
3089 }
3090
3091 struct nft_expr_info {
3092         const struct nft_expr_ops       *ops;
3093         const struct nlattr             *attr;
3094         struct nlattr                   *tb[NFT_EXPR_MAXATTR + 1];
3095 };
3096
3097 static int nf_tables_expr_parse(const struct nft_ctx *ctx,
3098                                 const struct nlattr *nla,
3099                                 struct nft_expr_info *info)
3100 {
3101         const struct nft_expr_type *type;
3102         const struct nft_expr_ops *ops;
3103         struct nlattr *tb[NFTA_EXPR_MAX + 1];
3104         int err;
3105
3106         err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
3107                                           nft_expr_policy, NULL);
3108         if (err < 0)
3109                 return err;
3110
3111         type = nft_expr_type_get(ctx->net, ctx->family, tb[NFTA_EXPR_NAME]);
3112         if (IS_ERR(type))
3113                 return PTR_ERR(type);
3114
3115         if (tb[NFTA_EXPR_DATA]) {
3116                 err = nla_parse_nested_deprecated(info->tb, type->maxattr,
3117                                                   tb[NFTA_EXPR_DATA],
3118                                                   type->policy, NULL);
3119                 if (err < 0)
3120                         goto err1;
3121         } else
3122                 memset(info->tb, 0, sizeof(info->tb[0]) * (type->maxattr + 1));
3123
3124         if (type->select_ops != NULL) {
3125                 ops = type->select_ops(ctx,
3126                                        (const struct nlattr * const *)info->tb);
3127                 if (IS_ERR(ops)) {
3128                         err = PTR_ERR(ops);
3129 #ifdef CONFIG_MODULES
3130                         if (err == -EAGAIN)
3131                                 if (nft_expr_type_request_module(ctx->net,
3132                                                                  ctx->family,
3133                                                                  tb[NFTA_EXPR_NAME]) != -EAGAIN)
3134                                         err = -ENOENT;
3135 #endif
3136                         goto err1;
3137                 }
3138         } else
3139                 ops = type->ops;
3140
3141         info->attr = nla;
3142         info->ops = ops;
3143
3144         return 0;
3145
3146 err1:
3147         module_put(type->owner);
3148         return err;
3149 }
3150
3151 int nft_expr_inner_parse(const struct nft_ctx *ctx, const struct nlattr *nla,
3152                          struct nft_expr_info *info)
3153 {
3154         struct nlattr *tb[NFTA_EXPR_MAX + 1];
3155         const struct nft_expr_type *type;
3156         int err;
3157
3158         err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
3159                                           nft_expr_policy, NULL);
3160         if (err < 0)
3161                 return err;
3162
3163         if (!tb[NFTA_EXPR_DATA])
3164                 return -EINVAL;
3165
3166         type = __nft_expr_type_get(ctx->family, tb[NFTA_EXPR_NAME]);
3167         if (!type)
3168                 return -ENOENT;
3169
3170         if (!type->inner_ops)
3171                 return -EOPNOTSUPP;
3172
3173         err = nla_parse_nested_deprecated(info->tb, type->maxattr,
3174                                           tb[NFTA_EXPR_DATA],
3175                                           type->policy, NULL);
3176         if (err < 0)
3177                 goto err_nla_parse;
3178
3179         info->attr = nla;
3180         info->ops = type->inner_ops;
3181
3182         return 0;
3183
3184 err_nla_parse:
3185         return err;
3186 }
3187
3188 static int nf_tables_newexpr(const struct nft_ctx *ctx,
3189                              const struct nft_expr_info *expr_info,
3190                              struct nft_expr *expr)
3191 {
3192         const struct nft_expr_ops *ops = expr_info->ops;
3193         int err;
3194
3195         expr->ops = ops;
3196         if (ops->init) {
3197                 err = ops->init(ctx, expr, (const struct nlattr **)expr_info->tb);
3198                 if (err < 0)
3199                         goto err1;
3200         }
3201
3202         return 0;
3203 err1:
3204         expr->ops = NULL;
3205         return err;
3206 }
3207
3208 static void nf_tables_expr_destroy(const struct nft_ctx *ctx,
3209                                    struct nft_expr *expr)
3210 {
3211         const struct nft_expr_type *type = expr->ops->type;
3212
3213         if (expr->ops->destroy)
3214                 expr->ops->destroy(ctx, expr);
3215         module_put(type->owner);
3216 }
3217
3218 static struct nft_expr *nft_expr_init(const struct nft_ctx *ctx,
3219                                       const struct nlattr *nla)
3220 {
3221         struct nft_expr_info expr_info;
3222         struct nft_expr *expr;
3223         struct module *owner;
3224         int err;
3225
3226         err = nf_tables_expr_parse(ctx, nla, &expr_info);
3227         if (err < 0)
3228                 goto err_expr_parse;
3229
3230         err = -EOPNOTSUPP;
3231         if (!(expr_info.ops->type->flags & NFT_EXPR_STATEFUL))
3232                 goto err_expr_stateful;
3233
3234         err = -ENOMEM;
3235         expr = kzalloc(expr_info.ops->size, GFP_KERNEL_ACCOUNT);
3236         if (expr == NULL)
3237                 goto err_expr_stateful;
3238
3239         err = nf_tables_newexpr(ctx, &expr_info, expr);
3240         if (err < 0)
3241                 goto err_expr_new;
3242
3243         return expr;
3244 err_expr_new:
3245         kfree(expr);
3246 err_expr_stateful:
3247         owner = expr_info.ops->type->owner;
3248         if (expr_info.ops->type->release_ops)
3249                 expr_info.ops->type->release_ops(expr_info.ops);
3250
3251         module_put(owner);
3252 err_expr_parse:
3253         return ERR_PTR(err);
3254 }
3255
3256 int nft_expr_clone(struct nft_expr *dst, struct nft_expr *src)
3257 {
3258         int err;
3259
3260         if (src->ops->clone) {
3261                 dst->ops = src->ops;
3262                 err = src->ops->clone(dst, src);
3263                 if (err < 0)
3264                         return err;
3265         } else {
3266                 memcpy(dst, src, src->ops->size);
3267         }
3268
3269         __module_get(src->ops->type->owner);
3270
3271         return 0;
3272 }
3273
3274 void nft_expr_destroy(const struct nft_ctx *ctx, struct nft_expr *expr)
3275 {
3276         nf_tables_expr_destroy(ctx, expr);
3277         kfree(expr);
3278 }
3279
3280 /*
3281  * Rules
3282  */
3283
3284 static struct nft_rule *__nft_rule_lookup(const struct nft_chain *chain,
3285                                           u64 handle)
3286 {
3287         struct nft_rule *rule;
3288
3289         // FIXME: this sucks
3290         list_for_each_entry_rcu(rule, &chain->rules, list) {
3291                 if (handle == rule->handle)
3292                         return rule;
3293         }
3294
3295         return ERR_PTR(-ENOENT);
3296 }
3297
3298 static struct nft_rule *nft_rule_lookup(const struct nft_chain *chain,
3299                                         const struct nlattr *nla)
3300 {
3301         if (nla == NULL)
3302                 return ERR_PTR(-EINVAL);
3303
3304         return __nft_rule_lookup(chain, be64_to_cpu(nla_get_be64(nla)));
3305 }
3306
3307 static const struct nla_policy nft_rule_policy[NFTA_RULE_MAX + 1] = {
3308         [NFTA_RULE_TABLE]       = { .type = NLA_STRING,
3309                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
3310         [NFTA_RULE_CHAIN]       = { .type = NLA_STRING,
3311                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
3312         [NFTA_RULE_HANDLE]      = { .type = NLA_U64 },
3313         [NFTA_RULE_EXPRESSIONS] = { .type = NLA_NESTED },
3314         [NFTA_RULE_COMPAT]      = { .type = NLA_NESTED },
3315         [NFTA_RULE_POSITION]    = { .type = NLA_U64 },
3316         [NFTA_RULE_USERDATA]    = { .type = NLA_BINARY,
3317                                     .len = NFT_USERDATA_MAXLEN },
3318         [NFTA_RULE_ID]          = { .type = NLA_U32 },
3319         [NFTA_RULE_POSITION_ID] = { .type = NLA_U32 },
3320         [NFTA_RULE_CHAIN_ID]    = { .type = NLA_U32 },
3321 };
3322
3323 static int nf_tables_fill_rule_info(struct sk_buff *skb, struct net *net,
3324                                     u32 portid, u32 seq, int event,
3325                                     u32 flags, int family,
3326                                     const struct nft_table *table,
3327                                     const struct nft_chain *chain,
3328                                     const struct nft_rule *rule, u64 handle,
3329                                     bool reset)
3330 {
3331         struct nlmsghdr *nlh;
3332         const struct nft_expr *expr, *next;
3333         struct nlattr *list;
3334         u16 type = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
3335
3336         nlh = nfnl_msg_put(skb, portid, seq, type, flags, family, NFNETLINK_V0,
3337                            nft_base_seq(net));
3338         if (!nlh)
3339                 goto nla_put_failure;
3340
3341         if (nla_put_string(skb, NFTA_RULE_TABLE, table->name))
3342                 goto nla_put_failure;
3343         if (nla_put_string(skb, NFTA_RULE_CHAIN, chain->name))
3344                 goto nla_put_failure;
3345         if (nla_put_be64(skb, NFTA_RULE_HANDLE, cpu_to_be64(rule->handle),
3346                          NFTA_RULE_PAD))
3347                 goto nla_put_failure;
3348
3349         if (event != NFT_MSG_DELRULE && handle) {
3350                 if (nla_put_be64(skb, NFTA_RULE_POSITION, cpu_to_be64(handle),
3351                                  NFTA_RULE_PAD))
3352                         goto nla_put_failure;
3353         }
3354
3355         if (chain->flags & NFT_CHAIN_HW_OFFLOAD)
3356                 nft_flow_rule_stats(chain, rule);
3357
3358         list = nla_nest_start_noflag(skb, NFTA_RULE_EXPRESSIONS);
3359         if (list == NULL)
3360                 goto nla_put_failure;
3361         nft_rule_for_each_expr(expr, next, rule) {
3362                 if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr, reset) < 0)
3363                         goto nla_put_failure;
3364         }
3365         nla_nest_end(skb, list);
3366
3367         if (rule->udata) {
3368                 struct nft_userdata *udata = nft_userdata(rule);
3369                 if (nla_put(skb, NFTA_RULE_USERDATA, udata->len + 1,
3370                             udata->data) < 0)
3371                         goto nla_put_failure;
3372         }
3373
3374         nlmsg_end(skb, nlh);
3375         return 0;
3376
3377 nla_put_failure:
3378         nlmsg_trim(skb, nlh);
3379         return -1;
3380 }
3381
3382 static void nf_tables_rule_notify(const struct nft_ctx *ctx,
3383                                   const struct nft_rule *rule, int event)
3384 {
3385         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
3386         const struct nft_rule *prule;
3387         struct sk_buff *skb;
3388         u64 handle = 0;
3389         u16 flags = 0;
3390         int err;
3391
3392         if (!ctx->report &&
3393             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
3394                 return;
3395
3396         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
3397         if (skb == NULL)
3398                 goto err;
3399
3400         if (event == NFT_MSG_NEWRULE &&
3401             !list_is_first(&rule->list, &ctx->chain->rules) &&
3402             !list_is_last(&rule->list, &ctx->chain->rules)) {
3403                 prule = list_prev_entry(rule, list);
3404                 handle = prule->handle;
3405         }
3406         if (ctx->flags & (NLM_F_APPEND | NLM_F_REPLACE))
3407                 flags |= NLM_F_APPEND;
3408         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
3409                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
3410
3411         err = nf_tables_fill_rule_info(skb, ctx->net, ctx->portid, ctx->seq,
3412                                        event, flags, ctx->family, ctx->table,
3413                                        ctx->chain, rule, handle, false);
3414         if (err < 0) {
3415                 kfree_skb(skb);
3416                 goto err;
3417         }
3418
3419         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
3420         return;
3421 err:
3422         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
3423 }
3424
3425 static void audit_log_rule_reset(const struct nft_table *table,
3426                                  unsigned int base_seq,
3427                                  unsigned int nentries)
3428 {
3429         char *buf = kasprintf(GFP_ATOMIC, "%s:%u",
3430                               table->name, base_seq);
3431
3432         audit_log_nfcfg(buf, table->family, nentries,
3433                         AUDIT_NFT_OP_RULE_RESET, GFP_ATOMIC);
3434         kfree(buf);
3435 }
3436
3437 struct nft_rule_dump_ctx {
3438         char *table;
3439         char *chain;
3440 };
3441
3442 static int __nf_tables_dump_rules(struct sk_buff *skb,
3443                                   unsigned int *idx,
3444                                   struct netlink_callback *cb,
3445                                   const struct nft_table *table,
3446                                   const struct nft_chain *chain,
3447                                   bool reset)
3448 {
3449         struct net *net = sock_net(skb->sk);
3450         const struct nft_rule *rule, *prule;
3451         unsigned int s_idx = cb->args[0];
3452         u64 handle;
3453
3454         prule = NULL;
3455         list_for_each_entry_rcu(rule, &chain->rules, list) {
3456                 if (!nft_is_active(net, rule))
3457                         goto cont_skip;
3458                 if (*idx < s_idx)
3459                         goto cont;
3460                 if (*idx > s_idx) {
3461                         memset(&cb->args[1], 0,
3462                                         sizeof(cb->args) - sizeof(cb->args[0]));
3463                 }
3464                 if (prule)
3465                         handle = prule->handle;
3466                 else
3467                         handle = 0;
3468
3469                 if (nf_tables_fill_rule_info(skb, net, NETLINK_CB(cb->skb).portid,
3470                                         cb->nlh->nlmsg_seq,
3471                                         NFT_MSG_NEWRULE,
3472                                         NLM_F_MULTI | NLM_F_APPEND,
3473                                         table->family,
3474                                         table, chain, rule, handle, reset) < 0)
3475                         return 1;
3476
3477                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
3478 cont:
3479                 prule = rule;
3480 cont_skip:
3481                 (*idx)++;
3482         }
3483
3484         if (reset && *idx)
3485                 audit_log_rule_reset(table, cb->seq, *idx);
3486
3487         return 0;
3488 }
3489
3490 static int nf_tables_dump_rules(struct sk_buff *skb,
3491                                 struct netlink_callback *cb)
3492 {
3493         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
3494         const struct nft_rule_dump_ctx *ctx = cb->data;
3495         struct nft_table *table;
3496         const struct nft_chain *chain;
3497         unsigned int idx = 0;
3498         struct net *net = sock_net(skb->sk);
3499         int family = nfmsg->nfgen_family;
3500         struct nftables_pernet *nft_net;
3501         bool reset = false;
3502
3503         if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETRULE_RESET)
3504                 reset = true;
3505
3506         rcu_read_lock();
3507         nft_net = nft_pernet(net);
3508         cb->seq = READ_ONCE(nft_net->base_seq);
3509
3510         list_for_each_entry_rcu(table, &nft_net->tables, list) {
3511                 if (family != NFPROTO_UNSPEC && family != table->family)
3512                         continue;
3513
3514                 if (ctx && ctx->table && strcmp(ctx->table, table->name) != 0)
3515                         continue;
3516
3517                 if (ctx && ctx->table && ctx->chain) {
3518                         struct rhlist_head *list, *tmp;
3519
3520                         list = rhltable_lookup(&table->chains_ht, ctx->chain,
3521                                                nft_chain_ht_params);
3522                         if (!list)
3523                                 goto done;
3524
3525                         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
3526                                 if (!nft_is_active(net, chain))
3527                                         continue;
3528                                 __nf_tables_dump_rules(skb, &idx,
3529                                                        cb, table, chain, reset);
3530                                 break;
3531                         }
3532                         goto done;
3533                 }
3534
3535                 list_for_each_entry_rcu(chain, &table->chains, list) {
3536                         if (__nf_tables_dump_rules(skb, &idx,
3537                                                    cb, table, chain, reset))
3538                                 goto done;
3539                 }
3540
3541                 if (ctx && ctx->table)
3542                         break;
3543         }
3544 done:
3545         rcu_read_unlock();
3546
3547         cb->args[0] = idx;
3548         return skb->len;
3549 }
3550
3551 static int nf_tables_dump_rules_start(struct netlink_callback *cb)
3552 {
3553         const struct nlattr * const *nla = cb->data;
3554         struct nft_rule_dump_ctx *ctx = NULL;
3555
3556         if (nla[NFTA_RULE_TABLE] || nla[NFTA_RULE_CHAIN]) {
3557                 ctx = kzalloc(sizeof(*ctx), GFP_ATOMIC);
3558                 if (!ctx)
3559                         return -ENOMEM;
3560
3561                 if (nla[NFTA_RULE_TABLE]) {
3562                         ctx->table = nla_strdup(nla[NFTA_RULE_TABLE],
3563                                                         GFP_ATOMIC);
3564                         if (!ctx->table) {
3565                                 kfree(ctx);
3566                                 return -ENOMEM;
3567                         }
3568                 }
3569                 if (nla[NFTA_RULE_CHAIN]) {
3570                         ctx->chain = nla_strdup(nla[NFTA_RULE_CHAIN],
3571                                                 GFP_ATOMIC);
3572                         if (!ctx->chain) {
3573                                 kfree(ctx->table);
3574                                 kfree(ctx);
3575                                 return -ENOMEM;
3576                         }
3577                 }
3578         }
3579
3580         cb->data = ctx;
3581         return 0;
3582 }
3583
3584 static int nf_tables_dump_rules_done(struct netlink_callback *cb)
3585 {
3586         struct nft_rule_dump_ctx *ctx = cb->data;
3587
3588         if (ctx) {
3589                 kfree(ctx->table);
3590                 kfree(ctx->chain);
3591                 kfree(ctx);
3592         }
3593         return 0;
3594 }
3595
3596 /* called with rcu_read_lock held */
3597 static int nf_tables_getrule(struct sk_buff *skb, const struct nfnl_info *info,
3598                              const struct nlattr * const nla[])
3599 {
3600         struct netlink_ext_ack *extack = info->extack;
3601         u8 genmask = nft_genmask_cur(info->net);
3602         u8 family = info->nfmsg->nfgen_family;
3603         const struct nft_chain *chain;
3604         const struct nft_rule *rule;
3605         struct net *net = info->net;
3606         struct nft_table *table;
3607         struct sk_buff *skb2;
3608         bool reset = false;
3609         int err;
3610
3611         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
3612                 struct netlink_dump_control c = {
3613                         .start= nf_tables_dump_rules_start,
3614                         .dump = nf_tables_dump_rules,
3615                         .done = nf_tables_dump_rules_done,
3616                         .module = THIS_MODULE,
3617                         .data = (void *)nla,
3618                 };
3619
3620                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
3621         }
3622
3623         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask, 0);
3624         if (IS_ERR(table)) {
3625                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3626                 return PTR_ERR(table);
3627         }
3628
3629         chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN], genmask);
3630         if (IS_ERR(chain)) {
3631                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3632                 return PTR_ERR(chain);
3633         }
3634
3635         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
3636         if (IS_ERR(rule)) {
3637                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3638                 return PTR_ERR(rule);
3639         }
3640
3641         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
3642         if (!skb2)
3643                 return -ENOMEM;
3644
3645         if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETRULE_RESET)
3646                 reset = true;
3647
3648         err = nf_tables_fill_rule_info(skb2, net, NETLINK_CB(skb).portid,
3649                                        info->nlh->nlmsg_seq, NFT_MSG_NEWRULE, 0,
3650                                        family, table, chain, rule, 0, reset);
3651         if (err < 0)
3652                 goto err_fill_rule_info;
3653
3654         if (reset)
3655                 audit_log_rule_reset(table, nft_pernet(net)->base_seq, 1);
3656
3657         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
3658
3659 err_fill_rule_info:
3660         kfree_skb(skb2);
3661         return err;
3662 }
3663
3664 void nf_tables_rule_destroy(const struct nft_ctx *ctx, struct nft_rule *rule)
3665 {
3666         struct nft_expr *expr, *next;
3667
3668         /*
3669          * Careful: some expressions might not be initialized in case this
3670          * is called on error from nf_tables_newrule().
3671          */
3672         expr = nft_expr_first(rule);
3673         while (nft_expr_more(rule, expr)) {
3674                 next = nft_expr_next(expr);
3675                 nf_tables_expr_destroy(ctx, expr);
3676                 expr = next;
3677         }
3678         kfree(rule);
3679 }
3680
3681 static void nf_tables_rule_release(const struct nft_ctx *ctx, struct nft_rule *rule)
3682 {
3683         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_RELEASE);
3684         nf_tables_rule_destroy(ctx, rule);
3685 }
3686
3687 int nft_chain_validate(const struct nft_ctx *ctx, const struct nft_chain *chain)
3688 {
3689         struct nft_expr *expr, *last;
3690         const struct nft_data *data;
3691         struct nft_rule *rule;
3692         int err;
3693
3694         if (ctx->level == NFT_JUMP_STACK_SIZE)
3695                 return -EMLINK;
3696
3697         list_for_each_entry(rule, &chain->rules, list) {
3698                 if (fatal_signal_pending(current))
3699                         return -EINTR;
3700
3701                 if (!nft_is_active_next(ctx->net, rule))
3702                         continue;
3703
3704                 nft_rule_for_each_expr(expr, last, rule) {
3705                         if (!expr->ops->validate)
3706                                 continue;
3707
3708                         err = expr->ops->validate(ctx, expr, &data);
3709                         if (err < 0)
3710                                 return err;
3711                 }
3712         }
3713
3714         return 0;
3715 }
3716 EXPORT_SYMBOL_GPL(nft_chain_validate);
3717
3718 static int nft_table_validate(struct net *net, const struct nft_table *table)
3719 {
3720         struct nft_chain *chain;
3721         struct nft_ctx ctx = {
3722                 .net    = net,
3723                 .family = table->family,
3724         };
3725         int err;
3726
3727         list_for_each_entry(chain, &table->chains, list) {
3728                 if (!nft_is_base_chain(chain))
3729                         continue;
3730
3731                 ctx.chain = chain;
3732                 err = nft_chain_validate(&ctx, chain);
3733                 if (err < 0)
3734                         return err;
3735
3736                 cond_resched();
3737         }
3738
3739         return 0;
3740 }
3741
3742 int nft_setelem_validate(const struct nft_ctx *ctx, struct nft_set *set,
3743                          const struct nft_set_iter *iter,
3744                          struct nft_set_elem *elem)
3745 {
3746         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
3747         struct nft_ctx *pctx = (struct nft_ctx *)ctx;
3748         const struct nft_data *data;
3749         int err;
3750
3751         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
3752             *nft_set_ext_flags(ext) & NFT_SET_ELEM_INTERVAL_END)
3753                 return 0;
3754
3755         data = nft_set_ext_data(ext);
3756         switch (data->verdict.code) {
3757         case NFT_JUMP:
3758         case NFT_GOTO:
3759                 pctx->level++;
3760                 err = nft_chain_validate(ctx, data->verdict.chain);
3761                 if (err < 0)
3762                         return err;
3763                 pctx->level--;
3764                 break;
3765         default:
3766                 break;
3767         }
3768
3769         return 0;
3770 }
3771
3772 int nft_set_catchall_validate(const struct nft_ctx *ctx, struct nft_set *set)
3773 {
3774         u8 genmask = nft_genmask_next(ctx->net);
3775         struct nft_set_elem_catchall *catchall;
3776         struct nft_set_elem elem;
3777         struct nft_set_ext *ext;
3778         int ret = 0;
3779
3780         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
3781                 ext = nft_set_elem_ext(set, catchall->elem);
3782                 if (!nft_set_elem_active(ext, genmask))
3783                         continue;
3784
3785                 elem.priv = catchall->elem;
3786                 ret = nft_setelem_validate(ctx, set, NULL, &elem);
3787                 if (ret < 0)
3788                         return ret;
3789         }
3790
3791         return ret;
3792 }
3793
3794 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
3795                                              const struct nft_chain *chain,
3796                                              const struct nlattr *nla);
3797
3798 #define NFT_RULE_MAXEXPRS       128
3799
3800 static int nf_tables_newrule(struct sk_buff *skb, const struct nfnl_info *info,
3801                              const struct nlattr * const nla[])
3802 {
3803         struct nftables_pernet *nft_net = nft_pernet(info->net);
3804         struct netlink_ext_ack *extack = info->extack;
3805         unsigned int size, i, n, ulen = 0, usize = 0;
3806         u8 genmask = nft_genmask_next(info->net);
3807         struct nft_rule *rule, *old_rule = NULL;
3808         struct nft_expr_info *expr_info = NULL;
3809         u8 family = info->nfmsg->nfgen_family;
3810         struct nft_flow_rule *flow = NULL;
3811         struct net *net = info->net;
3812         struct nft_userdata *udata;
3813         struct nft_table *table;
3814         struct nft_chain *chain;
3815         struct nft_trans *trans;
3816         u64 handle, pos_handle;
3817         struct nft_expr *expr;
3818         struct nft_ctx ctx;
3819         struct nlattr *tmp;
3820         int err, rem;
3821
3822         lockdep_assert_held(&nft_net->commit_mutex);
3823
3824         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
3825                                  NETLINK_CB(skb).portid);
3826         if (IS_ERR(table)) {
3827                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3828                 return PTR_ERR(table);
3829         }
3830
3831         if (nla[NFTA_RULE_CHAIN]) {
3832                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
3833                                          genmask);
3834                 if (IS_ERR(chain)) {
3835                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3836                         return PTR_ERR(chain);
3837                 }
3838
3839         } else if (nla[NFTA_RULE_CHAIN_ID]) {
3840                 chain = nft_chain_lookup_byid(net, table, nla[NFTA_RULE_CHAIN_ID],
3841                                               genmask);
3842                 if (IS_ERR(chain)) {
3843                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN_ID]);
3844                         return PTR_ERR(chain);
3845                 }
3846         } else {
3847                 return -EINVAL;
3848         }
3849
3850         if (nft_chain_is_bound(chain))
3851                 return -EOPNOTSUPP;
3852
3853         if (nla[NFTA_RULE_HANDLE]) {
3854                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_HANDLE]));
3855                 rule = __nft_rule_lookup(chain, handle);
3856                 if (IS_ERR(rule)) {
3857                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3858                         return PTR_ERR(rule);
3859                 }
3860
3861                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
3862                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3863                         return -EEXIST;
3864                 }
3865                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
3866                         old_rule = rule;
3867                 else
3868                         return -EOPNOTSUPP;
3869         } else {
3870                 if (!(info->nlh->nlmsg_flags & NLM_F_CREATE) ||
3871                     info->nlh->nlmsg_flags & NLM_F_REPLACE)
3872                         return -EINVAL;
3873                 handle = nf_tables_alloc_handle(table);
3874
3875                 if (nla[NFTA_RULE_POSITION]) {
3876                         pos_handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_POSITION]));
3877                         old_rule = __nft_rule_lookup(chain, pos_handle);
3878                         if (IS_ERR(old_rule)) {
3879                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION]);
3880                                 return PTR_ERR(old_rule);
3881                         }
3882                 } else if (nla[NFTA_RULE_POSITION_ID]) {
3883                         old_rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_POSITION_ID]);
3884                         if (IS_ERR(old_rule)) {
3885                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION_ID]);
3886                                 return PTR_ERR(old_rule);
3887                         }
3888                 }
3889         }
3890
3891         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
3892
3893         n = 0;
3894         size = 0;
3895         if (nla[NFTA_RULE_EXPRESSIONS]) {
3896                 expr_info = kvmalloc_array(NFT_RULE_MAXEXPRS,
3897                                            sizeof(struct nft_expr_info),
3898                                            GFP_KERNEL);
3899                 if (!expr_info)
3900                         return -ENOMEM;
3901
3902                 nla_for_each_nested(tmp, nla[NFTA_RULE_EXPRESSIONS], rem) {
3903                         err = -EINVAL;
3904                         if (nla_type(tmp) != NFTA_LIST_ELEM)
3905                                 goto err_release_expr;
3906                         if (n == NFT_RULE_MAXEXPRS)
3907                                 goto err_release_expr;
3908                         err = nf_tables_expr_parse(&ctx, tmp, &expr_info[n]);
3909                         if (err < 0) {
3910                                 NL_SET_BAD_ATTR(extack, tmp);
3911                                 goto err_release_expr;
3912                         }
3913                         size += expr_info[n].ops->size;
3914                         n++;
3915                 }
3916         }
3917         /* Check for overflow of dlen field */
3918         err = -EFBIG;
3919         if (size >= 1 << 12)
3920                 goto err_release_expr;
3921
3922         if (nla[NFTA_RULE_USERDATA]) {
3923                 ulen = nla_len(nla[NFTA_RULE_USERDATA]);
3924                 if (ulen > 0)
3925                         usize = sizeof(struct nft_userdata) + ulen;
3926         }
3927
3928         err = -ENOMEM;
3929         rule = kzalloc(sizeof(*rule) + size + usize, GFP_KERNEL_ACCOUNT);
3930         if (rule == NULL)
3931                 goto err_release_expr;
3932
3933         nft_activate_next(net, rule);
3934
3935         rule->handle = handle;
3936         rule->dlen   = size;
3937         rule->udata  = ulen ? 1 : 0;
3938
3939         if (ulen) {
3940                 udata = nft_userdata(rule);
3941                 udata->len = ulen - 1;
3942                 nla_memcpy(udata->data, nla[NFTA_RULE_USERDATA], ulen);
3943         }
3944
3945         expr = nft_expr_first(rule);
3946         for (i = 0; i < n; i++) {
3947                 err = nf_tables_newexpr(&ctx, &expr_info[i], expr);
3948                 if (err < 0) {
3949                         NL_SET_BAD_ATTR(extack, expr_info[i].attr);
3950                         goto err_release_rule;
3951                 }
3952
3953                 if (expr_info[i].ops->validate)
3954                         nft_validate_state_update(table, NFT_VALIDATE_NEED);
3955
3956                 expr_info[i].ops = NULL;
3957                 expr = nft_expr_next(expr);
3958         }
3959
3960         if (chain->flags & NFT_CHAIN_HW_OFFLOAD) {
3961                 flow = nft_flow_rule_create(net, rule);
3962                 if (IS_ERR(flow)) {
3963                         err = PTR_ERR(flow);
3964                         goto err_release_rule;
3965                 }
3966         }
3967
3968         if (!nft_use_inc(&chain->use)) {
3969                 err = -EMFILE;
3970                 goto err_release_rule;
3971         }
3972
3973         if (info->nlh->nlmsg_flags & NLM_F_REPLACE) {
3974                 err = nft_delrule(&ctx, old_rule);
3975                 if (err < 0)
3976                         goto err_destroy_flow_rule;
3977
3978                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
3979                 if (trans == NULL) {
3980                         err = -ENOMEM;
3981                         goto err_destroy_flow_rule;
3982                 }
3983                 list_add_tail_rcu(&rule->list, &old_rule->list);
3984         } else {
3985                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
3986                 if (!trans) {
3987                         err = -ENOMEM;
3988                         goto err_destroy_flow_rule;
3989                 }
3990
3991                 if (info->nlh->nlmsg_flags & NLM_F_APPEND) {
3992                         if (old_rule)
3993                                 list_add_rcu(&rule->list, &old_rule->list);
3994                         else
3995                                 list_add_tail_rcu(&rule->list, &chain->rules);
3996                  } else {
3997                         if (old_rule)
3998                                 list_add_tail_rcu(&rule->list, &old_rule->list);
3999                         else
4000                                 list_add_rcu(&rule->list, &chain->rules);
4001                 }
4002         }
4003         kvfree(expr_info);
4004
4005         if (flow)
4006                 nft_trans_flow_rule(trans) = flow;
4007
4008         if (table->validate_state == NFT_VALIDATE_DO)
4009                 return nft_table_validate(net, table);
4010
4011         return 0;
4012
4013 err_destroy_flow_rule:
4014         nft_use_dec_restore(&chain->use);
4015         if (flow)
4016                 nft_flow_rule_destroy(flow);
4017 err_release_rule:
4018         nft_rule_expr_deactivate(&ctx, rule, NFT_TRANS_PREPARE_ERROR);
4019         nf_tables_rule_destroy(&ctx, rule);
4020 err_release_expr:
4021         for (i = 0; i < n; i++) {
4022                 if (expr_info[i].ops) {
4023                         module_put(expr_info[i].ops->type->owner);
4024                         if (expr_info[i].ops->type->release_ops)
4025                                 expr_info[i].ops->type->release_ops(expr_info[i].ops);
4026                 }
4027         }
4028         kvfree(expr_info);
4029
4030         return err;
4031 }
4032
4033 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
4034                                              const struct nft_chain *chain,
4035                                              const struct nlattr *nla)
4036 {
4037         struct nftables_pernet *nft_net = nft_pernet(net);
4038         u32 id = ntohl(nla_get_be32(nla));
4039         struct nft_trans *trans;
4040
4041         list_for_each_entry(trans, &nft_net->commit_list, list) {
4042                 if (trans->msg_type == NFT_MSG_NEWRULE &&
4043                     trans->ctx.chain == chain &&
4044                     id == nft_trans_rule_id(trans))
4045                         return nft_trans_rule(trans);
4046         }
4047         return ERR_PTR(-ENOENT);
4048 }
4049
4050 static int nf_tables_delrule(struct sk_buff *skb, const struct nfnl_info *info,
4051                              const struct nlattr * const nla[])
4052 {
4053         struct netlink_ext_ack *extack = info->extack;
4054         u8 genmask = nft_genmask_next(info->net);
4055         u8 family = info->nfmsg->nfgen_family;
4056         struct nft_chain *chain = NULL;
4057         struct net *net = info->net;
4058         struct nft_table *table;
4059         struct nft_rule *rule;
4060         struct nft_ctx ctx;
4061         int err = 0;
4062
4063         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
4064                                  NETLINK_CB(skb).portid);
4065         if (IS_ERR(table)) {
4066                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
4067                 return PTR_ERR(table);
4068         }
4069
4070         if (nla[NFTA_RULE_CHAIN]) {
4071                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
4072                                          genmask);
4073                 if (IS_ERR(chain)) {
4074                         if (PTR_ERR(chain) == -ENOENT &&
4075                             NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYRULE)
4076                                 return 0;
4077
4078                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
4079                         return PTR_ERR(chain);
4080                 }
4081                 if (nft_chain_is_bound(chain))
4082                         return -EOPNOTSUPP;
4083         }
4084
4085         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
4086
4087         if (chain) {
4088                 if (nla[NFTA_RULE_HANDLE]) {
4089                         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
4090                         if (IS_ERR(rule)) {
4091                                 if (PTR_ERR(rule) == -ENOENT &&
4092                                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYRULE)
4093                                         return 0;
4094
4095                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
4096                                 return PTR_ERR(rule);
4097                         }
4098
4099                         err = nft_delrule(&ctx, rule);
4100                 } else if (nla[NFTA_RULE_ID]) {
4101                         rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_ID]);
4102                         if (IS_ERR(rule)) {
4103                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_ID]);
4104                                 return PTR_ERR(rule);
4105                         }
4106
4107                         err = nft_delrule(&ctx, rule);
4108                 } else {
4109                         err = nft_delrule_by_chain(&ctx);
4110                 }
4111         } else {
4112                 list_for_each_entry(chain, &table->chains, list) {
4113                         if (!nft_is_active_next(net, chain))
4114                                 continue;
4115                         if (nft_chain_is_bound(chain))
4116                                 continue;
4117
4118                         ctx.chain = chain;
4119                         err = nft_delrule_by_chain(&ctx);
4120                         if (err < 0)
4121                                 break;
4122                 }
4123         }
4124
4125         return err;
4126 }
4127
4128 /*
4129  * Sets
4130  */
4131 static const struct nft_set_type *nft_set_types[] = {
4132         &nft_set_hash_fast_type,
4133         &nft_set_hash_type,
4134         &nft_set_rhash_type,
4135         &nft_set_bitmap_type,
4136         &nft_set_rbtree_type,
4137 #if defined(CONFIG_X86_64) && !defined(CONFIG_UML)
4138         &nft_set_pipapo_avx2_type,
4139 #endif
4140         &nft_set_pipapo_type,
4141 };
4142
4143 #define NFT_SET_FEATURES        (NFT_SET_INTERVAL | NFT_SET_MAP | \
4144                                  NFT_SET_TIMEOUT | NFT_SET_OBJECT | \
4145                                  NFT_SET_EVAL)
4146
4147 static bool nft_set_ops_candidate(const struct nft_set_type *type, u32 flags)
4148 {
4149         return (flags & type->features) == (flags & NFT_SET_FEATURES);
4150 }
4151
4152 /*
4153  * Select a set implementation based on the data characteristics and the
4154  * given policy. The total memory use might not be known if no size is
4155  * given, in that case the amount of memory per element is used.
4156  */
4157 static const struct nft_set_ops *
4158 nft_select_set_ops(const struct nft_ctx *ctx,
4159                    const struct nlattr * const nla[],
4160                    const struct nft_set_desc *desc)
4161 {
4162         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
4163         const struct nft_set_ops *ops, *bops;
4164         struct nft_set_estimate est, best;
4165         const struct nft_set_type *type;
4166         u32 flags = 0;
4167         int i;
4168
4169         lockdep_assert_held(&nft_net->commit_mutex);
4170         lockdep_nfnl_nft_mutex_not_held();
4171
4172         if (nla[NFTA_SET_FLAGS] != NULL)
4173                 flags = ntohl(nla_get_be32(nla[NFTA_SET_FLAGS]));
4174
4175         bops        = NULL;
4176         best.size   = ~0;
4177         best.lookup = ~0;
4178         best.space  = ~0;
4179
4180         for (i = 0; i < ARRAY_SIZE(nft_set_types); i++) {
4181                 type = nft_set_types[i];
4182                 ops = &type->ops;
4183
4184                 if (!nft_set_ops_candidate(type, flags))
4185                         continue;
4186                 if (!ops->estimate(desc, flags, &est))
4187                         continue;
4188
4189                 switch (desc->policy) {
4190                 case NFT_SET_POL_PERFORMANCE:
4191                         if (est.lookup < best.lookup)
4192                                 break;
4193                         if (est.lookup == best.lookup &&
4194                             est.space < best.space)
4195                                 break;
4196                         continue;
4197                 case NFT_SET_POL_MEMORY:
4198                         if (!desc->size) {
4199                                 if (est.space < best.space)
4200                                         break;
4201                                 if (est.space == best.space &&
4202                                     est.lookup < best.lookup)
4203                                         break;
4204                         } else if (est.size < best.size || !bops) {
4205                                 break;
4206                         }
4207                         continue;
4208                 default:
4209                         break;
4210                 }
4211
4212                 bops = ops;
4213                 best = est;
4214         }
4215
4216         if (bops != NULL)
4217                 return bops;
4218
4219         return ERR_PTR(-EOPNOTSUPP);
4220 }
4221
4222 static const struct nla_policy nft_set_policy[NFTA_SET_MAX + 1] = {
4223         [NFTA_SET_TABLE]                = { .type = NLA_STRING,
4224                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
4225         [NFTA_SET_NAME]                 = { .type = NLA_STRING,
4226                                             .len = NFT_SET_MAXNAMELEN - 1 },
4227         [NFTA_SET_FLAGS]                = { .type = NLA_U32 },
4228         [NFTA_SET_KEY_TYPE]             = { .type = NLA_U32 },
4229         [NFTA_SET_KEY_LEN]              = { .type = NLA_U32 },
4230         [NFTA_SET_DATA_TYPE]            = { .type = NLA_U32 },
4231         [NFTA_SET_DATA_LEN]             = { .type = NLA_U32 },
4232         [NFTA_SET_POLICY]               = { .type = NLA_U32 },
4233         [NFTA_SET_DESC]                 = { .type = NLA_NESTED },
4234         [NFTA_SET_ID]                   = { .type = NLA_U32 },
4235         [NFTA_SET_TIMEOUT]              = { .type = NLA_U64 },
4236         [NFTA_SET_GC_INTERVAL]          = { .type = NLA_U32 },
4237         [NFTA_SET_USERDATA]             = { .type = NLA_BINARY,
4238                                             .len  = NFT_USERDATA_MAXLEN },
4239         [NFTA_SET_OBJ_TYPE]             = { .type = NLA_U32 },
4240         [NFTA_SET_HANDLE]               = { .type = NLA_U64 },
4241         [NFTA_SET_EXPR]                 = { .type = NLA_NESTED },
4242         [NFTA_SET_EXPRESSIONS]          = { .type = NLA_NESTED },
4243 };
4244
4245 static const struct nla_policy nft_set_desc_policy[NFTA_SET_DESC_MAX + 1] = {
4246         [NFTA_SET_DESC_SIZE]            = { .type = NLA_U32 },
4247         [NFTA_SET_DESC_CONCAT]          = { .type = NLA_NESTED },
4248 };
4249
4250 static struct nft_set *nft_set_lookup(const struct nft_table *table,
4251                                       const struct nlattr *nla, u8 genmask)
4252 {
4253         struct nft_set *set;
4254
4255         if (nla == NULL)
4256                 return ERR_PTR(-EINVAL);
4257
4258         list_for_each_entry_rcu(set, &table->sets, list) {
4259                 if (!nla_strcmp(nla, set->name) &&
4260                     nft_active_genmask(set, genmask))
4261                         return set;
4262         }
4263         return ERR_PTR(-ENOENT);
4264 }
4265
4266 static struct nft_set *nft_set_lookup_byhandle(const struct nft_table *table,
4267                                                const struct nlattr *nla,
4268                                                u8 genmask)
4269 {
4270         struct nft_set *set;
4271
4272         list_for_each_entry(set, &table->sets, list) {
4273                 if (be64_to_cpu(nla_get_be64(nla)) == set->handle &&
4274                     nft_active_genmask(set, genmask))
4275                         return set;
4276         }
4277         return ERR_PTR(-ENOENT);
4278 }
4279
4280 static struct nft_set *nft_set_lookup_byid(const struct net *net,
4281                                            const struct nft_table *table,
4282                                            const struct nlattr *nla, u8 genmask)
4283 {
4284         struct nftables_pernet *nft_net = nft_pernet(net);
4285         u32 id = ntohl(nla_get_be32(nla));
4286         struct nft_trans *trans;
4287
4288         list_for_each_entry(trans, &nft_net->commit_list, list) {
4289                 if (trans->msg_type == NFT_MSG_NEWSET) {
4290                         struct nft_set *set = nft_trans_set(trans);
4291
4292                         if (id == nft_trans_set_id(trans) &&
4293                             set->table == table &&
4294                             nft_active_genmask(set, genmask))
4295                                 return set;
4296                 }
4297         }
4298         return ERR_PTR(-ENOENT);
4299 }
4300
4301 struct nft_set *nft_set_lookup_global(const struct net *net,
4302                                       const struct nft_table *table,
4303                                       const struct nlattr *nla_set_name,
4304                                       const struct nlattr *nla_set_id,
4305                                       u8 genmask)
4306 {
4307         struct nft_set *set;
4308
4309         set = nft_set_lookup(table, nla_set_name, genmask);
4310         if (IS_ERR(set)) {
4311                 if (!nla_set_id)
4312                         return set;
4313
4314                 set = nft_set_lookup_byid(net, table, nla_set_id, genmask);
4315         }
4316         return set;
4317 }
4318 EXPORT_SYMBOL_GPL(nft_set_lookup_global);
4319
4320 static int nf_tables_set_alloc_name(struct nft_ctx *ctx, struct nft_set *set,
4321                                     const char *name)
4322 {
4323         const struct nft_set *i;
4324         const char *p;
4325         unsigned long *inuse;
4326         unsigned int n = 0, min = 0;
4327
4328         p = strchr(name, '%');
4329         if (p != NULL) {
4330                 if (p[1] != 'd' || strchr(p + 2, '%'))
4331                         return -EINVAL;
4332
4333                 inuse = (unsigned long *)get_zeroed_page(GFP_KERNEL);
4334                 if (inuse == NULL)
4335                         return -ENOMEM;
4336 cont:
4337                 list_for_each_entry(i, &ctx->table->sets, list) {
4338                         int tmp;
4339
4340                         if (!nft_is_active_next(ctx->net, i))
4341                                 continue;
4342                         if (!sscanf(i->name, name, &tmp))
4343                                 continue;
4344                         if (tmp < min || tmp >= min + BITS_PER_BYTE * PAGE_SIZE)
4345                                 continue;
4346
4347                         set_bit(tmp - min, inuse);
4348                 }
4349
4350                 n = find_first_zero_bit(inuse, BITS_PER_BYTE * PAGE_SIZE);
4351                 if (n >= BITS_PER_BYTE * PAGE_SIZE) {
4352                         min += BITS_PER_BYTE * PAGE_SIZE;
4353                         memset(inuse, 0, PAGE_SIZE);
4354                         goto cont;
4355                 }
4356                 free_page((unsigned long)inuse);
4357         }
4358
4359         set->name = kasprintf(GFP_KERNEL_ACCOUNT, name, min + n);
4360         if (!set->name)
4361                 return -ENOMEM;
4362
4363         list_for_each_entry(i, &ctx->table->sets, list) {
4364                 if (!nft_is_active_next(ctx->net, i))
4365                         continue;
4366                 if (!strcmp(set->name, i->name)) {
4367                         kfree(set->name);
4368                         set->name = NULL;
4369                         return -ENFILE;
4370                 }
4371         }
4372         return 0;
4373 }
4374
4375 int nf_msecs_to_jiffies64(const struct nlattr *nla, u64 *result)
4376 {
4377         u64 ms = be64_to_cpu(nla_get_be64(nla));
4378         u64 max = (u64)(~((u64)0));
4379
4380         max = div_u64(max, NSEC_PER_MSEC);
4381         if (ms >= max)
4382                 return -ERANGE;
4383
4384         ms *= NSEC_PER_MSEC;
4385         *result = nsecs_to_jiffies64(ms);
4386         return 0;
4387 }
4388
4389 __be64 nf_jiffies64_to_msecs(u64 input)
4390 {
4391         return cpu_to_be64(jiffies64_to_msecs(input));
4392 }
4393
4394 static int nf_tables_fill_set_concat(struct sk_buff *skb,
4395                                      const struct nft_set *set)
4396 {
4397         struct nlattr *concat, *field;
4398         int i;
4399
4400         concat = nla_nest_start_noflag(skb, NFTA_SET_DESC_CONCAT);
4401         if (!concat)
4402                 return -ENOMEM;
4403
4404         for (i = 0; i < set->field_count; i++) {
4405                 field = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
4406                 if (!field)
4407                         return -ENOMEM;
4408
4409                 if (nla_put_be32(skb, NFTA_SET_FIELD_LEN,
4410                                  htonl(set->field_len[i])))
4411                         return -ENOMEM;
4412
4413                 nla_nest_end(skb, field);
4414         }
4415
4416         nla_nest_end(skb, concat);
4417
4418         return 0;
4419 }
4420
4421 static int nf_tables_fill_set(struct sk_buff *skb, const struct nft_ctx *ctx,
4422                               const struct nft_set *set, u16 event, u16 flags)
4423 {
4424         u64 timeout = READ_ONCE(set->timeout);
4425         u32 gc_int = READ_ONCE(set->gc_int);
4426         u32 portid = ctx->portid;
4427         struct nlmsghdr *nlh;
4428         struct nlattr *nest;
4429         u32 seq = ctx->seq;
4430         int i;
4431
4432         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
4433         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
4434                            NFNETLINK_V0, nft_base_seq(ctx->net));
4435         if (!nlh)
4436                 goto nla_put_failure;
4437
4438         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
4439                 goto nla_put_failure;
4440         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
4441                 goto nla_put_failure;
4442         if (nla_put_be64(skb, NFTA_SET_HANDLE, cpu_to_be64(set->handle),
4443                          NFTA_SET_PAD))
4444                 goto nla_put_failure;
4445
4446         if (event == NFT_MSG_DELSET) {
4447                 nlmsg_end(skb, nlh);
4448                 return 0;
4449         }
4450
4451         if (set->flags != 0)
4452                 if (nla_put_be32(skb, NFTA_SET_FLAGS, htonl(set->flags)))
4453                         goto nla_put_failure;
4454
4455         if (nla_put_be32(skb, NFTA_SET_KEY_TYPE, htonl(set->ktype)))
4456                 goto nla_put_failure;
4457         if (nla_put_be32(skb, NFTA_SET_KEY_LEN, htonl(set->klen)))
4458                 goto nla_put_failure;
4459         if (set->flags & NFT_SET_MAP) {
4460                 if (nla_put_be32(skb, NFTA_SET_DATA_TYPE, htonl(set->dtype)))
4461                         goto nla_put_failure;
4462                 if (nla_put_be32(skb, NFTA_SET_DATA_LEN, htonl(set->dlen)))
4463                         goto nla_put_failure;
4464         }
4465         if (set->flags & NFT_SET_OBJECT &&
4466             nla_put_be32(skb, NFTA_SET_OBJ_TYPE, htonl(set->objtype)))
4467                 goto nla_put_failure;
4468
4469         if (timeout &&
4470             nla_put_be64(skb, NFTA_SET_TIMEOUT,
4471                          nf_jiffies64_to_msecs(timeout),
4472                          NFTA_SET_PAD))
4473                 goto nla_put_failure;
4474         if (gc_int &&
4475             nla_put_be32(skb, NFTA_SET_GC_INTERVAL, htonl(gc_int)))
4476                 goto nla_put_failure;
4477
4478         if (set->policy != NFT_SET_POL_PERFORMANCE) {
4479                 if (nla_put_be32(skb, NFTA_SET_POLICY, htonl(set->policy)))
4480                         goto nla_put_failure;
4481         }
4482
4483         if (set->udata &&
4484             nla_put(skb, NFTA_SET_USERDATA, set->udlen, set->udata))
4485                 goto nla_put_failure;
4486
4487         nest = nla_nest_start_noflag(skb, NFTA_SET_DESC);
4488         if (!nest)
4489                 goto nla_put_failure;
4490         if (set->size &&
4491             nla_put_be32(skb, NFTA_SET_DESC_SIZE, htonl(set->size)))
4492                 goto nla_put_failure;
4493
4494         if (set->field_count > 1 &&
4495             nf_tables_fill_set_concat(skb, set))
4496                 goto nla_put_failure;
4497
4498         nla_nest_end(skb, nest);
4499
4500         if (set->num_exprs == 1) {
4501                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPR);
4502                 if (nf_tables_fill_expr_info(skb, set->exprs[0], false) < 0)
4503                         goto nla_put_failure;
4504
4505                 nla_nest_end(skb, nest);
4506         } else if (set->num_exprs > 1) {
4507                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPRESSIONS);
4508                 if (nest == NULL)
4509                         goto nla_put_failure;
4510
4511                 for (i = 0; i < set->num_exprs; i++) {
4512                         if (nft_expr_dump(skb, NFTA_LIST_ELEM,
4513                                           set->exprs[i], false) < 0)
4514                                 goto nla_put_failure;
4515                 }
4516                 nla_nest_end(skb, nest);
4517         }
4518
4519         nlmsg_end(skb, nlh);
4520         return 0;
4521
4522 nla_put_failure:
4523         nlmsg_trim(skb, nlh);
4524         return -1;
4525 }
4526
4527 static void nf_tables_set_notify(const struct nft_ctx *ctx,
4528                                  const struct nft_set *set, int event,
4529                                  gfp_t gfp_flags)
4530 {
4531         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
4532         u32 portid = ctx->portid;
4533         struct sk_buff *skb;
4534         u16 flags = 0;
4535         int err;
4536
4537         if (!ctx->report &&
4538             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
4539                 return;
4540
4541         skb = nlmsg_new(NLMSG_GOODSIZE, gfp_flags);
4542         if (skb == NULL)
4543                 goto err;
4544
4545         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
4546                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
4547
4548         err = nf_tables_fill_set(skb, ctx, set, event, flags);
4549         if (err < 0) {
4550                 kfree_skb(skb);
4551                 goto err;
4552         }
4553
4554         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
4555         return;
4556 err:
4557         nfnetlink_set_err(ctx->net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
4558 }
4559
4560 static int nf_tables_dump_sets(struct sk_buff *skb, struct netlink_callback *cb)
4561 {
4562         const struct nft_set *set;
4563         unsigned int idx, s_idx = cb->args[0];
4564         struct nft_table *table, *cur_table = (struct nft_table *)cb->args[2];
4565         struct net *net = sock_net(skb->sk);
4566         struct nft_ctx *ctx = cb->data, ctx_set;
4567         struct nftables_pernet *nft_net;
4568
4569         if (cb->args[1])
4570                 return skb->len;
4571
4572         rcu_read_lock();
4573         nft_net = nft_pernet(net);
4574         cb->seq = READ_ONCE(nft_net->base_seq);
4575
4576         list_for_each_entry_rcu(table, &nft_net->tables, list) {
4577                 if (ctx->family != NFPROTO_UNSPEC &&
4578                     ctx->family != table->family)
4579                         continue;
4580
4581                 if (ctx->table && ctx->table != table)
4582                         continue;
4583
4584                 if (cur_table) {
4585                         if (cur_table != table)
4586                                 continue;
4587
4588                         cur_table = NULL;
4589                 }
4590                 idx = 0;
4591                 list_for_each_entry_rcu(set, &table->sets, list) {
4592                         if (idx < s_idx)
4593                                 goto cont;
4594                         if (!nft_is_active(net, set))
4595                                 goto cont;
4596
4597                         ctx_set = *ctx;
4598                         ctx_set.table = table;
4599                         ctx_set.family = table->family;
4600
4601                         if (nf_tables_fill_set(skb, &ctx_set, set,
4602                                                NFT_MSG_NEWSET,
4603                                                NLM_F_MULTI) < 0) {
4604                                 cb->args[0] = idx;
4605                                 cb->args[2] = (unsigned long) table;
4606                                 goto done;
4607                         }
4608                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
4609 cont:
4610                         idx++;
4611                 }
4612                 if (s_idx)
4613                         s_idx = 0;
4614         }
4615         cb->args[1] = 1;
4616 done:
4617         rcu_read_unlock();
4618         return skb->len;
4619 }
4620
4621 static int nf_tables_dump_sets_start(struct netlink_callback *cb)
4622 {
4623         struct nft_ctx *ctx_dump = NULL;
4624
4625         ctx_dump = kmemdup(cb->data, sizeof(*ctx_dump), GFP_ATOMIC);
4626         if (ctx_dump == NULL)
4627                 return -ENOMEM;
4628
4629         cb->data = ctx_dump;
4630         return 0;
4631 }
4632
4633 static int nf_tables_dump_sets_done(struct netlink_callback *cb)
4634 {
4635         kfree(cb->data);
4636         return 0;
4637 }
4638
4639 /* called with rcu_read_lock held */
4640 static int nf_tables_getset(struct sk_buff *skb, const struct nfnl_info *info,
4641                             const struct nlattr * const nla[])
4642 {
4643         struct netlink_ext_ack *extack = info->extack;
4644         u8 genmask = nft_genmask_cur(info->net);
4645         u8 family = info->nfmsg->nfgen_family;
4646         struct nft_table *table = NULL;
4647         struct net *net = info->net;
4648         const struct nft_set *set;
4649         struct sk_buff *skb2;
4650         struct nft_ctx ctx;
4651         int err;
4652
4653         if (nla[NFTA_SET_TABLE]) {
4654                 table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
4655                                          genmask, 0);
4656                 if (IS_ERR(table)) {
4657                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
4658                         return PTR_ERR(table);
4659                 }
4660         }
4661
4662         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
4663
4664         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
4665                 struct netlink_dump_control c = {
4666                         .start = nf_tables_dump_sets_start,
4667                         .dump = nf_tables_dump_sets,
4668                         .done = nf_tables_dump_sets_done,
4669                         .data = &ctx,
4670                         .module = THIS_MODULE,
4671                 };
4672
4673                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
4674         }
4675
4676         /* Only accept unspec with dump */
4677         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
4678                 return -EAFNOSUPPORT;
4679         if (!nla[NFTA_SET_TABLE])
4680                 return -EINVAL;
4681
4682         set = nft_set_lookup(table, nla[NFTA_SET_NAME], genmask);
4683         if (IS_ERR(set))
4684                 return PTR_ERR(set);
4685
4686         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
4687         if (skb2 == NULL)
4688                 return -ENOMEM;
4689
4690         err = nf_tables_fill_set(skb2, &ctx, set, NFT_MSG_NEWSET, 0);
4691         if (err < 0)
4692                 goto err_fill_set_info;
4693
4694         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
4695
4696 err_fill_set_info:
4697         kfree_skb(skb2);
4698         return err;
4699 }
4700
4701 static const struct nla_policy nft_concat_policy[NFTA_SET_FIELD_MAX + 1] = {
4702         [NFTA_SET_FIELD_LEN]    = { .type = NLA_U32 },
4703 };
4704
4705 static int nft_set_desc_concat_parse(const struct nlattr *attr,
4706                                      struct nft_set_desc *desc)
4707 {
4708         struct nlattr *tb[NFTA_SET_FIELD_MAX + 1];
4709         u32 len;
4710         int err;
4711
4712         if (desc->field_count >= ARRAY_SIZE(desc->field_len))
4713                 return -E2BIG;
4714
4715         err = nla_parse_nested_deprecated(tb, NFTA_SET_FIELD_MAX, attr,
4716                                           nft_concat_policy, NULL);
4717         if (err < 0)
4718                 return err;
4719
4720         if (!tb[NFTA_SET_FIELD_LEN])
4721                 return -EINVAL;
4722
4723         len = ntohl(nla_get_be32(tb[NFTA_SET_FIELD_LEN]));
4724         if (!len || len > U8_MAX)
4725                 return -EINVAL;
4726
4727         desc->field_len[desc->field_count++] = len;
4728
4729         return 0;
4730 }
4731
4732 static int nft_set_desc_concat(struct nft_set_desc *desc,
4733                                const struct nlattr *nla)
4734 {
4735         struct nlattr *attr;
4736         u32 num_regs = 0;
4737         int rem, err, i;
4738
4739         nla_for_each_nested(attr, nla, rem) {
4740                 if (nla_type(attr) != NFTA_LIST_ELEM)
4741                         return -EINVAL;
4742
4743                 err = nft_set_desc_concat_parse(attr, desc);
4744                 if (err < 0)
4745                         return err;
4746         }
4747
4748         for (i = 0; i < desc->field_count; i++)
4749                 num_regs += DIV_ROUND_UP(desc->field_len[i], sizeof(u32));
4750
4751         if (num_regs > NFT_REG32_COUNT)
4752                 return -E2BIG;
4753
4754         return 0;
4755 }
4756
4757 static int nf_tables_set_desc_parse(struct nft_set_desc *desc,
4758                                     const struct nlattr *nla)
4759 {
4760         struct nlattr *da[NFTA_SET_DESC_MAX + 1];
4761         int err;
4762
4763         err = nla_parse_nested_deprecated(da, NFTA_SET_DESC_MAX, nla,
4764                                           nft_set_desc_policy, NULL);
4765         if (err < 0)
4766                 return err;
4767
4768         if (da[NFTA_SET_DESC_SIZE] != NULL)
4769                 desc->size = ntohl(nla_get_be32(da[NFTA_SET_DESC_SIZE]));
4770         if (da[NFTA_SET_DESC_CONCAT])
4771                 err = nft_set_desc_concat(desc, da[NFTA_SET_DESC_CONCAT]);
4772
4773         return err;
4774 }
4775
4776 static int nft_set_expr_alloc(struct nft_ctx *ctx, struct nft_set *set,
4777                               const struct nlattr * const *nla,
4778                               struct nft_expr **exprs, int *num_exprs,
4779                               u32 flags)
4780 {
4781         struct nft_expr *expr;
4782         int err, i;
4783
4784         if (nla[NFTA_SET_EXPR]) {
4785                 expr = nft_set_elem_expr_alloc(ctx, set, nla[NFTA_SET_EXPR]);
4786                 if (IS_ERR(expr)) {
4787                         err = PTR_ERR(expr);
4788                         goto err_set_expr_alloc;
4789                 }
4790                 exprs[0] = expr;
4791                 (*num_exprs)++;
4792         } else if (nla[NFTA_SET_EXPRESSIONS]) {
4793                 struct nlattr *tmp;
4794                 int left;
4795
4796                 if (!(flags & NFT_SET_EXPR)) {
4797                         err = -EINVAL;
4798                         goto err_set_expr_alloc;
4799                 }
4800                 i = 0;
4801                 nla_for_each_nested(tmp, nla[NFTA_SET_EXPRESSIONS], left) {
4802                         if (i == NFT_SET_EXPR_MAX) {
4803                                 err = -E2BIG;
4804                                 goto err_set_expr_alloc;
4805                         }
4806                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
4807                                 err = -EINVAL;
4808                                 goto err_set_expr_alloc;
4809                         }
4810                         expr = nft_set_elem_expr_alloc(ctx, set, tmp);
4811                         if (IS_ERR(expr)) {
4812                                 err = PTR_ERR(expr);
4813                                 goto err_set_expr_alloc;
4814                         }
4815                         exprs[i++] = expr;
4816                         (*num_exprs)++;
4817                 }
4818         }
4819
4820         return 0;
4821
4822 err_set_expr_alloc:
4823         for (i = 0; i < *num_exprs; i++)
4824                 nft_expr_destroy(ctx, exprs[i]);
4825
4826         return err;
4827 }
4828
4829 static bool nft_set_is_same(const struct nft_set *set,
4830                             const struct nft_set_desc *desc,
4831                             struct nft_expr *exprs[], u32 num_exprs, u32 flags)
4832 {
4833         int i;
4834
4835         if (set->ktype != desc->ktype ||
4836             set->dtype != desc->dtype ||
4837             set->flags != flags ||
4838             set->klen != desc->klen ||
4839             set->dlen != desc->dlen ||
4840             set->field_count != desc->field_count ||
4841             set->num_exprs != num_exprs)
4842                 return false;
4843
4844         for (i = 0; i < desc->field_count; i++) {
4845                 if (set->field_len[i] != desc->field_len[i])
4846                         return false;
4847         }
4848
4849         for (i = 0; i < num_exprs; i++) {
4850                 if (set->exprs[i]->ops != exprs[i]->ops)
4851                         return false;
4852         }
4853
4854         return true;
4855 }
4856
4857 static int nf_tables_newset(struct sk_buff *skb, const struct nfnl_info *info,
4858                             const struct nlattr * const nla[])
4859 {
4860         struct netlink_ext_ack *extack = info->extack;
4861         u8 genmask = nft_genmask_next(info->net);
4862         u8 family = info->nfmsg->nfgen_family;
4863         const struct nft_set_ops *ops;
4864         struct net *net = info->net;
4865         struct nft_set_desc desc;
4866         struct nft_table *table;
4867         unsigned char *udata;
4868         struct nft_set *set;
4869         struct nft_ctx ctx;
4870         size_t alloc_size;
4871         int num_exprs = 0;
4872         char *name;
4873         int err, i;
4874         u16 udlen;
4875         u32 flags;
4876         u64 size;
4877
4878         if (nla[NFTA_SET_TABLE] == NULL ||
4879             nla[NFTA_SET_NAME] == NULL ||
4880             nla[NFTA_SET_KEY_LEN] == NULL ||
4881             nla[NFTA_SET_ID] == NULL)
4882                 return -EINVAL;
4883
4884         memset(&desc, 0, sizeof(desc));
4885
4886         desc.ktype = NFT_DATA_VALUE;
4887         if (nla[NFTA_SET_KEY_TYPE] != NULL) {
4888                 desc.ktype = ntohl(nla_get_be32(nla[NFTA_SET_KEY_TYPE]));
4889                 if ((desc.ktype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK)
4890                         return -EINVAL;
4891         }
4892
4893         desc.klen = ntohl(nla_get_be32(nla[NFTA_SET_KEY_LEN]));
4894         if (desc.klen == 0 || desc.klen > NFT_DATA_VALUE_MAXLEN)
4895                 return -EINVAL;
4896
4897         flags = 0;
4898         if (nla[NFTA_SET_FLAGS] != NULL) {
4899                 flags = ntohl(nla_get_be32(nla[NFTA_SET_FLAGS]));
4900                 if (flags & ~(NFT_SET_ANONYMOUS | NFT_SET_CONSTANT |
4901                               NFT_SET_INTERVAL | NFT_SET_TIMEOUT |
4902                               NFT_SET_MAP | NFT_SET_EVAL |
4903                               NFT_SET_OBJECT | NFT_SET_CONCAT | NFT_SET_EXPR))
4904                         return -EOPNOTSUPP;
4905                 /* Only one of these operations is supported */
4906                 if ((flags & (NFT_SET_MAP | NFT_SET_OBJECT)) ==
4907                              (NFT_SET_MAP | NFT_SET_OBJECT))
4908                         return -EOPNOTSUPP;
4909                 if ((flags & (NFT_SET_EVAL | NFT_SET_OBJECT)) ==
4910                              (NFT_SET_EVAL | NFT_SET_OBJECT))
4911                         return -EOPNOTSUPP;
4912         }
4913
4914         desc.dtype = 0;
4915         if (nla[NFTA_SET_DATA_TYPE] != NULL) {
4916                 if (!(flags & NFT_SET_MAP))
4917                         return -EINVAL;
4918
4919                 desc.dtype = ntohl(nla_get_be32(nla[NFTA_SET_DATA_TYPE]));
4920                 if ((desc.dtype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK &&
4921                     desc.dtype != NFT_DATA_VERDICT)
4922                         return -EINVAL;
4923
4924                 if (desc.dtype != NFT_DATA_VERDICT) {
4925                         if (nla[NFTA_SET_DATA_LEN] == NULL)
4926                                 return -EINVAL;
4927                         desc.dlen = ntohl(nla_get_be32(nla[NFTA_SET_DATA_LEN]));
4928                         if (desc.dlen == 0 || desc.dlen > NFT_DATA_VALUE_MAXLEN)
4929                                 return -EINVAL;
4930                 } else
4931                         desc.dlen = sizeof(struct nft_verdict);
4932         } else if (flags & NFT_SET_MAP)
4933                 return -EINVAL;
4934
4935         if (nla[NFTA_SET_OBJ_TYPE] != NULL) {
4936                 if (!(flags & NFT_SET_OBJECT))
4937                         return -EINVAL;
4938
4939                 desc.objtype = ntohl(nla_get_be32(nla[NFTA_SET_OBJ_TYPE]));
4940                 if (desc.objtype == NFT_OBJECT_UNSPEC ||
4941                     desc.objtype > NFT_OBJECT_MAX)
4942                         return -EOPNOTSUPP;
4943         } else if (flags & NFT_SET_OBJECT)
4944                 return -EINVAL;
4945         else
4946                 desc.objtype = NFT_OBJECT_UNSPEC;
4947
4948         desc.timeout = 0;
4949         if (nla[NFTA_SET_TIMEOUT] != NULL) {
4950                 if (!(flags & NFT_SET_TIMEOUT))
4951                         return -EINVAL;
4952
4953                 if (flags & NFT_SET_ANONYMOUS)
4954                         return -EOPNOTSUPP;
4955
4956                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_TIMEOUT], &desc.timeout);
4957                 if (err)
4958                         return err;
4959         }
4960         desc.gc_int = 0;
4961         if (nla[NFTA_SET_GC_INTERVAL] != NULL) {
4962                 if (!(flags & NFT_SET_TIMEOUT))
4963                         return -EINVAL;
4964
4965                 if (flags & NFT_SET_ANONYMOUS)
4966                         return -EOPNOTSUPP;
4967
4968                 desc.gc_int = ntohl(nla_get_be32(nla[NFTA_SET_GC_INTERVAL]));
4969         }
4970
4971         desc.policy = NFT_SET_POL_PERFORMANCE;
4972         if (nla[NFTA_SET_POLICY] != NULL)
4973                 desc.policy = ntohl(nla_get_be32(nla[NFTA_SET_POLICY]));
4974
4975         if (nla[NFTA_SET_DESC] != NULL) {
4976                 err = nf_tables_set_desc_parse(&desc, nla[NFTA_SET_DESC]);
4977                 if (err < 0)
4978                         return err;
4979
4980                 if (desc.field_count > 1 && !(flags & NFT_SET_CONCAT))
4981                         return -EINVAL;
4982         } else if (flags & NFT_SET_CONCAT) {
4983                 return -EINVAL;
4984         }
4985
4986         if (nla[NFTA_SET_EXPR] || nla[NFTA_SET_EXPRESSIONS])
4987                 desc.expr = true;
4988
4989         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family, genmask,
4990                                  NETLINK_CB(skb).portid);
4991         if (IS_ERR(table)) {
4992                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
4993                 return PTR_ERR(table);
4994         }
4995
4996         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
4997
4998         set = nft_set_lookup(table, nla[NFTA_SET_NAME], genmask);
4999         if (IS_ERR(set)) {
5000                 if (PTR_ERR(set) != -ENOENT) {
5001                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
5002                         return PTR_ERR(set);
5003                 }
5004         } else {
5005                 struct nft_expr *exprs[NFT_SET_EXPR_MAX] = {};
5006
5007                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
5008                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
5009                         return -EEXIST;
5010                 }
5011                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
5012                         return -EOPNOTSUPP;
5013
5014                 if (nft_set_is_anonymous(set))
5015                         return -EOPNOTSUPP;
5016
5017                 err = nft_set_expr_alloc(&ctx, set, nla, exprs, &num_exprs, flags);
5018                 if (err < 0)
5019                         return err;
5020
5021                 err = 0;
5022                 if (!nft_set_is_same(set, &desc, exprs, num_exprs, flags)) {
5023                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
5024                         err = -EEXIST;
5025                 }
5026
5027                 for (i = 0; i < num_exprs; i++)
5028                         nft_expr_destroy(&ctx, exprs[i]);
5029
5030                 if (err < 0)
5031                         return err;
5032
5033                 return __nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set, &desc);
5034         }
5035
5036         if (!(info->nlh->nlmsg_flags & NLM_F_CREATE))
5037                 return -ENOENT;
5038
5039         ops = nft_select_set_ops(&ctx, nla, &desc);
5040         if (IS_ERR(ops))
5041                 return PTR_ERR(ops);
5042
5043         udlen = 0;
5044         if (nla[NFTA_SET_USERDATA])
5045                 udlen = nla_len(nla[NFTA_SET_USERDATA]);
5046
5047         size = 0;
5048         if (ops->privsize != NULL)
5049                 size = ops->privsize(nla, &desc);
5050         alloc_size = sizeof(*set) + size + udlen;
5051         if (alloc_size < size || alloc_size > INT_MAX)
5052                 return -ENOMEM;
5053
5054         if (!nft_use_inc(&table->use))
5055                 return -EMFILE;
5056
5057         set = kvzalloc(alloc_size, GFP_KERNEL_ACCOUNT);
5058         if (!set) {
5059                 err = -ENOMEM;
5060                 goto err_alloc;
5061         }
5062
5063         name = nla_strdup(nla[NFTA_SET_NAME], GFP_KERNEL_ACCOUNT);
5064         if (!name) {
5065                 err = -ENOMEM;
5066                 goto err_set_name;
5067         }
5068
5069         err = nf_tables_set_alloc_name(&ctx, set, name);
5070         kfree(name);
5071         if (err < 0)
5072                 goto err_set_name;
5073
5074         udata = NULL;
5075         if (udlen) {
5076                 udata = set->data + size;
5077                 nla_memcpy(udata, nla[NFTA_SET_USERDATA], udlen);
5078         }
5079
5080         INIT_LIST_HEAD(&set->bindings);
5081         INIT_LIST_HEAD(&set->catchall_list);
5082         refcount_set(&set->refs, 1);
5083         set->table = table;
5084         write_pnet(&set->net, net);
5085         set->ops = ops;
5086         set->ktype = desc.ktype;
5087         set->klen = desc.klen;
5088         set->dtype = desc.dtype;
5089         set->objtype = desc.objtype;
5090         set->dlen = desc.dlen;
5091         set->flags = flags;
5092         set->size = desc.size;
5093         set->policy = desc.policy;
5094         set->udlen = udlen;
5095         set->udata = udata;
5096         set->timeout = desc.timeout;
5097         set->gc_int = desc.gc_int;
5098
5099         set->field_count = desc.field_count;
5100         for (i = 0; i < desc.field_count; i++)
5101                 set->field_len[i] = desc.field_len[i];
5102
5103         err = ops->init(set, &desc, nla);
5104         if (err < 0)
5105                 goto err_set_init;
5106
5107         err = nft_set_expr_alloc(&ctx, set, nla, set->exprs, &num_exprs, flags);
5108         if (err < 0)
5109                 goto err_set_destroy;
5110
5111         set->num_exprs = num_exprs;
5112         set->handle = nf_tables_alloc_handle(table);
5113         INIT_LIST_HEAD(&set->pending_update);
5114
5115         err = nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set);
5116         if (err < 0)
5117                 goto err_set_expr_alloc;
5118
5119         list_add_tail_rcu(&set->list, &table->sets);
5120
5121         return 0;
5122
5123 err_set_expr_alloc:
5124         for (i = 0; i < set->num_exprs; i++)
5125                 nft_expr_destroy(&ctx, set->exprs[i]);
5126 err_set_destroy:
5127         ops->destroy(&ctx, set);
5128 err_set_init:
5129         kfree(set->name);
5130 err_set_name:
5131         kvfree(set);
5132 err_alloc:
5133         nft_use_dec_restore(&table->use);
5134
5135         return err;
5136 }
5137
5138 static void nft_set_catchall_destroy(const struct nft_ctx *ctx,
5139                                      struct nft_set *set)
5140 {
5141         struct nft_set_elem_catchall *next, *catchall;
5142
5143         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
5144                 list_del_rcu(&catchall->list);
5145                 nf_tables_set_elem_destroy(ctx, set, catchall->elem);
5146                 kfree_rcu(catchall, rcu);
5147         }
5148 }
5149
5150 static void nft_set_put(struct nft_set *set)
5151 {
5152         if (refcount_dec_and_test(&set->refs)) {
5153                 kfree(set->name);
5154                 kvfree(set);
5155         }
5156 }
5157
5158 static void nft_set_destroy(const struct nft_ctx *ctx, struct nft_set *set)
5159 {
5160         int i;
5161
5162         if (WARN_ON(set->use > 0))
5163                 return;
5164
5165         for (i = 0; i < set->num_exprs; i++)
5166                 nft_expr_destroy(ctx, set->exprs[i]);
5167
5168         set->ops->destroy(ctx, set);
5169         nft_set_catchall_destroy(ctx, set);
5170         nft_set_put(set);
5171 }
5172
5173 static int nf_tables_delset(struct sk_buff *skb, const struct nfnl_info *info,
5174                             const struct nlattr * const nla[])
5175 {
5176         struct netlink_ext_ack *extack = info->extack;
5177         u8 genmask = nft_genmask_next(info->net);
5178         u8 family = info->nfmsg->nfgen_family;
5179         struct net *net = info->net;
5180         const struct nlattr *attr;
5181         struct nft_table *table;
5182         struct nft_set *set;
5183         struct nft_ctx ctx;
5184
5185         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
5186                 return -EAFNOSUPPORT;
5187
5188         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
5189                                  genmask, NETLINK_CB(skb).portid);
5190         if (IS_ERR(table)) {
5191                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
5192                 return PTR_ERR(table);
5193         }
5194
5195         if (nla[NFTA_SET_HANDLE]) {
5196                 attr = nla[NFTA_SET_HANDLE];
5197                 set = nft_set_lookup_byhandle(table, attr, genmask);
5198         } else {
5199                 attr = nla[NFTA_SET_NAME];
5200                 set = nft_set_lookup(table, attr, genmask);
5201         }
5202
5203         if (IS_ERR(set)) {
5204                 if (PTR_ERR(set) == -ENOENT &&
5205                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYSET)
5206                         return 0;
5207
5208                 NL_SET_BAD_ATTR(extack, attr);
5209                 return PTR_ERR(set);
5210         }
5211         if (set->use ||
5212             (info->nlh->nlmsg_flags & NLM_F_NONREC &&
5213              atomic_read(&set->nelems) > 0)) {
5214                 NL_SET_BAD_ATTR(extack, attr);
5215                 return -EBUSY;
5216         }
5217
5218         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
5219
5220         return nft_delset(&ctx, set);
5221 }
5222
5223 static int nft_validate_register_store(const struct nft_ctx *ctx,
5224                                        enum nft_registers reg,
5225                                        const struct nft_data *data,
5226                                        enum nft_data_types type,
5227                                        unsigned int len);
5228
5229 static int nft_setelem_data_validate(const struct nft_ctx *ctx,
5230                                      struct nft_set *set,
5231                                      struct nft_set_elem *elem)
5232 {
5233         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
5234         enum nft_registers dreg;
5235
5236         dreg = nft_type_to_reg(set->dtype);
5237         return nft_validate_register_store(ctx, dreg, nft_set_ext_data(ext),
5238                                            set->dtype == NFT_DATA_VERDICT ?
5239                                            NFT_DATA_VERDICT : NFT_DATA_VALUE,
5240                                            set->dlen);
5241 }
5242
5243 static int nf_tables_bind_check_setelem(const struct nft_ctx *ctx,
5244                                         struct nft_set *set,
5245                                         const struct nft_set_iter *iter,
5246                                         struct nft_set_elem *elem)
5247 {
5248         return nft_setelem_data_validate(ctx, set, elem);
5249 }
5250
5251 static int nft_set_catchall_bind_check(const struct nft_ctx *ctx,
5252                                        struct nft_set *set)
5253 {
5254         u8 genmask = nft_genmask_next(ctx->net);
5255         struct nft_set_elem_catchall *catchall;
5256         struct nft_set_elem elem;
5257         struct nft_set_ext *ext;
5258         int ret = 0;
5259
5260         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5261                 ext = nft_set_elem_ext(set, catchall->elem);
5262                 if (!nft_set_elem_active(ext, genmask))
5263                         continue;
5264
5265                 elem.priv = catchall->elem;
5266                 ret = nft_setelem_data_validate(ctx, set, &elem);
5267                 if (ret < 0)
5268                         break;
5269         }
5270
5271         return ret;
5272 }
5273
5274 int nf_tables_bind_set(const struct nft_ctx *ctx, struct nft_set *set,
5275                        struct nft_set_binding *binding)
5276 {
5277         struct nft_set_binding *i;
5278         struct nft_set_iter iter;
5279
5280         if (!list_empty(&set->bindings) && nft_set_is_anonymous(set))
5281                 return -EBUSY;
5282
5283         if (binding->flags & NFT_SET_MAP) {
5284                 /* If the set is already bound to the same chain all
5285                  * jumps are already validated for that chain.
5286                  */
5287                 list_for_each_entry(i, &set->bindings, list) {
5288                         if (i->flags & NFT_SET_MAP &&
5289                             i->chain == binding->chain)
5290                                 goto bind;
5291                 }
5292
5293                 iter.genmask    = nft_genmask_next(ctx->net);
5294                 iter.skip       = 0;
5295                 iter.count      = 0;
5296                 iter.err        = 0;
5297                 iter.fn         = nf_tables_bind_check_setelem;
5298
5299                 set->ops->walk(ctx, set, &iter);
5300                 if (!iter.err)
5301                         iter.err = nft_set_catchall_bind_check(ctx, set);
5302
5303                 if (iter.err < 0)
5304                         return iter.err;
5305         }
5306 bind:
5307         if (!nft_use_inc(&set->use))
5308                 return -EMFILE;
5309
5310         binding->chain = ctx->chain;
5311         list_add_tail_rcu(&binding->list, &set->bindings);
5312         nft_set_trans_bind(ctx, set);
5313
5314         return 0;
5315 }
5316 EXPORT_SYMBOL_GPL(nf_tables_bind_set);
5317
5318 static void nf_tables_unbind_set(const struct nft_ctx *ctx, struct nft_set *set,
5319                                  struct nft_set_binding *binding, bool event)
5320 {
5321         list_del_rcu(&binding->list);
5322
5323         if (list_empty(&set->bindings) && nft_set_is_anonymous(set)) {
5324                 list_del_rcu(&set->list);
5325                 if (event)
5326                         nf_tables_set_notify(ctx, set, NFT_MSG_DELSET,
5327                                              GFP_KERNEL);
5328         }
5329 }
5330
5331 static void nft_setelem_data_activate(const struct net *net,
5332                                       const struct nft_set *set,
5333                                       struct nft_set_elem *elem);
5334
5335 static int nft_mapelem_activate(const struct nft_ctx *ctx,
5336                                 struct nft_set *set,
5337                                 const struct nft_set_iter *iter,
5338                                 struct nft_set_elem *elem)
5339 {
5340         nft_setelem_data_activate(ctx->net, set, elem);
5341
5342         return 0;
5343 }
5344
5345 static void nft_map_catchall_activate(const struct nft_ctx *ctx,
5346                                       struct nft_set *set)
5347 {
5348         u8 genmask = nft_genmask_next(ctx->net);
5349         struct nft_set_elem_catchall *catchall;
5350         struct nft_set_elem elem;
5351         struct nft_set_ext *ext;
5352
5353         list_for_each_entry(catchall, &set->catchall_list, list) {
5354                 ext = nft_set_elem_ext(set, catchall->elem);
5355                 if (!nft_set_elem_active(ext, genmask))
5356                         continue;
5357
5358                 elem.priv = catchall->elem;
5359                 nft_setelem_data_activate(ctx->net, set, &elem);
5360                 break;
5361         }
5362 }
5363
5364 static void nft_map_activate(const struct nft_ctx *ctx, struct nft_set *set)
5365 {
5366         struct nft_set_iter iter = {
5367                 .genmask        = nft_genmask_next(ctx->net),
5368                 .fn             = nft_mapelem_activate,
5369         };
5370
5371         set->ops->walk(ctx, set, &iter);
5372         WARN_ON_ONCE(iter.err);
5373
5374         nft_map_catchall_activate(ctx, set);
5375 }
5376
5377 void nf_tables_activate_set(const struct nft_ctx *ctx, struct nft_set *set)
5378 {
5379         if (nft_set_is_anonymous(set)) {
5380                 if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
5381                         nft_map_activate(ctx, set);
5382
5383                 nft_clear(ctx->net, set);
5384         }
5385
5386         nft_use_inc_restore(&set->use);
5387 }
5388 EXPORT_SYMBOL_GPL(nf_tables_activate_set);
5389
5390 void nf_tables_deactivate_set(const struct nft_ctx *ctx, struct nft_set *set,
5391                               struct nft_set_binding *binding,
5392                               enum nft_trans_phase phase)
5393 {
5394         switch (phase) {
5395         case NFT_TRANS_PREPARE_ERROR:
5396                 nft_set_trans_unbind(ctx, set);
5397                 if (nft_set_is_anonymous(set))
5398                         nft_deactivate_next(ctx->net, set);
5399                 else
5400                         list_del_rcu(&binding->list);
5401
5402                 nft_use_dec(&set->use);
5403                 break;
5404         case NFT_TRANS_PREPARE:
5405                 if (nft_set_is_anonymous(set)) {
5406                         if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
5407                                 nft_map_deactivate(ctx, set);
5408
5409                         nft_deactivate_next(ctx->net, set);
5410                 }
5411                 nft_use_dec(&set->use);
5412                 return;
5413         case NFT_TRANS_ABORT:
5414         case NFT_TRANS_RELEASE:
5415                 if (nft_set_is_anonymous(set) &&
5416                     set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
5417                         nft_map_deactivate(ctx, set);
5418
5419                 nft_use_dec(&set->use);
5420                 fallthrough;
5421         default:
5422                 nf_tables_unbind_set(ctx, set, binding,
5423                                      phase == NFT_TRANS_COMMIT);
5424         }
5425 }
5426 EXPORT_SYMBOL_GPL(nf_tables_deactivate_set);
5427
5428 void nf_tables_destroy_set(const struct nft_ctx *ctx, struct nft_set *set)
5429 {
5430         if (list_empty(&set->bindings) && nft_set_is_anonymous(set))
5431                 nft_set_destroy(ctx, set);
5432 }
5433 EXPORT_SYMBOL_GPL(nf_tables_destroy_set);
5434
5435 const struct nft_set_ext_type nft_set_ext_types[] = {
5436         [NFT_SET_EXT_KEY]               = {
5437                 .align  = __alignof__(u32),
5438         },
5439         [NFT_SET_EXT_DATA]              = {
5440                 .align  = __alignof__(u32),
5441         },
5442         [NFT_SET_EXT_EXPRESSIONS]       = {
5443                 .align  = __alignof__(struct nft_set_elem_expr),
5444         },
5445         [NFT_SET_EXT_OBJREF]            = {
5446                 .len    = sizeof(struct nft_object *),
5447                 .align  = __alignof__(struct nft_object *),
5448         },
5449         [NFT_SET_EXT_FLAGS]             = {
5450                 .len    = sizeof(u8),
5451                 .align  = __alignof__(u8),
5452         },
5453         [NFT_SET_EXT_TIMEOUT]           = {
5454                 .len    = sizeof(u64),
5455                 .align  = __alignof__(u64),
5456         },
5457         [NFT_SET_EXT_EXPIRATION]        = {
5458                 .len    = sizeof(u64),
5459                 .align  = __alignof__(u64),
5460         },
5461         [NFT_SET_EXT_USERDATA]          = {
5462                 .len    = sizeof(struct nft_userdata),
5463                 .align  = __alignof__(struct nft_userdata),
5464         },
5465         [NFT_SET_EXT_KEY_END]           = {
5466                 .align  = __alignof__(u32),
5467         },
5468 };
5469
5470 /*
5471  * Set elements
5472  */
5473
5474 static const struct nla_policy nft_set_elem_policy[NFTA_SET_ELEM_MAX + 1] = {
5475         [NFTA_SET_ELEM_KEY]             = { .type = NLA_NESTED },
5476         [NFTA_SET_ELEM_DATA]            = { .type = NLA_NESTED },
5477         [NFTA_SET_ELEM_FLAGS]           = { .type = NLA_U32 },
5478         [NFTA_SET_ELEM_TIMEOUT]         = { .type = NLA_U64 },
5479         [NFTA_SET_ELEM_EXPIRATION]      = { .type = NLA_U64 },
5480         [NFTA_SET_ELEM_USERDATA]        = { .type = NLA_BINARY,
5481                                             .len = NFT_USERDATA_MAXLEN },
5482         [NFTA_SET_ELEM_EXPR]            = { .type = NLA_NESTED },
5483         [NFTA_SET_ELEM_OBJREF]          = { .type = NLA_STRING,
5484                                             .len = NFT_OBJ_MAXNAMELEN - 1 },
5485         [NFTA_SET_ELEM_KEY_END]         = { .type = NLA_NESTED },
5486         [NFTA_SET_ELEM_EXPRESSIONS]     = { .type = NLA_NESTED },
5487 };
5488
5489 static const struct nla_policy nft_set_elem_list_policy[NFTA_SET_ELEM_LIST_MAX + 1] = {
5490         [NFTA_SET_ELEM_LIST_TABLE]      = { .type = NLA_STRING,
5491                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
5492         [NFTA_SET_ELEM_LIST_SET]        = { .type = NLA_STRING,
5493                                             .len = NFT_SET_MAXNAMELEN - 1 },
5494         [NFTA_SET_ELEM_LIST_ELEMENTS]   = { .type = NLA_NESTED },
5495         [NFTA_SET_ELEM_LIST_SET_ID]     = { .type = NLA_U32 },
5496 };
5497
5498 static int nft_set_elem_expr_dump(struct sk_buff *skb,
5499                                   const struct nft_set *set,
5500                                   const struct nft_set_ext *ext,
5501                                   bool reset)
5502 {
5503         struct nft_set_elem_expr *elem_expr;
5504         u32 size, num_exprs = 0;
5505         struct nft_expr *expr;
5506         struct nlattr *nest;
5507
5508         elem_expr = nft_set_ext_expr(ext);
5509         nft_setelem_expr_foreach(expr, elem_expr, size)
5510                 num_exprs++;
5511
5512         if (num_exprs == 1) {
5513                 expr = nft_setelem_expr_at(elem_expr, 0);
5514                 if (nft_expr_dump(skb, NFTA_SET_ELEM_EXPR, expr, reset) < 0)
5515                         return -1;
5516
5517                 return 0;
5518         } else if (num_exprs > 1) {
5519                 nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_EXPRESSIONS);
5520                 if (nest == NULL)
5521                         goto nla_put_failure;
5522
5523                 nft_setelem_expr_foreach(expr, elem_expr, size) {
5524                         expr = nft_setelem_expr_at(elem_expr, size);
5525                         if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr, reset) < 0)
5526                                 goto nla_put_failure;
5527                 }
5528                 nla_nest_end(skb, nest);
5529         }
5530         return 0;
5531
5532 nla_put_failure:
5533         return -1;
5534 }
5535
5536 static int nf_tables_fill_setelem(struct sk_buff *skb,
5537                                   const struct nft_set *set,
5538                                   const struct nft_set_elem *elem,
5539                                   bool reset)
5540 {
5541         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
5542         unsigned char *b = skb_tail_pointer(skb);
5543         struct nlattr *nest;
5544         u64 timeout = 0;
5545
5546         nest = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
5547         if (nest == NULL)
5548                 goto nla_put_failure;
5549
5550         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
5551             nft_data_dump(skb, NFTA_SET_ELEM_KEY, nft_set_ext_key(ext),
5552                           NFT_DATA_VALUE, set->klen) < 0)
5553                 goto nla_put_failure;
5554
5555         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
5556             nft_data_dump(skb, NFTA_SET_ELEM_KEY_END, nft_set_ext_key_end(ext),
5557                           NFT_DATA_VALUE, set->klen) < 0)
5558                 goto nla_put_failure;
5559
5560         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
5561             nft_data_dump(skb, NFTA_SET_ELEM_DATA, nft_set_ext_data(ext),
5562                           set->dtype == NFT_DATA_VERDICT ? NFT_DATA_VERDICT : NFT_DATA_VALUE,
5563                           set->dlen) < 0)
5564                 goto nla_put_failure;
5565
5566         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS) &&
5567             nft_set_elem_expr_dump(skb, set, ext, reset))
5568                 goto nla_put_failure;
5569
5570         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
5571             nla_put_string(skb, NFTA_SET_ELEM_OBJREF,
5572                            (*nft_set_ext_obj(ext))->key.name) < 0)
5573                 goto nla_put_failure;
5574
5575         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
5576             nla_put_be32(skb, NFTA_SET_ELEM_FLAGS,
5577                          htonl(*nft_set_ext_flags(ext))))
5578                 goto nla_put_failure;
5579
5580         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT)) {
5581                 timeout = *nft_set_ext_timeout(ext);
5582                 if (nla_put_be64(skb, NFTA_SET_ELEM_TIMEOUT,
5583                                  nf_jiffies64_to_msecs(timeout),
5584                                  NFTA_SET_ELEM_PAD))
5585                         goto nla_put_failure;
5586         } else if (set->flags & NFT_SET_TIMEOUT) {
5587                 timeout = READ_ONCE(set->timeout);
5588         }
5589
5590         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
5591                 u64 expires, now = get_jiffies_64();
5592
5593                 expires = *nft_set_ext_expiration(ext);
5594                 if (time_before64(now, expires))
5595                         expires -= now;
5596                 else
5597                         expires = 0;
5598
5599                 if (nla_put_be64(skb, NFTA_SET_ELEM_EXPIRATION,
5600                                  nf_jiffies64_to_msecs(expires),
5601                                  NFTA_SET_ELEM_PAD))
5602                         goto nla_put_failure;
5603
5604                 if (reset)
5605                         *nft_set_ext_expiration(ext) = now + timeout;
5606         }
5607
5608         if (nft_set_ext_exists(ext, NFT_SET_EXT_USERDATA)) {
5609                 struct nft_userdata *udata;
5610
5611                 udata = nft_set_ext_userdata(ext);
5612                 if (nla_put(skb, NFTA_SET_ELEM_USERDATA,
5613                             udata->len + 1, udata->data))
5614                         goto nla_put_failure;
5615         }
5616
5617         nla_nest_end(skb, nest);
5618         return 0;
5619
5620 nla_put_failure:
5621         nlmsg_trim(skb, b);
5622         return -EMSGSIZE;
5623 }
5624
5625 struct nft_set_dump_args {
5626         const struct netlink_callback   *cb;
5627         struct nft_set_iter             iter;
5628         struct sk_buff                  *skb;
5629         bool                            reset;
5630 };
5631
5632 static int nf_tables_dump_setelem(const struct nft_ctx *ctx,
5633                                   struct nft_set *set,
5634                                   const struct nft_set_iter *iter,
5635                                   struct nft_set_elem *elem)
5636 {
5637         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
5638         struct nft_set_dump_args *args;
5639
5640         if (nft_set_elem_expired(ext))
5641                 return 0;
5642
5643         args = container_of(iter, struct nft_set_dump_args, iter);
5644         return nf_tables_fill_setelem(args->skb, set, elem, args->reset);
5645 }
5646
5647 static void audit_log_nft_set_reset(const struct nft_table *table,
5648                                     unsigned int base_seq,
5649                                     unsigned int nentries)
5650 {
5651         char *buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, base_seq);
5652
5653         audit_log_nfcfg(buf, table->family, nentries,
5654                         AUDIT_NFT_OP_SETELEM_RESET, GFP_ATOMIC);
5655         kfree(buf);
5656 }
5657
5658 struct nft_set_dump_ctx {
5659         const struct nft_set    *set;
5660         struct nft_ctx          ctx;
5661 };
5662
5663 static int nft_set_catchall_dump(struct net *net, struct sk_buff *skb,
5664                                  const struct nft_set *set, bool reset,
5665                                  unsigned int base_seq)
5666 {
5667         struct nft_set_elem_catchall *catchall;
5668         u8 genmask = nft_genmask_cur(net);
5669         struct nft_set_elem elem;
5670         struct nft_set_ext *ext;
5671         int ret = 0;
5672
5673         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5674                 ext = nft_set_elem_ext(set, catchall->elem);
5675                 if (!nft_set_elem_active(ext, genmask) ||
5676                     nft_set_elem_expired(ext))
5677                         continue;
5678
5679                 elem.priv = catchall->elem;
5680                 ret = nf_tables_fill_setelem(skb, set, &elem, reset);
5681                 if (reset && !ret)
5682                         audit_log_nft_set_reset(set->table, base_seq, 1);
5683                 break;
5684         }
5685
5686         return ret;
5687 }
5688
5689 static int nf_tables_dump_set(struct sk_buff *skb, struct netlink_callback *cb)
5690 {
5691         struct nft_set_dump_ctx *dump_ctx = cb->data;
5692         struct net *net = sock_net(skb->sk);
5693         struct nftables_pernet *nft_net;
5694         struct nft_table *table;
5695         struct nft_set *set;
5696         struct nft_set_dump_args args;
5697         bool set_found = false;
5698         struct nlmsghdr *nlh;
5699         struct nlattr *nest;
5700         bool reset = false;
5701         u32 portid, seq;
5702         int event;
5703
5704         rcu_read_lock();
5705         nft_net = nft_pernet(net);
5706         cb->seq = READ_ONCE(nft_net->base_seq);
5707
5708         list_for_each_entry_rcu(table, &nft_net->tables, list) {
5709                 if (dump_ctx->ctx.family != NFPROTO_UNSPEC &&
5710                     dump_ctx->ctx.family != table->family)
5711                         continue;
5712
5713                 if (table != dump_ctx->ctx.table)
5714                         continue;
5715
5716                 list_for_each_entry_rcu(set, &table->sets, list) {
5717                         if (set == dump_ctx->set) {
5718                                 set_found = true;
5719                                 break;
5720                         }
5721                 }
5722                 break;
5723         }
5724
5725         if (!set_found) {
5726                 rcu_read_unlock();
5727                 return -ENOENT;
5728         }
5729
5730         event  = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWSETELEM);
5731         portid = NETLINK_CB(cb->skb).portid;
5732         seq    = cb->nlh->nlmsg_seq;
5733
5734         nlh = nfnl_msg_put(skb, portid, seq, event, NLM_F_MULTI,
5735                            table->family, NFNETLINK_V0, nft_base_seq(net));
5736         if (!nlh)
5737                 goto nla_put_failure;
5738
5739         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_TABLE, table->name))
5740                 goto nla_put_failure;
5741         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_SET, set->name))
5742                 goto nla_put_failure;
5743
5744         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
5745         if (nest == NULL)
5746                 goto nla_put_failure;
5747
5748         if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETSETELEM_RESET)
5749                 reset = true;
5750
5751         args.cb                 = cb;
5752         args.skb                = skb;
5753         args.reset              = reset;
5754         args.iter.genmask       = nft_genmask_cur(net);
5755         args.iter.skip          = cb->args[0];
5756         args.iter.count         = 0;
5757         args.iter.err           = 0;
5758         args.iter.fn            = nf_tables_dump_setelem;
5759         set->ops->walk(&dump_ctx->ctx, set, &args.iter);
5760
5761         if (!args.iter.err && args.iter.count == cb->args[0])
5762                 args.iter.err = nft_set_catchall_dump(net, skb, set,
5763                                                       reset, cb->seq);
5764         nla_nest_end(skb, nest);
5765         nlmsg_end(skb, nlh);
5766
5767         if (reset && args.iter.count > args.iter.skip)
5768                 audit_log_nft_set_reset(table, cb->seq,
5769                                         args.iter.count - args.iter.skip);
5770
5771         rcu_read_unlock();
5772
5773         if (args.iter.err && args.iter.err != -EMSGSIZE)
5774                 return args.iter.err;
5775         if (args.iter.count == cb->args[0])
5776                 return 0;
5777
5778         cb->args[0] = args.iter.count;
5779         return skb->len;
5780
5781 nla_put_failure:
5782         rcu_read_unlock();
5783         return -ENOSPC;
5784 }
5785
5786 static int nf_tables_dump_set_start(struct netlink_callback *cb)
5787 {
5788         struct nft_set_dump_ctx *dump_ctx = cb->data;
5789
5790         cb->data = kmemdup(dump_ctx, sizeof(*dump_ctx), GFP_ATOMIC);
5791
5792         return cb->data ? 0 : -ENOMEM;
5793 }
5794
5795 static int nf_tables_dump_set_done(struct netlink_callback *cb)
5796 {
5797         kfree(cb->data);
5798         return 0;
5799 }
5800
5801 static int nf_tables_fill_setelem_info(struct sk_buff *skb,
5802                                        const struct nft_ctx *ctx, u32 seq,
5803                                        u32 portid, int event, u16 flags,
5804                                        const struct nft_set *set,
5805                                        const struct nft_set_elem *elem,
5806                                        bool reset)
5807 {
5808         struct nlmsghdr *nlh;
5809         struct nlattr *nest;
5810         int err;
5811
5812         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
5813         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
5814                            NFNETLINK_V0, nft_base_seq(ctx->net));
5815         if (!nlh)
5816                 goto nla_put_failure;
5817
5818         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
5819                 goto nla_put_failure;
5820         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
5821                 goto nla_put_failure;
5822
5823         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
5824         if (nest == NULL)
5825                 goto nla_put_failure;
5826
5827         err = nf_tables_fill_setelem(skb, set, elem, reset);
5828         if (err < 0)
5829                 goto nla_put_failure;
5830
5831         nla_nest_end(skb, nest);
5832
5833         nlmsg_end(skb, nlh);
5834         return 0;
5835
5836 nla_put_failure:
5837         nlmsg_trim(skb, nlh);
5838         return -1;
5839 }
5840
5841 static int nft_setelem_parse_flags(const struct nft_set *set,
5842                                    const struct nlattr *attr, u32 *flags)
5843 {
5844         if (attr == NULL)
5845                 return 0;
5846
5847         *flags = ntohl(nla_get_be32(attr));
5848         if (*flags & ~(NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
5849                 return -EOPNOTSUPP;
5850         if (!(set->flags & NFT_SET_INTERVAL) &&
5851             *flags & NFT_SET_ELEM_INTERVAL_END)
5852                 return -EINVAL;
5853         if ((*flags & (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL)) ==
5854             (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
5855                 return -EINVAL;
5856
5857         return 0;
5858 }
5859
5860 static int nft_setelem_parse_key(struct nft_ctx *ctx, struct nft_set *set,
5861                                  struct nft_data *key, struct nlattr *attr)
5862 {
5863         struct nft_data_desc desc = {
5864                 .type   = NFT_DATA_VALUE,
5865                 .size   = NFT_DATA_VALUE_MAXLEN,
5866                 .len    = set->klen,
5867         };
5868
5869         return nft_data_init(ctx, key, &desc, attr);
5870 }
5871
5872 static int nft_setelem_parse_data(struct nft_ctx *ctx, struct nft_set *set,
5873                                   struct nft_data_desc *desc,
5874                                   struct nft_data *data,
5875                                   struct nlattr *attr)
5876 {
5877         u32 dtype;
5878
5879         if (set->dtype == NFT_DATA_VERDICT)
5880                 dtype = NFT_DATA_VERDICT;
5881         else
5882                 dtype = NFT_DATA_VALUE;
5883
5884         desc->type = dtype;
5885         desc->size = NFT_DATA_VALUE_MAXLEN;
5886         desc->len = set->dlen;
5887         desc->flags = NFT_DATA_DESC_SETELEM;
5888
5889         return nft_data_init(ctx, data, desc, attr);
5890 }
5891
5892 static void *nft_setelem_catchall_get(const struct net *net,
5893                                       const struct nft_set *set)
5894 {
5895         struct nft_set_elem_catchall *catchall;
5896         u8 genmask = nft_genmask_cur(net);
5897         struct nft_set_ext *ext;
5898         void *priv = NULL;
5899
5900         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5901                 ext = nft_set_elem_ext(set, catchall->elem);
5902                 if (!nft_set_elem_active(ext, genmask) ||
5903                     nft_set_elem_expired(ext))
5904                         continue;
5905
5906                 priv = catchall->elem;
5907                 break;
5908         }
5909
5910         return priv;
5911 }
5912
5913 static int nft_setelem_get(struct nft_ctx *ctx, struct nft_set *set,
5914                            struct nft_set_elem *elem, u32 flags)
5915 {
5916         void *priv;
5917
5918         if (!(flags & NFT_SET_ELEM_CATCHALL)) {
5919                 priv = set->ops->get(ctx->net, set, elem, flags);
5920                 if (IS_ERR(priv))
5921                         return PTR_ERR(priv);
5922         } else {
5923                 priv = nft_setelem_catchall_get(ctx->net, set);
5924                 if (!priv)
5925                         return -ENOENT;
5926         }
5927         elem->priv = priv;
5928
5929         return 0;
5930 }
5931
5932 static int nft_get_set_elem(struct nft_ctx *ctx, struct nft_set *set,
5933                             const struct nlattr *attr, bool reset)
5934 {
5935         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
5936         struct nft_set_elem elem;
5937         struct sk_buff *skb;
5938         uint32_t flags = 0;
5939         int err;
5940
5941         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
5942                                           nft_set_elem_policy, NULL);
5943         if (err < 0)
5944                 return err;
5945
5946         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
5947         if (err < 0)
5948                 return err;
5949
5950         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
5951                 return -EINVAL;
5952
5953         if (nla[NFTA_SET_ELEM_KEY]) {
5954                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
5955                                             nla[NFTA_SET_ELEM_KEY]);
5956                 if (err < 0)
5957                         return err;
5958         }
5959
5960         if (nla[NFTA_SET_ELEM_KEY_END]) {
5961                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
5962                                             nla[NFTA_SET_ELEM_KEY_END]);
5963                 if (err < 0)
5964                         return err;
5965         }
5966
5967         err = nft_setelem_get(ctx, set, &elem, flags);
5968         if (err < 0)
5969                 return err;
5970
5971         err = -ENOMEM;
5972         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_ATOMIC);
5973         if (skb == NULL)
5974                 return err;
5975
5976         err = nf_tables_fill_setelem_info(skb, ctx, ctx->seq, ctx->portid,
5977                                           NFT_MSG_NEWSETELEM, 0, set, &elem,
5978                                           reset);
5979         if (err < 0)
5980                 goto err_fill_setelem;
5981
5982         return nfnetlink_unicast(skb, ctx->net, ctx->portid);
5983
5984 err_fill_setelem:
5985         kfree_skb(skb);
5986         return err;
5987 }
5988
5989 /* called with rcu_read_lock held */
5990 static int nf_tables_getsetelem(struct sk_buff *skb,
5991                                 const struct nfnl_info *info,
5992                                 const struct nlattr * const nla[])
5993 {
5994         struct netlink_ext_ack *extack = info->extack;
5995         u8 genmask = nft_genmask_cur(info->net);
5996         u8 family = info->nfmsg->nfgen_family;
5997         int rem, err = 0, nelems = 0;
5998         struct net *net = info->net;
5999         struct nft_table *table;
6000         struct nft_set *set;
6001         struct nlattr *attr;
6002         struct nft_ctx ctx;
6003         bool reset = false;
6004
6005         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
6006                                  genmask, 0);
6007         if (IS_ERR(table)) {
6008                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
6009                 return PTR_ERR(table);
6010         }
6011
6012         set = nft_set_lookup(table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
6013         if (IS_ERR(set))
6014                 return PTR_ERR(set);
6015
6016         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
6017
6018         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
6019                 struct netlink_dump_control c = {
6020                         .start = nf_tables_dump_set_start,
6021                         .dump = nf_tables_dump_set,
6022                         .done = nf_tables_dump_set_done,
6023                         .module = THIS_MODULE,
6024                 };
6025                 struct nft_set_dump_ctx dump_ctx = {
6026                         .set = set,
6027                         .ctx = ctx,
6028                 };
6029
6030                 c.data = &dump_ctx;
6031                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
6032         }
6033
6034         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
6035                 return -EINVAL;
6036
6037         if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETSETELEM_RESET)
6038                 reset = true;
6039
6040         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
6041                 err = nft_get_set_elem(&ctx, set, attr, reset);
6042                 if (err < 0) {
6043                         NL_SET_BAD_ATTR(extack, attr);
6044                         break;
6045                 }
6046                 nelems++;
6047         }
6048
6049         if (reset)
6050                 audit_log_nft_set_reset(table, nft_pernet(net)->base_seq,
6051                                         nelems);
6052
6053         return err;
6054 }
6055
6056 static void nf_tables_setelem_notify(const struct nft_ctx *ctx,
6057                                      const struct nft_set *set,
6058                                      const struct nft_set_elem *elem,
6059                                      int event)
6060 {
6061         struct nftables_pernet *nft_net;
6062         struct net *net = ctx->net;
6063         u32 portid = ctx->portid;
6064         struct sk_buff *skb;
6065         u16 flags = 0;
6066         int err;
6067
6068         if (!ctx->report && !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
6069                 return;
6070
6071         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
6072         if (skb == NULL)
6073                 goto err;
6074
6075         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
6076                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
6077
6078         err = nf_tables_fill_setelem_info(skb, ctx, 0, portid, event, flags,
6079                                           set, elem, false);
6080         if (err < 0) {
6081                 kfree_skb(skb);
6082                 goto err;
6083         }
6084
6085         nft_net = nft_pernet(net);
6086         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
6087         return;
6088 err:
6089         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
6090 }
6091
6092 static struct nft_trans *nft_trans_elem_alloc(struct nft_ctx *ctx,
6093                                               int msg_type,
6094                                               struct nft_set *set)
6095 {
6096         struct nft_trans *trans;
6097
6098         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_elem));
6099         if (trans == NULL)
6100                 return NULL;
6101
6102         nft_trans_elem_set(trans) = set;
6103         return trans;
6104 }
6105
6106 struct nft_expr *nft_set_elem_expr_alloc(const struct nft_ctx *ctx,
6107                                          const struct nft_set *set,
6108                                          const struct nlattr *attr)
6109 {
6110         struct nft_expr *expr;
6111         int err;
6112
6113         expr = nft_expr_init(ctx, attr);
6114         if (IS_ERR(expr))
6115                 return expr;
6116
6117         err = -EOPNOTSUPP;
6118         if (expr->ops->type->flags & NFT_EXPR_GC) {
6119                 if (set->flags & NFT_SET_TIMEOUT)
6120                         goto err_set_elem_expr;
6121                 if (!set->ops->gc_init)
6122                         goto err_set_elem_expr;
6123                 set->ops->gc_init(set);
6124         }
6125
6126         return expr;
6127
6128 err_set_elem_expr:
6129         nft_expr_destroy(ctx, expr);
6130         return ERR_PTR(err);
6131 }
6132
6133 static int nft_set_ext_check(const struct nft_set_ext_tmpl *tmpl, u8 id, u32 len)
6134 {
6135         len += nft_set_ext_types[id].len;
6136         if (len > tmpl->ext_len[id] ||
6137             len > U8_MAX)
6138                 return -1;
6139
6140         return 0;
6141 }
6142
6143 static int nft_set_ext_memcpy(const struct nft_set_ext_tmpl *tmpl, u8 id,
6144                               void *to, const void *from, u32 len)
6145 {
6146         if (nft_set_ext_check(tmpl, id, len) < 0)
6147                 return -1;
6148
6149         memcpy(to, from, len);
6150
6151         return 0;
6152 }
6153
6154 void *nft_set_elem_init(const struct nft_set *set,
6155                         const struct nft_set_ext_tmpl *tmpl,
6156                         const u32 *key, const u32 *key_end,
6157                         const u32 *data, u64 timeout, u64 expiration, gfp_t gfp)
6158 {
6159         struct nft_set_ext *ext;
6160         void *elem;
6161
6162         elem = kzalloc(set->ops->elemsize + tmpl->len, gfp);
6163         if (elem == NULL)
6164                 return ERR_PTR(-ENOMEM);
6165
6166         ext = nft_set_elem_ext(set, elem);
6167         nft_set_ext_init(ext, tmpl);
6168
6169         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
6170             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY,
6171                                nft_set_ext_key(ext), key, set->klen) < 0)
6172                 goto err_ext_check;
6173
6174         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
6175             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY_END,
6176                                nft_set_ext_key_end(ext), key_end, set->klen) < 0)
6177                 goto err_ext_check;
6178
6179         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
6180             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_DATA,
6181                                nft_set_ext_data(ext), data, set->dlen) < 0)
6182                 goto err_ext_check;
6183
6184         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
6185                 *nft_set_ext_expiration(ext) = get_jiffies_64() + expiration;
6186                 if (expiration == 0)
6187                         *nft_set_ext_expiration(ext) += timeout;
6188         }
6189         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT))
6190                 *nft_set_ext_timeout(ext) = timeout;
6191
6192         return elem;
6193
6194 err_ext_check:
6195         kfree(elem);
6196
6197         return ERR_PTR(-EINVAL);
6198 }
6199
6200 static void __nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
6201                                         struct nft_expr *expr)
6202 {
6203         if (expr->ops->destroy_clone) {
6204                 expr->ops->destroy_clone(ctx, expr);
6205                 module_put(expr->ops->type->owner);
6206         } else {
6207                 nf_tables_expr_destroy(ctx, expr);
6208         }
6209 }
6210
6211 static void nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
6212                                       struct nft_set_elem_expr *elem_expr)
6213 {
6214         struct nft_expr *expr;
6215         u32 size;
6216
6217         nft_setelem_expr_foreach(expr, elem_expr, size)
6218                 __nft_set_elem_expr_destroy(ctx, expr);
6219 }
6220
6221 /* Drop references and destroy. Called from gc, dynset and abort path. */
6222 void nft_set_elem_destroy(const struct nft_set *set, void *elem,
6223                           bool destroy_expr)
6224 {
6225         struct nft_set_ext *ext = nft_set_elem_ext(set, elem);
6226         struct nft_ctx ctx = {
6227                 .net    = read_pnet(&set->net),
6228                 .family = set->table->family,
6229         };
6230
6231         nft_data_release(nft_set_ext_key(ext), NFT_DATA_VALUE);
6232         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
6233                 nft_data_release(nft_set_ext_data(ext), set->dtype);
6234         if (destroy_expr && nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
6235                 nft_set_elem_expr_destroy(&ctx, nft_set_ext_expr(ext));
6236
6237         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
6238                 nft_use_dec(&(*nft_set_ext_obj(ext))->use);
6239         kfree(elem);
6240 }
6241 EXPORT_SYMBOL_GPL(nft_set_elem_destroy);
6242
6243 /* Destroy element. References have been already dropped in the preparation
6244  * path via nft_setelem_data_deactivate().
6245  */
6246 void nf_tables_set_elem_destroy(const struct nft_ctx *ctx,
6247                                 const struct nft_set *set, void *elem)
6248 {
6249         struct nft_set_ext *ext = nft_set_elem_ext(set, elem);
6250
6251         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
6252                 nft_set_elem_expr_destroy(ctx, nft_set_ext_expr(ext));
6253
6254         kfree(elem);
6255 }
6256
6257 int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set,
6258                             struct nft_expr *expr_array[])
6259 {
6260         struct nft_expr *expr;
6261         int err, i, k;
6262
6263         for (i = 0; i < set->num_exprs; i++) {
6264                 expr = kzalloc(set->exprs[i]->ops->size, GFP_KERNEL_ACCOUNT);
6265                 if (!expr)
6266                         goto err_expr;
6267
6268                 err = nft_expr_clone(expr, set->exprs[i]);
6269                 if (err < 0) {
6270                         kfree(expr);
6271                         goto err_expr;
6272                 }
6273                 expr_array[i] = expr;
6274         }
6275
6276         return 0;
6277
6278 err_expr:
6279         for (k = i - 1; k >= 0; k--)
6280                 nft_expr_destroy(ctx, expr_array[k]);
6281
6282         return -ENOMEM;
6283 }
6284
6285 static int nft_set_elem_expr_setup(struct nft_ctx *ctx,
6286                                    const struct nft_set_ext_tmpl *tmpl,
6287                                    const struct nft_set_ext *ext,
6288                                    struct nft_expr *expr_array[],
6289                                    u32 num_exprs)
6290 {
6291         struct nft_set_elem_expr *elem_expr = nft_set_ext_expr(ext);
6292         u32 len = sizeof(struct nft_set_elem_expr);
6293         struct nft_expr *expr;
6294         int i, err;
6295
6296         if (num_exprs == 0)
6297                 return 0;
6298
6299         for (i = 0; i < num_exprs; i++)
6300                 len += expr_array[i]->ops->size;
6301
6302         if (nft_set_ext_check(tmpl, NFT_SET_EXT_EXPRESSIONS, len) < 0)
6303                 return -EINVAL;
6304
6305         for (i = 0; i < num_exprs; i++) {
6306                 expr = nft_setelem_expr_at(elem_expr, elem_expr->size);
6307                 err = nft_expr_clone(expr, expr_array[i]);
6308                 if (err < 0)
6309                         goto err_elem_expr_setup;
6310
6311                 elem_expr->size += expr_array[i]->ops->size;
6312                 nft_expr_destroy(ctx, expr_array[i]);
6313                 expr_array[i] = NULL;
6314         }
6315
6316         return 0;
6317
6318 err_elem_expr_setup:
6319         for (; i < num_exprs; i++) {
6320                 nft_expr_destroy(ctx, expr_array[i]);
6321                 expr_array[i] = NULL;
6322         }
6323
6324         return -ENOMEM;
6325 }
6326
6327 struct nft_set_ext *nft_set_catchall_lookup(const struct net *net,
6328                                             const struct nft_set *set)
6329 {
6330         struct nft_set_elem_catchall *catchall;
6331         u8 genmask = nft_genmask_cur(net);
6332         struct nft_set_ext *ext;
6333
6334         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
6335                 ext = nft_set_elem_ext(set, catchall->elem);
6336                 if (nft_set_elem_active(ext, genmask) &&
6337                     !nft_set_elem_expired(ext) &&
6338                     !nft_set_elem_is_dead(ext))
6339                         return ext;
6340         }
6341
6342         return NULL;
6343 }
6344 EXPORT_SYMBOL_GPL(nft_set_catchall_lookup);
6345
6346 static int nft_setelem_catchall_insert(const struct net *net,
6347                                        struct nft_set *set,
6348                                        const struct nft_set_elem *elem,
6349                                        struct nft_set_ext **pext)
6350 {
6351         struct nft_set_elem_catchall *catchall;
6352         u8 genmask = nft_genmask_next(net);
6353         struct nft_set_ext *ext;
6354
6355         list_for_each_entry(catchall, &set->catchall_list, list) {
6356                 ext = nft_set_elem_ext(set, catchall->elem);
6357                 if (nft_set_elem_active(ext, genmask)) {
6358                         *pext = ext;
6359                         return -EEXIST;
6360                 }
6361         }
6362
6363         catchall = kmalloc(sizeof(*catchall), GFP_KERNEL);
6364         if (!catchall)
6365                 return -ENOMEM;
6366
6367         catchall->elem = elem->priv;
6368         list_add_tail_rcu(&catchall->list, &set->catchall_list);
6369
6370         return 0;
6371 }
6372
6373 static int nft_setelem_insert(const struct net *net,
6374                               struct nft_set *set,
6375                               const struct nft_set_elem *elem,
6376                               struct nft_set_ext **ext, unsigned int flags)
6377 {
6378         int ret;
6379
6380         if (flags & NFT_SET_ELEM_CATCHALL)
6381                 ret = nft_setelem_catchall_insert(net, set, elem, ext);
6382         else
6383                 ret = set->ops->insert(net, set, elem, ext);
6384
6385         return ret;
6386 }
6387
6388 static bool nft_setelem_is_catchall(const struct nft_set *set,
6389                                     const struct nft_set_elem *elem)
6390 {
6391         struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6392
6393         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
6394             *nft_set_ext_flags(ext) & NFT_SET_ELEM_CATCHALL)
6395                 return true;
6396
6397         return false;
6398 }
6399
6400 static void nft_setelem_activate(struct net *net, struct nft_set *set,
6401                                  struct nft_set_elem *elem)
6402 {
6403         struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6404
6405         if (nft_setelem_is_catchall(set, elem)) {
6406                 nft_set_elem_change_active(net, set, ext);
6407         } else {
6408                 set->ops->activate(net, set, elem);
6409         }
6410 }
6411
6412 static int nft_setelem_catchall_deactivate(const struct net *net,
6413                                            struct nft_set *set,
6414                                            struct nft_set_elem *elem)
6415 {
6416         struct nft_set_elem_catchall *catchall;
6417         struct nft_set_ext *ext;
6418
6419         list_for_each_entry(catchall, &set->catchall_list, list) {
6420                 ext = nft_set_elem_ext(set, catchall->elem);
6421                 if (!nft_is_active(net, ext))
6422                         continue;
6423
6424                 kfree(elem->priv);
6425                 elem->priv = catchall->elem;
6426                 nft_set_elem_change_active(net, set, ext);
6427                 return 0;
6428         }
6429
6430         return -ENOENT;
6431 }
6432
6433 static int __nft_setelem_deactivate(const struct net *net,
6434                                     struct nft_set *set,
6435                                     struct nft_set_elem *elem)
6436 {
6437         void *priv;
6438
6439         priv = set->ops->deactivate(net, set, elem);
6440         if (!priv)
6441                 return -ENOENT;
6442
6443         kfree(elem->priv);
6444         elem->priv = priv;
6445         set->ndeact++;
6446
6447         return 0;
6448 }
6449
6450 static int nft_setelem_deactivate(const struct net *net,
6451                                   struct nft_set *set,
6452                                   struct nft_set_elem *elem, u32 flags)
6453 {
6454         int ret;
6455
6456         if (flags & NFT_SET_ELEM_CATCHALL)
6457                 ret = nft_setelem_catchall_deactivate(net, set, elem);
6458         else
6459                 ret = __nft_setelem_deactivate(net, set, elem);
6460
6461         return ret;
6462 }
6463
6464 static void nft_setelem_catchall_remove(const struct net *net,
6465                                         const struct nft_set *set,
6466                                         const struct nft_set_elem *elem)
6467 {
6468         struct nft_set_elem_catchall *catchall, *next;
6469
6470         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
6471                 if (catchall->elem == elem->priv) {
6472                         list_del_rcu(&catchall->list);
6473                         kfree_rcu(catchall, rcu);
6474                         break;
6475                 }
6476         }
6477 }
6478
6479 static void nft_setelem_remove(const struct net *net,
6480                                const struct nft_set *set,
6481                                const struct nft_set_elem *elem)
6482 {
6483         if (nft_setelem_is_catchall(set, elem))
6484                 nft_setelem_catchall_remove(net, set, elem);
6485         else
6486                 set->ops->remove(net, set, elem);
6487 }
6488
6489 static bool nft_setelem_valid_key_end(const struct nft_set *set,
6490                                       struct nlattr **nla, u32 flags)
6491 {
6492         if ((set->flags & (NFT_SET_CONCAT | NFT_SET_INTERVAL)) ==
6493                           (NFT_SET_CONCAT | NFT_SET_INTERVAL)) {
6494                 if (flags & NFT_SET_ELEM_INTERVAL_END)
6495                         return false;
6496
6497                 if (nla[NFTA_SET_ELEM_KEY_END] &&
6498                     flags & NFT_SET_ELEM_CATCHALL)
6499                         return false;
6500         } else {
6501                 if (nla[NFTA_SET_ELEM_KEY_END])
6502                         return false;
6503         }
6504
6505         return true;
6506 }
6507
6508 static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set,
6509                             const struct nlattr *attr, u32 nlmsg_flags)
6510 {
6511         struct nft_expr *expr_array[NFT_SET_EXPR_MAX] = {};
6512         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
6513         u8 genmask = nft_genmask_next(ctx->net);
6514         u32 flags = 0, size = 0, num_exprs = 0;
6515         struct nft_set_ext_tmpl tmpl;
6516         struct nft_set_ext *ext, *ext2;
6517         struct nft_set_elem elem;
6518         struct nft_set_binding *binding;
6519         struct nft_object *obj = NULL;
6520         struct nft_userdata *udata;
6521         struct nft_data_desc desc;
6522         enum nft_registers dreg;
6523         struct nft_trans *trans;
6524         u64 timeout;
6525         u64 expiration;
6526         int err, i;
6527         u8 ulen;
6528
6529         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
6530                                           nft_set_elem_policy, NULL);
6531         if (err < 0)
6532                 return err;
6533
6534         nft_set_ext_prepare(&tmpl);
6535
6536         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
6537         if (err < 0)
6538                 return err;
6539
6540         if (((flags & NFT_SET_ELEM_CATCHALL) && nla[NFTA_SET_ELEM_KEY]) ||
6541             (!(flags & NFT_SET_ELEM_CATCHALL) && !nla[NFTA_SET_ELEM_KEY]))
6542                 return -EINVAL;
6543
6544         if (flags != 0) {
6545                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
6546                 if (err < 0)
6547                         return err;
6548         }
6549
6550         if (set->flags & NFT_SET_MAP) {
6551                 if (nla[NFTA_SET_ELEM_DATA] == NULL &&
6552                     !(flags & NFT_SET_ELEM_INTERVAL_END))
6553                         return -EINVAL;
6554         } else {
6555                 if (nla[NFTA_SET_ELEM_DATA] != NULL)
6556                         return -EINVAL;
6557         }
6558
6559         if (set->flags & NFT_SET_OBJECT) {
6560                 if (!nla[NFTA_SET_ELEM_OBJREF] &&
6561                     !(flags & NFT_SET_ELEM_INTERVAL_END))
6562                         return -EINVAL;
6563         } else {
6564                 if (nla[NFTA_SET_ELEM_OBJREF])
6565                         return -EINVAL;
6566         }
6567
6568         if (!nft_setelem_valid_key_end(set, nla, flags))
6569                 return -EINVAL;
6570
6571         if ((flags & NFT_SET_ELEM_INTERVAL_END) &&
6572              (nla[NFTA_SET_ELEM_DATA] ||
6573               nla[NFTA_SET_ELEM_OBJREF] ||
6574               nla[NFTA_SET_ELEM_TIMEOUT] ||
6575               nla[NFTA_SET_ELEM_EXPIRATION] ||
6576               nla[NFTA_SET_ELEM_USERDATA] ||
6577               nla[NFTA_SET_ELEM_EXPR] ||
6578               nla[NFTA_SET_ELEM_KEY_END] ||
6579               nla[NFTA_SET_ELEM_EXPRESSIONS]))
6580                 return -EINVAL;
6581
6582         timeout = 0;
6583         if (nla[NFTA_SET_ELEM_TIMEOUT] != NULL) {
6584                 if (!(set->flags & NFT_SET_TIMEOUT))
6585                         return -EINVAL;
6586                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_TIMEOUT],
6587                                             &timeout);
6588                 if (err)
6589                         return err;
6590         } else if (set->flags & NFT_SET_TIMEOUT &&
6591                    !(flags & NFT_SET_ELEM_INTERVAL_END)) {
6592                 timeout = READ_ONCE(set->timeout);
6593         }
6594
6595         expiration = 0;
6596         if (nla[NFTA_SET_ELEM_EXPIRATION] != NULL) {
6597                 if (!(set->flags & NFT_SET_TIMEOUT))
6598                         return -EINVAL;
6599                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_EXPIRATION],
6600                                             &expiration);
6601                 if (err)
6602                         return err;
6603         }
6604
6605         if (nla[NFTA_SET_ELEM_EXPR]) {
6606                 struct nft_expr *expr;
6607
6608                 if (set->num_exprs && set->num_exprs != 1)
6609                         return -EOPNOTSUPP;
6610
6611                 expr = nft_set_elem_expr_alloc(ctx, set,
6612                                                nla[NFTA_SET_ELEM_EXPR]);
6613                 if (IS_ERR(expr))
6614                         return PTR_ERR(expr);
6615
6616                 expr_array[0] = expr;
6617                 num_exprs = 1;
6618
6619                 if (set->num_exprs && set->exprs[0]->ops != expr->ops) {
6620                         err = -EOPNOTSUPP;
6621                         goto err_set_elem_expr;
6622                 }
6623         } else if (nla[NFTA_SET_ELEM_EXPRESSIONS]) {
6624                 struct nft_expr *expr;
6625                 struct nlattr *tmp;
6626                 int left;
6627
6628                 i = 0;
6629                 nla_for_each_nested(tmp, nla[NFTA_SET_ELEM_EXPRESSIONS], left) {
6630                         if (i == NFT_SET_EXPR_MAX ||
6631                             (set->num_exprs && set->num_exprs == i)) {
6632                                 err = -E2BIG;
6633                                 goto err_set_elem_expr;
6634                         }
6635                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
6636                                 err = -EINVAL;
6637                                 goto err_set_elem_expr;
6638                         }
6639                         expr = nft_set_elem_expr_alloc(ctx, set, tmp);
6640                         if (IS_ERR(expr)) {
6641                                 err = PTR_ERR(expr);
6642                                 goto err_set_elem_expr;
6643                         }
6644                         expr_array[i] = expr;
6645                         num_exprs++;
6646
6647                         if (set->num_exprs && expr->ops != set->exprs[i]->ops) {
6648                                 err = -EOPNOTSUPP;
6649                                 goto err_set_elem_expr;
6650                         }
6651                         i++;
6652                 }
6653                 if (set->num_exprs && set->num_exprs != i) {
6654                         err = -EOPNOTSUPP;
6655                         goto err_set_elem_expr;
6656                 }
6657         } else if (set->num_exprs > 0 &&
6658                    !(flags & NFT_SET_ELEM_INTERVAL_END)) {
6659                 err = nft_set_elem_expr_clone(ctx, set, expr_array);
6660                 if (err < 0)
6661                         goto err_set_elem_expr_clone;
6662
6663                 num_exprs = set->num_exprs;
6664         }
6665
6666         if (nla[NFTA_SET_ELEM_KEY]) {
6667                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
6668                                             nla[NFTA_SET_ELEM_KEY]);
6669                 if (err < 0)
6670                         goto err_set_elem_expr;
6671
6672                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
6673                 if (err < 0)
6674                         goto err_parse_key;
6675         }
6676
6677         if (nla[NFTA_SET_ELEM_KEY_END]) {
6678                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
6679                                             nla[NFTA_SET_ELEM_KEY_END]);
6680                 if (err < 0)
6681                         goto err_parse_key;
6682
6683                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
6684                 if (err < 0)
6685                         goto err_parse_key_end;
6686         }
6687
6688         if (timeout > 0) {
6689                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_EXPIRATION);
6690                 if (err < 0)
6691                         goto err_parse_key_end;
6692
6693                 if (timeout != READ_ONCE(set->timeout)) {
6694                         err = nft_set_ext_add(&tmpl, NFT_SET_EXT_TIMEOUT);
6695                         if (err < 0)
6696                                 goto err_parse_key_end;
6697                 }
6698         }
6699
6700         if (num_exprs) {
6701                 for (i = 0; i < num_exprs; i++)
6702                         size += expr_array[i]->ops->size;
6703
6704                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_EXPRESSIONS,
6705                                              sizeof(struct nft_set_elem_expr) + size);
6706                 if (err < 0)
6707                         goto err_parse_key_end;
6708         }
6709
6710         if (nla[NFTA_SET_ELEM_OBJREF] != NULL) {
6711                 obj = nft_obj_lookup(ctx->net, ctx->table,
6712                                      nla[NFTA_SET_ELEM_OBJREF],
6713                                      set->objtype, genmask);
6714                 if (IS_ERR(obj)) {
6715                         err = PTR_ERR(obj);
6716                         obj = NULL;
6717                         goto err_parse_key_end;
6718                 }
6719
6720                 if (!nft_use_inc(&obj->use)) {
6721                         err = -EMFILE;
6722                         obj = NULL;
6723                         goto err_parse_key_end;
6724                 }
6725
6726                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_OBJREF);
6727                 if (err < 0)
6728                         goto err_parse_key_end;
6729         }
6730
6731         if (nla[NFTA_SET_ELEM_DATA] != NULL) {
6732                 err = nft_setelem_parse_data(ctx, set, &desc, &elem.data.val,
6733                                              nla[NFTA_SET_ELEM_DATA]);
6734                 if (err < 0)
6735                         goto err_parse_key_end;
6736
6737                 dreg = nft_type_to_reg(set->dtype);
6738                 list_for_each_entry(binding, &set->bindings, list) {
6739                         struct nft_ctx bind_ctx = {
6740                                 .net    = ctx->net,
6741                                 .family = ctx->family,
6742                                 .table  = ctx->table,
6743                                 .chain  = (struct nft_chain *)binding->chain,
6744                         };
6745
6746                         if (!(binding->flags & NFT_SET_MAP))
6747                                 continue;
6748
6749                         err = nft_validate_register_store(&bind_ctx, dreg,
6750                                                           &elem.data.val,
6751                                                           desc.type, desc.len);
6752                         if (err < 0)
6753                                 goto err_parse_data;
6754
6755                         if (desc.type == NFT_DATA_VERDICT &&
6756                             (elem.data.val.verdict.code == NFT_GOTO ||
6757                              elem.data.val.verdict.code == NFT_JUMP))
6758                                 nft_validate_state_update(ctx->table,
6759                                                           NFT_VALIDATE_NEED);
6760                 }
6761
6762                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_DATA, desc.len);
6763                 if (err < 0)
6764                         goto err_parse_data;
6765         }
6766
6767         /* The full maximum length of userdata can exceed the maximum
6768          * offset value (U8_MAX) for following extensions, therefor it
6769          * must be the last extension added.
6770          */
6771         ulen = 0;
6772         if (nla[NFTA_SET_ELEM_USERDATA] != NULL) {
6773                 ulen = nla_len(nla[NFTA_SET_ELEM_USERDATA]);
6774                 if (ulen > 0) {
6775                         err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_USERDATA,
6776                                                      ulen);
6777                         if (err < 0)
6778                                 goto err_parse_data;
6779                 }
6780         }
6781
6782         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
6783                                       elem.key_end.val.data, elem.data.val.data,
6784                                       timeout, expiration, GFP_KERNEL_ACCOUNT);
6785         if (IS_ERR(elem.priv)) {
6786                 err = PTR_ERR(elem.priv);
6787                 goto err_parse_data;
6788         }
6789
6790         ext = nft_set_elem_ext(set, elem.priv);
6791         if (flags)
6792                 *nft_set_ext_flags(ext) = flags;
6793
6794         if (obj)
6795                 *nft_set_ext_obj(ext) = obj;
6796
6797         if (ulen > 0) {
6798                 if (nft_set_ext_check(&tmpl, NFT_SET_EXT_USERDATA, ulen) < 0) {
6799                         err = -EINVAL;
6800                         goto err_elem_free;
6801                 }
6802                 udata = nft_set_ext_userdata(ext);
6803                 udata->len = ulen - 1;
6804                 nla_memcpy(&udata->data, nla[NFTA_SET_ELEM_USERDATA], ulen);
6805         }
6806         err = nft_set_elem_expr_setup(ctx, &tmpl, ext, expr_array, num_exprs);
6807         if (err < 0)
6808                 goto err_elem_free;
6809
6810         trans = nft_trans_elem_alloc(ctx, NFT_MSG_NEWSETELEM, set);
6811         if (trans == NULL) {
6812                 err = -ENOMEM;
6813                 goto err_elem_free;
6814         }
6815
6816         ext->genmask = nft_genmask_cur(ctx->net);
6817
6818         err = nft_setelem_insert(ctx->net, set, &elem, &ext2, flags);
6819         if (err) {
6820                 if (err == -EEXIST) {
6821                         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) ^
6822                             nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) ||
6823                             nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) ^
6824                             nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF))
6825                                 goto err_element_clash;
6826                         if ((nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
6827                              nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) &&
6828                              memcmp(nft_set_ext_data(ext),
6829                                     nft_set_ext_data(ext2), set->dlen) != 0) ||
6830                             (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
6831                              nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF) &&
6832                              *nft_set_ext_obj(ext) != *nft_set_ext_obj(ext2)))
6833                                 goto err_element_clash;
6834                         else if (!(nlmsg_flags & NLM_F_EXCL))
6835                                 err = 0;
6836                 } else if (err == -ENOTEMPTY) {
6837                         /* ENOTEMPTY reports overlapping between this element
6838                          * and an existing one.
6839                          */
6840                         err = -EEXIST;
6841                 }
6842                 goto err_element_clash;
6843         }
6844
6845         if (!(flags & NFT_SET_ELEM_CATCHALL)) {
6846                 unsigned int max = set->size ? set->size + set->ndeact : UINT_MAX;
6847
6848                 if (!atomic_add_unless(&set->nelems, 1, max)) {
6849                         err = -ENFILE;
6850                         goto err_set_full;
6851                 }
6852         }
6853
6854         nft_trans_elem(trans) = elem;
6855         nft_trans_commit_list_add_tail(ctx->net, trans);
6856         return 0;
6857
6858 err_set_full:
6859         nft_setelem_remove(ctx->net, set, &elem);
6860 err_element_clash:
6861         kfree(trans);
6862 err_elem_free:
6863         nf_tables_set_elem_destroy(ctx, set, elem.priv);
6864 err_parse_data:
6865         if (nla[NFTA_SET_ELEM_DATA] != NULL)
6866                 nft_data_release(&elem.data.val, desc.type);
6867 err_parse_key_end:
6868         if (obj)
6869                 nft_use_dec_restore(&obj->use);
6870
6871         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
6872 err_parse_key:
6873         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
6874 err_set_elem_expr:
6875         for (i = 0; i < num_exprs && expr_array[i]; i++)
6876                 nft_expr_destroy(ctx, expr_array[i]);
6877 err_set_elem_expr_clone:
6878         return err;
6879 }
6880
6881 static int nf_tables_newsetelem(struct sk_buff *skb,
6882                                 const struct nfnl_info *info,
6883                                 const struct nlattr * const nla[])
6884 {
6885         struct netlink_ext_ack *extack = info->extack;
6886         u8 genmask = nft_genmask_next(info->net);
6887         u8 family = info->nfmsg->nfgen_family;
6888         struct net *net = info->net;
6889         const struct nlattr *attr;
6890         struct nft_table *table;
6891         struct nft_set *set;
6892         struct nft_ctx ctx;
6893         int rem, err;
6894
6895         if (nla[NFTA_SET_ELEM_LIST_ELEMENTS] == NULL)
6896                 return -EINVAL;
6897
6898         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
6899                                  genmask, NETLINK_CB(skb).portid);
6900         if (IS_ERR(table)) {
6901                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
6902                 return PTR_ERR(table);
6903         }
6904
6905         set = nft_set_lookup_global(net, table, nla[NFTA_SET_ELEM_LIST_SET],
6906                                     nla[NFTA_SET_ELEM_LIST_SET_ID], genmask);
6907         if (IS_ERR(set))
6908                 return PTR_ERR(set);
6909
6910         if (!list_empty(&set->bindings) &&
6911             (set->flags & (NFT_SET_CONSTANT | NFT_SET_ANONYMOUS)))
6912                 return -EBUSY;
6913
6914         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
6915
6916         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
6917                 err = nft_add_set_elem(&ctx, set, attr, info->nlh->nlmsg_flags);
6918                 if (err < 0) {
6919                         NL_SET_BAD_ATTR(extack, attr);
6920                         return err;
6921                 }
6922         }
6923
6924         if (table->validate_state == NFT_VALIDATE_DO)
6925                 return nft_table_validate(net, table);
6926
6927         return 0;
6928 }
6929
6930 /**
6931  *      nft_data_hold - hold a nft_data item
6932  *
6933  *      @data: struct nft_data to release
6934  *      @type: type of data
6935  *
6936  *      Hold a nft_data item. NFT_DATA_VALUE types can be silently discarded,
6937  *      NFT_DATA_VERDICT bumps the reference to chains in case of NFT_JUMP and
6938  *      NFT_GOTO verdicts. This function must be called on active data objects
6939  *      from the second phase of the commit protocol.
6940  */
6941 void nft_data_hold(const struct nft_data *data, enum nft_data_types type)
6942 {
6943         struct nft_chain *chain;
6944
6945         if (type == NFT_DATA_VERDICT) {
6946                 switch (data->verdict.code) {
6947                 case NFT_JUMP:
6948                 case NFT_GOTO:
6949                         chain = data->verdict.chain;
6950                         nft_use_inc_restore(&chain->use);
6951                         break;
6952                 }
6953         }
6954 }
6955
6956 static void nft_setelem_data_activate(const struct net *net,
6957                                       const struct nft_set *set,
6958                                       struct nft_set_elem *elem)
6959 {
6960         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6961
6962         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
6963                 nft_data_hold(nft_set_ext_data(ext), set->dtype);
6964         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
6965                 nft_use_inc_restore(&(*nft_set_ext_obj(ext))->use);
6966 }
6967
6968 void nft_setelem_data_deactivate(const struct net *net,
6969                                  const struct nft_set *set,
6970                                  struct nft_set_elem *elem)
6971 {
6972         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6973
6974         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
6975                 nft_data_release(nft_set_ext_data(ext), set->dtype);
6976         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
6977                 nft_use_dec(&(*nft_set_ext_obj(ext))->use);
6978 }
6979
6980 static int nft_del_setelem(struct nft_ctx *ctx, struct nft_set *set,
6981                            const struct nlattr *attr)
6982 {
6983         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
6984         struct nft_set_ext_tmpl tmpl;
6985         struct nft_set_elem elem;
6986         struct nft_set_ext *ext;
6987         struct nft_trans *trans;
6988         u32 flags = 0;
6989         int err;
6990
6991         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
6992                                           nft_set_elem_policy, NULL);
6993         if (err < 0)
6994                 return err;
6995
6996         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
6997         if (err < 0)
6998                 return err;
6999
7000         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
7001                 return -EINVAL;
7002
7003         if (!nft_setelem_valid_key_end(set, nla, flags))
7004                 return -EINVAL;
7005
7006         nft_set_ext_prepare(&tmpl);
7007
7008         if (flags != 0) {
7009                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
7010                 if (err < 0)
7011                         return err;
7012         }
7013
7014         if (nla[NFTA_SET_ELEM_KEY]) {
7015                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
7016                                             nla[NFTA_SET_ELEM_KEY]);
7017                 if (err < 0)
7018                         return err;
7019
7020                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
7021                 if (err < 0)
7022                         goto fail_elem;
7023         }
7024
7025         if (nla[NFTA_SET_ELEM_KEY_END]) {
7026                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
7027                                             nla[NFTA_SET_ELEM_KEY_END]);
7028                 if (err < 0)
7029                         goto fail_elem;
7030
7031                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
7032                 if (err < 0)
7033                         goto fail_elem_key_end;
7034         }
7035
7036         err = -ENOMEM;
7037         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
7038                                       elem.key_end.val.data, NULL, 0, 0,
7039                                       GFP_KERNEL_ACCOUNT);
7040         if (IS_ERR(elem.priv)) {
7041                 err = PTR_ERR(elem.priv);
7042                 goto fail_elem_key_end;
7043         }
7044
7045         ext = nft_set_elem_ext(set, elem.priv);
7046         if (flags)
7047                 *nft_set_ext_flags(ext) = flags;
7048
7049         trans = nft_trans_elem_alloc(ctx, NFT_MSG_DELSETELEM, set);
7050         if (trans == NULL)
7051                 goto fail_trans;
7052
7053         err = nft_setelem_deactivate(ctx->net, set, &elem, flags);
7054         if (err < 0)
7055                 goto fail_ops;
7056
7057         nft_setelem_data_deactivate(ctx->net, set, &elem);
7058
7059         nft_trans_elem(trans) = elem;
7060         nft_trans_commit_list_add_tail(ctx->net, trans);
7061         return 0;
7062
7063 fail_ops:
7064         kfree(trans);
7065 fail_trans:
7066         kfree(elem.priv);
7067 fail_elem_key_end:
7068         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
7069 fail_elem:
7070         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
7071         return err;
7072 }
7073
7074 static int nft_setelem_flush(const struct nft_ctx *ctx,
7075                              struct nft_set *set,
7076                              const struct nft_set_iter *iter,
7077                              struct nft_set_elem *elem)
7078 {
7079         struct nft_trans *trans;
7080         int err;
7081
7082         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
7083                                     sizeof(struct nft_trans_elem), GFP_ATOMIC);
7084         if (!trans)
7085                 return -ENOMEM;
7086
7087         if (!set->ops->flush(ctx->net, set, elem->priv)) {
7088                 err = -ENOENT;
7089                 goto err1;
7090         }
7091         set->ndeact++;
7092
7093         nft_setelem_data_deactivate(ctx->net, set, elem);
7094         nft_trans_elem_set(trans) = set;
7095         nft_trans_elem(trans) = *elem;
7096         nft_trans_commit_list_add_tail(ctx->net, trans);
7097
7098         return 0;
7099 err1:
7100         kfree(trans);
7101         return err;
7102 }
7103
7104 static int __nft_set_catchall_flush(const struct nft_ctx *ctx,
7105                                     struct nft_set *set,
7106                                     struct nft_set_elem *elem)
7107 {
7108         struct nft_trans *trans;
7109
7110         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
7111                                     sizeof(struct nft_trans_elem), GFP_KERNEL);
7112         if (!trans)
7113                 return -ENOMEM;
7114
7115         nft_setelem_data_deactivate(ctx->net, set, elem);
7116         nft_trans_elem_set(trans) = set;
7117         nft_trans_elem(trans) = *elem;
7118         nft_trans_commit_list_add_tail(ctx->net, trans);
7119
7120         return 0;
7121 }
7122
7123 static int nft_set_catchall_flush(const struct nft_ctx *ctx,
7124                                   struct nft_set *set)
7125 {
7126         u8 genmask = nft_genmask_next(ctx->net);
7127         struct nft_set_elem_catchall *catchall;
7128         struct nft_set_elem elem;
7129         struct nft_set_ext *ext;
7130         int ret = 0;
7131
7132         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
7133                 ext = nft_set_elem_ext(set, catchall->elem);
7134                 if (!nft_set_elem_active(ext, genmask))
7135                         continue;
7136
7137                 elem.priv = catchall->elem;
7138                 ret = __nft_set_catchall_flush(ctx, set, &elem);
7139                 if (ret < 0)
7140                         break;
7141                 nft_set_elem_change_active(ctx->net, set, ext);
7142         }
7143
7144         return ret;
7145 }
7146
7147 static int nft_set_flush(struct nft_ctx *ctx, struct nft_set *set, u8 genmask)
7148 {
7149         struct nft_set_iter iter = {
7150                 .genmask        = genmask,
7151                 .fn             = nft_setelem_flush,
7152         };
7153
7154         set->ops->walk(ctx, set, &iter);
7155         if (!iter.err)
7156                 iter.err = nft_set_catchall_flush(ctx, set);
7157
7158         return iter.err;
7159 }
7160
7161 static int nf_tables_delsetelem(struct sk_buff *skb,
7162                                 const struct nfnl_info *info,
7163                                 const struct nlattr * const nla[])
7164 {
7165         struct netlink_ext_ack *extack = info->extack;
7166         u8 genmask = nft_genmask_next(info->net);
7167         u8 family = info->nfmsg->nfgen_family;
7168         struct net *net = info->net;
7169         const struct nlattr *attr;
7170         struct nft_table *table;
7171         struct nft_set *set;
7172         struct nft_ctx ctx;
7173         int rem, err = 0;
7174
7175         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
7176                                  genmask, NETLINK_CB(skb).portid);
7177         if (IS_ERR(table)) {
7178                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
7179                 return PTR_ERR(table);
7180         }
7181
7182         set = nft_set_lookup(table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
7183         if (IS_ERR(set))
7184                 return PTR_ERR(set);
7185
7186         if (!list_empty(&set->bindings) &&
7187             (set->flags & (NFT_SET_CONSTANT | NFT_SET_ANONYMOUS)))
7188                 return -EBUSY;
7189
7190         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7191
7192         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
7193                 return nft_set_flush(&ctx, set, genmask);
7194
7195         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
7196                 err = nft_del_setelem(&ctx, set, attr);
7197                 if (err == -ENOENT &&
7198                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYSETELEM)
7199                         continue;
7200
7201                 if (err < 0) {
7202                         NL_SET_BAD_ATTR(extack, attr);
7203                         break;
7204                 }
7205         }
7206         return err;
7207 }
7208
7209 /*
7210  * Stateful objects
7211  */
7212
7213 /**
7214  *      nft_register_obj- register nf_tables stateful object type
7215  *      @obj_type: object type
7216  *
7217  *      Registers the object type for use with nf_tables. Returns zero on
7218  *      success or a negative errno code otherwise.
7219  */
7220 int nft_register_obj(struct nft_object_type *obj_type)
7221 {
7222         if (obj_type->type == NFT_OBJECT_UNSPEC)
7223                 return -EINVAL;
7224
7225         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7226         list_add_rcu(&obj_type->list, &nf_tables_objects);
7227         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7228         return 0;
7229 }
7230 EXPORT_SYMBOL_GPL(nft_register_obj);
7231
7232 /**
7233  *      nft_unregister_obj - unregister nf_tables object type
7234  *      @obj_type: object type
7235  *
7236  *      Unregisters the object type for use with nf_tables.
7237  */
7238 void nft_unregister_obj(struct nft_object_type *obj_type)
7239 {
7240         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7241         list_del_rcu(&obj_type->list);
7242         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7243 }
7244 EXPORT_SYMBOL_GPL(nft_unregister_obj);
7245
7246 struct nft_object *nft_obj_lookup(const struct net *net,
7247                                   const struct nft_table *table,
7248                                   const struct nlattr *nla, u32 objtype,
7249                                   u8 genmask)
7250 {
7251         struct nft_object_hash_key k = { .table = table };
7252         char search[NFT_OBJ_MAXNAMELEN];
7253         struct rhlist_head *tmp, *list;
7254         struct nft_object *obj;
7255
7256         nla_strscpy(search, nla, sizeof(search));
7257         k.name = search;
7258
7259         WARN_ON_ONCE(!rcu_read_lock_held() &&
7260                      !lockdep_commit_lock_is_held(net));
7261
7262         rcu_read_lock();
7263         list = rhltable_lookup(&nft_objname_ht, &k, nft_objname_ht_params);
7264         if (!list)
7265                 goto out;
7266
7267         rhl_for_each_entry_rcu(obj, tmp, list, rhlhead) {
7268                 if (objtype == obj->ops->type->type &&
7269                     nft_active_genmask(obj, genmask)) {
7270                         rcu_read_unlock();
7271                         return obj;
7272                 }
7273         }
7274 out:
7275         rcu_read_unlock();
7276         return ERR_PTR(-ENOENT);
7277 }
7278 EXPORT_SYMBOL_GPL(nft_obj_lookup);
7279
7280 static struct nft_object *nft_obj_lookup_byhandle(const struct nft_table *table,
7281                                                   const struct nlattr *nla,
7282                                                   u32 objtype, u8 genmask)
7283 {
7284         struct nft_object *obj;
7285
7286         list_for_each_entry(obj, &table->objects, list) {
7287                 if (be64_to_cpu(nla_get_be64(nla)) == obj->handle &&
7288                     objtype == obj->ops->type->type &&
7289                     nft_active_genmask(obj, genmask))
7290                         return obj;
7291         }
7292         return ERR_PTR(-ENOENT);
7293 }
7294
7295 static const struct nla_policy nft_obj_policy[NFTA_OBJ_MAX + 1] = {
7296         [NFTA_OBJ_TABLE]        = { .type = NLA_STRING,
7297                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
7298         [NFTA_OBJ_NAME]         = { .type = NLA_STRING,
7299                                     .len = NFT_OBJ_MAXNAMELEN - 1 },
7300         [NFTA_OBJ_TYPE]         = { .type = NLA_U32 },
7301         [NFTA_OBJ_DATA]         = { .type = NLA_NESTED },
7302         [NFTA_OBJ_HANDLE]       = { .type = NLA_U64},
7303         [NFTA_OBJ_USERDATA]     = { .type = NLA_BINARY,
7304                                     .len = NFT_USERDATA_MAXLEN },
7305 };
7306
7307 static struct nft_object *nft_obj_init(const struct nft_ctx *ctx,
7308                                        const struct nft_object_type *type,
7309                                        const struct nlattr *attr)
7310 {
7311         struct nlattr **tb;
7312         const struct nft_object_ops *ops;
7313         struct nft_object *obj;
7314         int err = -ENOMEM;
7315
7316         tb = kmalloc_array(type->maxattr + 1, sizeof(*tb), GFP_KERNEL);
7317         if (!tb)
7318                 goto err1;
7319
7320         if (attr) {
7321                 err = nla_parse_nested_deprecated(tb, type->maxattr, attr,
7322                                                   type->policy, NULL);
7323                 if (err < 0)
7324                         goto err2;
7325         } else {
7326                 memset(tb, 0, sizeof(tb[0]) * (type->maxattr + 1));
7327         }
7328
7329         if (type->select_ops) {
7330                 ops = type->select_ops(ctx, (const struct nlattr * const *)tb);
7331                 if (IS_ERR(ops)) {
7332                         err = PTR_ERR(ops);
7333                         goto err2;
7334                 }
7335         } else {
7336                 ops = type->ops;
7337         }
7338
7339         err = -ENOMEM;
7340         obj = kzalloc(sizeof(*obj) + ops->size, GFP_KERNEL_ACCOUNT);
7341         if (!obj)
7342                 goto err2;
7343
7344         err = ops->init(ctx, (const struct nlattr * const *)tb, obj);
7345         if (err < 0)
7346                 goto err3;
7347
7348         obj->ops = ops;
7349
7350         kfree(tb);
7351         return obj;
7352 err3:
7353         kfree(obj);
7354 err2:
7355         kfree(tb);
7356 err1:
7357         return ERR_PTR(err);
7358 }
7359
7360 static int nft_object_dump(struct sk_buff *skb, unsigned int attr,
7361                            struct nft_object *obj, bool reset)
7362 {
7363         struct nlattr *nest;
7364
7365         nest = nla_nest_start_noflag(skb, attr);
7366         if (!nest)
7367                 goto nla_put_failure;
7368         if (obj->ops->dump(skb, obj, reset) < 0)
7369                 goto nla_put_failure;
7370         nla_nest_end(skb, nest);
7371         return 0;
7372
7373 nla_put_failure:
7374         return -1;
7375 }
7376
7377 static const struct nft_object_type *__nft_obj_type_get(u32 objtype)
7378 {
7379         const struct nft_object_type *type;
7380
7381         list_for_each_entry(type, &nf_tables_objects, list) {
7382                 if (objtype == type->type)
7383                         return type;
7384         }
7385         return NULL;
7386 }
7387
7388 static const struct nft_object_type *
7389 nft_obj_type_get(struct net *net, u32 objtype)
7390 {
7391         const struct nft_object_type *type;
7392
7393         type = __nft_obj_type_get(objtype);
7394         if (type != NULL && try_module_get(type->owner))
7395                 return type;
7396
7397         lockdep_nfnl_nft_mutex_not_held();
7398 #ifdef CONFIG_MODULES
7399         if (type == NULL) {
7400                 if (nft_request_module(net, "nft-obj-%u", objtype) == -EAGAIN)
7401                         return ERR_PTR(-EAGAIN);
7402         }
7403 #endif
7404         return ERR_PTR(-ENOENT);
7405 }
7406
7407 static int nf_tables_updobj(const struct nft_ctx *ctx,
7408                             const struct nft_object_type *type,
7409                             const struct nlattr *attr,
7410                             struct nft_object *obj)
7411 {
7412         struct nft_object *newobj;
7413         struct nft_trans *trans;
7414         int err = -ENOMEM;
7415
7416         if (!try_module_get(type->owner))
7417                 return -ENOENT;
7418
7419         trans = nft_trans_alloc(ctx, NFT_MSG_NEWOBJ,
7420                                 sizeof(struct nft_trans_obj));
7421         if (!trans)
7422                 goto err_trans;
7423
7424         newobj = nft_obj_init(ctx, type, attr);
7425         if (IS_ERR(newobj)) {
7426                 err = PTR_ERR(newobj);
7427                 goto err_free_trans;
7428         }
7429
7430         nft_trans_obj(trans) = obj;
7431         nft_trans_obj_update(trans) = true;
7432         nft_trans_obj_newobj(trans) = newobj;
7433         nft_trans_commit_list_add_tail(ctx->net, trans);
7434
7435         return 0;
7436
7437 err_free_trans:
7438         kfree(trans);
7439 err_trans:
7440         module_put(type->owner);
7441         return err;
7442 }
7443
7444 static int nf_tables_newobj(struct sk_buff *skb, const struct nfnl_info *info,
7445                             const struct nlattr * const nla[])
7446 {
7447         struct netlink_ext_ack *extack = info->extack;
7448         u8 genmask = nft_genmask_next(info->net);
7449         u8 family = info->nfmsg->nfgen_family;
7450         const struct nft_object_type *type;
7451         struct net *net = info->net;
7452         struct nft_table *table;
7453         struct nft_object *obj;
7454         struct nft_ctx ctx;
7455         u32 objtype;
7456         int err;
7457
7458         if (!nla[NFTA_OBJ_TYPE] ||
7459             !nla[NFTA_OBJ_NAME] ||
7460             !nla[NFTA_OBJ_DATA])
7461                 return -EINVAL;
7462
7463         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
7464                                  NETLINK_CB(skb).portid);
7465         if (IS_ERR(table)) {
7466                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7467                 return PTR_ERR(table);
7468         }
7469
7470         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7471         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
7472         if (IS_ERR(obj)) {
7473                 err = PTR_ERR(obj);
7474                 if (err != -ENOENT) {
7475                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7476                         return err;
7477                 }
7478         } else {
7479                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
7480                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7481                         return -EEXIST;
7482                 }
7483                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
7484                         return -EOPNOTSUPP;
7485
7486                 type = __nft_obj_type_get(objtype);
7487                 if (WARN_ON_ONCE(!type))
7488                         return -ENOENT;
7489
7490                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7491
7492                 return nf_tables_updobj(&ctx, type, nla[NFTA_OBJ_DATA], obj);
7493         }
7494
7495         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7496
7497         if (!nft_use_inc(&table->use))
7498                 return -EMFILE;
7499
7500         type = nft_obj_type_get(net, objtype);
7501         if (IS_ERR(type)) {
7502                 err = PTR_ERR(type);
7503                 goto err_type;
7504         }
7505
7506         obj = nft_obj_init(&ctx, type, nla[NFTA_OBJ_DATA]);
7507         if (IS_ERR(obj)) {
7508                 err = PTR_ERR(obj);
7509                 goto err_init;
7510         }
7511         obj->key.table = table;
7512         obj->handle = nf_tables_alloc_handle(table);
7513
7514         obj->key.name = nla_strdup(nla[NFTA_OBJ_NAME], GFP_KERNEL_ACCOUNT);
7515         if (!obj->key.name) {
7516                 err = -ENOMEM;
7517                 goto err_strdup;
7518         }
7519
7520         if (nla[NFTA_OBJ_USERDATA]) {
7521                 obj->udata = nla_memdup(nla[NFTA_OBJ_USERDATA], GFP_KERNEL_ACCOUNT);
7522                 if (obj->udata == NULL)
7523                         goto err_userdata;
7524
7525                 obj->udlen = nla_len(nla[NFTA_OBJ_USERDATA]);
7526         }
7527
7528         err = nft_trans_obj_add(&ctx, NFT_MSG_NEWOBJ, obj);
7529         if (err < 0)
7530                 goto err_trans;
7531
7532         err = rhltable_insert(&nft_objname_ht, &obj->rhlhead,
7533                               nft_objname_ht_params);
7534         if (err < 0)
7535                 goto err_obj_ht;
7536
7537         list_add_tail_rcu(&obj->list, &table->objects);
7538
7539         return 0;
7540 err_obj_ht:
7541         /* queued in transaction log */
7542         INIT_LIST_HEAD(&obj->list);
7543         return err;
7544 err_trans:
7545         kfree(obj->udata);
7546 err_userdata:
7547         kfree(obj->key.name);
7548 err_strdup:
7549         if (obj->ops->destroy)
7550                 obj->ops->destroy(&ctx, obj);
7551         kfree(obj);
7552 err_init:
7553         module_put(type->owner);
7554 err_type:
7555         nft_use_dec_restore(&table->use);
7556
7557         return err;
7558 }
7559
7560 static int nf_tables_fill_obj_info(struct sk_buff *skb, struct net *net,
7561                                    u32 portid, u32 seq, int event, u32 flags,
7562                                    int family, const struct nft_table *table,
7563                                    struct nft_object *obj, bool reset)
7564 {
7565         struct nlmsghdr *nlh;
7566
7567         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
7568         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
7569                            NFNETLINK_V0, nft_base_seq(net));
7570         if (!nlh)
7571                 goto nla_put_failure;
7572
7573         if (nla_put_string(skb, NFTA_OBJ_TABLE, table->name) ||
7574             nla_put_string(skb, NFTA_OBJ_NAME, obj->key.name) ||
7575             nla_put_be64(skb, NFTA_OBJ_HANDLE, cpu_to_be64(obj->handle),
7576                          NFTA_OBJ_PAD))
7577                 goto nla_put_failure;
7578
7579         if (event == NFT_MSG_DELOBJ) {
7580                 nlmsg_end(skb, nlh);
7581                 return 0;
7582         }
7583
7584         if (nla_put_be32(skb, NFTA_OBJ_TYPE, htonl(obj->ops->type->type)) ||
7585             nla_put_be32(skb, NFTA_OBJ_USE, htonl(obj->use)) ||
7586             nft_object_dump(skb, NFTA_OBJ_DATA, obj, reset))
7587                 goto nla_put_failure;
7588
7589         if (obj->udata &&
7590             nla_put(skb, NFTA_OBJ_USERDATA, obj->udlen, obj->udata))
7591                 goto nla_put_failure;
7592
7593         nlmsg_end(skb, nlh);
7594         return 0;
7595
7596 nla_put_failure:
7597         nlmsg_trim(skb, nlh);
7598         return -1;
7599 }
7600
7601 struct nft_obj_filter {
7602         char            *table;
7603         u32             type;
7604 };
7605
7606 static int nf_tables_dump_obj(struct sk_buff *skb, struct netlink_callback *cb)
7607 {
7608         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
7609         const struct nft_table *table;
7610         unsigned int idx = 0, s_idx = cb->args[0];
7611         struct nft_obj_filter *filter = cb->data;
7612         struct net *net = sock_net(skb->sk);
7613         int family = nfmsg->nfgen_family;
7614         struct nftables_pernet *nft_net;
7615         struct nft_object *obj;
7616         bool reset = false;
7617
7618         if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
7619                 reset = true;
7620
7621         rcu_read_lock();
7622         nft_net = nft_pernet(net);
7623         cb->seq = READ_ONCE(nft_net->base_seq);
7624
7625         list_for_each_entry_rcu(table, &nft_net->tables, list) {
7626                 if (family != NFPROTO_UNSPEC && family != table->family)
7627                         continue;
7628
7629                 list_for_each_entry_rcu(obj, &table->objects, list) {
7630                         if (!nft_is_active(net, obj))
7631                                 goto cont;
7632                         if (idx < s_idx)
7633                                 goto cont;
7634                         if (idx > s_idx)
7635                                 memset(&cb->args[1], 0,
7636                                        sizeof(cb->args) - sizeof(cb->args[0]));
7637                         if (filter && filter->table &&
7638                             strcmp(filter->table, table->name))
7639                                 goto cont;
7640                         if (filter &&
7641                             filter->type != NFT_OBJECT_UNSPEC &&
7642                             obj->ops->type->type != filter->type)
7643                                 goto cont;
7644                         if (reset) {
7645                                 char *buf = kasprintf(GFP_ATOMIC,
7646                                                       "%s:%u",
7647                                                       table->name,
7648                                                       nft_net->base_seq);
7649
7650                                 audit_log_nfcfg(buf,
7651                                                 family,
7652                                                 obj->handle,
7653                                                 AUDIT_NFT_OP_OBJ_RESET,
7654                                                 GFP_ATOMIC);
7655                                 kfree(buf);
7656                         }
7657
7658                         if (nf_tables_fill_obj_info(skb, net, NETLINK_CB(cb->skb).portid,
7659                                                     cb->nlh->nlmsg_seq,
7660                                                     NFT_MSG_NEWOBJ,
7661                                                     NLM_F_MULTI | NLM_F_APPEND,
7662                                                     table->family, table,
7663                                                     obj, reset) < 0)
7664                                 goto done;
7665
7666                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
7667 cont:
7668                         idx++;
7669                 }
7670         }
7671 done:
7672         rcu_read_unlock();
7673
7674         cb->args[0] = idx;
7675         return skb->len;
7676 }
7677
7678 static int nf_tables_dump_obj_start(struct netlink_callback *cb)
7679 {
7680         const struct nlattr * const *nla = cb->data;
7681         struct nft_obj_filter *filter = NULL;
7682
7683         if (nla[NFTA_OBJ_TABLE] || nla[NFTA_OBJ_TYPE]) {
7684                 filter = kzalloc(sizeof(*filter), GFP_ATOMIC);
7685                 if (!filter)
7686                         return -ENOMEM;
7687
7688                 if (nla[NFTA_OBJ_TABLE]) {
7689                         filter->table = nla_strdup(nla[NFTA_OBJ_TABLE], GFP_ATOMIC);
7690                         if (!filter->table) {
7691                                 kfree(filter);
7692                                 return -ENOMEM;
7693                         }
7694                 }
7695
7696                 if (nla[NFTA_OBJ_TYPE])
7697                         filter->type = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7698         }
7699
7700         cb->data = filter;
7701         return 0;
7702 }
7703
7704 static int nf_tables_dump_obj_done(struct netlink_callback *cb)
7705 {
7706         struct nft_obj_filter *filter = cb->data;
7707
7708         if (filter) {
7709                 kfree(filter->table);
7710                 kfree(filter);
7711         }
7712
7713         return 0;
7714 }
7715
7716 /* called with rcu_read_lock held */
7717 static int nf_tables_getobj(struct sk_buff *skb, const struct nfnl_info *info,
7718                             const struct nlattr * const nla[])
7719 {
7720         struct netlink_ext_ack *extack = info->extack;
7721         u8 genmask = nft_genmask_cur(info->net);
7722         u8 family = info->nfmsg->nfgen_family;
7723         const struct nft_table *table;
7724         struct net *net = info->net;
7725         struct nft_object *obj;
7726         struct sk_buff *skb2;
7727         bool reset = false;
7728         u32 objtype;
7729         int err;
7730
7731         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
7732                 struct netlink_dump_control c = {
7733                         .start = nf_tables_dump_obj_start,
7734                         .dump = nf_tables_dump_obj,
7735                         .done = nf_tables_dump_obj_done,
7736                         .module = THIS_MODULE,
7737                         .data = (void *)nla,
7738                 };
7739
7740                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
7741         }
7742
7743         if (!nla[NFTA_OBJ_NAME] ||
7744             !nla[NFTA_OBJ_TYPE])
7745                 return -EINVAL;
7746
7747         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask, 0);
7748         if (IS_ERR(table)) {
7749                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7750                 return PTR_ERR(table);
7751         }
7752
7753         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7754         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
7755         if (IS_ERR(obj)) {
7756                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7757                 return PTR_ERR(obj);
7758         }
7759
7760         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
7761         if (!skb2)
7762                 return -ENOMEM;
7763
7764         if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
7765                 reset = true;
7766
7767         if (reset) {
7768                 const struct nftables_pernet *nft_net;
7769                 char *buf;
7770
7771                 nft_net = nft_pernet(net);
7772                 buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, nft_net->base_seq);
7773
7774                 audit_log_nfcfg(buf,
7775                                 family,
7776                                 obj->handle,
7777                                 AUDIT_NFT_OP_OBJ_RESET,
7778                                 GFP_ATOMIC);
7779                 kfree(buf);
7780         }
7781
7782         err = nf_tables_fill_obj_info(skb2, net, NETLINK_CB(skb).portid,
7783                                       info->nlh->nlmsg_seq, NFT_MSG_NEWOBJ, 0,
7784                                       family, table, obj, reset);
7785         if (err < 0)
7786                 goto err_fill_obj_info;
7787
7788         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
7789
7790 err_fill_obj_info:
7791         kfree_skb(skb2);
7792         return err;
7793 }
7794
7795 static void nft_obj_destroy(const struct nft_ctx *ctx, struct nft_object *obj)
7796 {
7797         if (obj->ops->destroy)
7798                 obj->ops->destroy(ctx, obj);
7799
7800         module_put(obj->ops->type->owner);
7801         kfree(obj->key.name);
7802         kfree(obj->udata);
7803         kfree(obj);
7804 }
7805
7806 static int nf_tables_delobj(struct sk_buff *skb, const struct nfnl_info *info,
7807                             const struct nlattr * const nla[])
7808 {
7809         struct netlink_ext_ack *extack = info->extack;
7810         u8 genmask = nft_genmask_next(info->net);
7811         u8 family = info->nfmsg->nfgen_family;
7812         struct net *net = info->net;
7813         const struct nlattr *attr;
7814         struct nft_table *table;
7815         struct nft_object *obj;
7816         struct nft_ctx ctx;
7817         u32 objtype;
7818
7819         if (!nla[NFTA_OBJ_TYPE] ||
7820             (!nla[NFTA_OBJ_NAME] && !nla[NFTA_OBJ_HANDLE]))
7821                 return -EINVAL;
7822
7823         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
7824                                  NETLINK_CB(skb).portid);
7825         if (IS_ERR(table)) {
7826                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7827                 return PTR_ERR(table);
7828         }
7829
7830         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7831         if (nla[NFTA_OBJ_HANDLE]) {
7832                 attr = nla[NFTA_OBJ_HANDLE];
7833                 obj = nft_obj_lookup_byhandle(table, attr, objtype, genmask);
7834         } else {
7835                 attr = nla[NFTA_OBJ_NAME];
7836                 obj = nft_obj_lookup(net, table, attr, objtype, genmask);
7837         }
7838
7839         if (IS_ERR(obj)) {
7840                 if (PTR_ERR(obj) == -ENOENT &&
7841                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYOBJ)
7842                         return 0;
7843
7844                 NL_SET_BAD_ATTR(extack, attr);
7845                 return PTR_ERR(obj);
7846         }
7847         if (obj->use > 0) {
7848                 NL_SET_BAD_ATTR(extack, attr);
7849                 return -EBUSY;
7850         }
7851
7852         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7853
7854         return nft_delobj(&ctx, obj);
7855 }
7856
7857 void nft_obj_notify(struct net *net, const struct nft_table *table,
7858                     struct nft_object *obj, u32 portid, u32 seq, int event,
7859                     u16 flags, int family, int report, gfp_t gfp)
7860 {
7861         struct nftables_pernet *nft_net = nft_pernet(net);
7862         struct sk_buff *skb;
7863         int err;
7864         char *buf = kasprintf(gfp, "%s:%u",
7865                               table->name, nft_net->base_seq);
7866
7867         audit_log_nfcfg(buf,
7868                         family,
7869                         obj->handle,
7870                         event == NFT_MSG_NEWOBJ ?
7871                                  AUDIT_NFT_OP_OBJ_REGISTER :
7872                                  AUDIT_NFT_OP_OBJ_UNREGISTER,
7873                         gfp);
7874         kfree(buf);
7875
7876         if (!report &&
7877             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
7878                 return;
7879
7880         skb = nlmsg_new(NLMSG_GOODSIZE, gfp);
7881         if (skb == NULL)
7882                 goto err;
7883
7884         err = nf_tables_fill_obj_info(skb, net, portid, seq, event,
7885                                       flags & (NLM_F_CREATE | NLM_F_EXCL),
7886                                       family, table, obj, false);
7887         if (err < 0) {
7888                 kfree_skb(skb);
7889                 goto err;
7890         }
7891
7892         nft_notify_enqueue(skb, report, &nft_net->notify_list);
7893         return;
7894 err:
7895         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
7896 }
7897 EXPORT_SYMBOL_GPL(nft_obj_notify);
7898
7899 static void nf_tables_obj_notify(const struct nft_ctx *ctx,
7900                                  struct nft_object *obj, int event)
7901 {
7902         nft_obj_notify(ctx->net, ctx->table, obj, ctx->portid, ctx->seq, event,
7903                        ctx->flags, ctx->family, ctx->report, GFP_KERNEL);
7904 }
7905
7906 /*
7907  * Flow tables
7908  */
7909 void nft_register_flowtable_type(struct nf_flowtable_type *type)
7910 {
7911         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7912         list_add_tail_rcu(&type->list, &nf_tables_flowtables);
7913         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7914 }
7915 EXPORT_SYMBOL_GPL(nft_register_flowtable_type);
7916
7917 void nft_unregister_flowtable_type(struct nf_flowtable_type *type)
7918 {
7919         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7920         list_del_rcu(&type->list);
7921         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7922 }
7923 EXPORT_SYMBOL_GPL(nft_unregister_flowtable_type);
7924
7925 static const struct nla_policy nft_flowtable_policy[NFTA_FLOWTABLE_MAX + 1] = {
7926         [NFTA_FLOWTABLE_TABLE]          = { .type = NLA_STRING,
7927                                             .len = NFT_NAME_MAXLEN - 1 },
7928         [NFTA_FLOWTABLE_NAME]           = { .type = NLA_STRING,
7929                                             .len = NFT_NAME_MAXLEN - 1 },
7930         [NFTA_FLOWTABLE_HOOK]           = { .type = NLA_NESTED },
7931         [NFTA_FLOWTABLE_HANDLE]         = { .type = NLA_U64 },
7932         [NFTA_FLOWTABLE_FLAGS]          = { .type = NLA_U32 },
7933 };
7934
7935 struct nft_flowtable *nft_flowtable_lookup(const struct nft_table *table,
7936                                            const struct nlattr *nla, u8 genmask)
7937 {
7938         struct nft_flowtable *flowtable;
7939
7940         list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
7941                 if (!nla_strcmp(nla, flowtable->name) &&
7942                     nft_active_genmask(flowtable, genmask))
7943                         return flowtable;
7944         }
7945         return ERR_PTR(-ENOENT);
7946 }
7947 EXPORT_SYMBOL_GPL(nft_flowtable_lookup);
7948
7949 void nf_tables_deactivate_flowtable(const struct nft_ctx *ctx,
7950                                     struct nft_flowtable *flowtable,
7951                                     enum nft_trans_phase phase)
7952 {
7953         switch (phase) {
7954         case NFT_TRANS_PREPARE_ERROR:
7955         case NFT_TRANS_PREPARE:
7956         case NFT_TRANS_ABORT:
7957         case NFT_TRANS_RELEASE:
7958                 nft_use_dec(&flowtable->use);
7959                 fallthrough;
7960         default:
7961                 return;
7962         }
7963 }
7964 EXPORT_SYMBOL_GPL(nf_tables_deactivate_flowtable);
7965
7966 static struct nft_flowtable *
7967 nft_flowtable_lookup_byhandle(const struct nft_table *table,
7968                               const struct nlattr *nla, u8 genmask)
7969 {
7970        struct nft_flowtable *flowtable;
7971
7972        list_for_each_entry(flowtable, &table->flowtables, list) {
7973                if (be64_to_cpu(nla_get_be64(nla)) == flowtable->handle &&
7974                    nft_active_genmask(flowtable, genmask))
7975                        return flowtable;
7976        }
7977        return ERR_PTR(-ENOENT);
7978 }
7979
7980 struct nft_flowtable_hook {
7981         u32                     num;
7982         int                     priority;
7983         struct list_head        list;
7984 };
7985
7986 static const struct nla_policy nft_flowtable_hook_policy[NFTA_FLOWTABLE_HOOK_MAX + 1] = {
7987         [NFTA_FLOWTABLE_HOOK_NUM]       = { .type = NLA_U32 },
7988         [NFTA_FLOWTABLE_HOOK_PRIORITY]  = { .type = NLA_U32 },
7989         [NFTA_FLOWTABLE_HOOK_DEVS]      = { .type = NLA_NESTED },
7990 };
7991
7992 static int nft_flowtable_parse_hook(const struct nft_ctx *ctx,
7993                                     const struct nlattr * const nla[],
7994                                     struct nft_flowtable_hook *flowtable_hook,
7995                                     struct nft_flowtable *flowtable,
7996                                     struct netlink_ext_ack *extack, bool add)
7997 {
7998         struct nlattr *tb[NFTA_FLOWTABLE_HOOK_MAX + 1];
7999         struct nft_hook *hook;
8000         int hooknum, priority;
8001         int err;
8002
8003         INIT_LIST_HEAD(&flowtable_hook->list);
8004
8005         err = nla_parse_nested_deprecated(tb, NFTA_FLOWTABLE_HOOK_MAX,
8006                                           nla[NFTA_FLOWTABLE_HOOK],
8007                                           nft_flowtable_hook_policy, NULL);
8008         if (err < 0)
8009                 return err;
8010
8011         if (add) {
8012                 if (!tb[NFTA_FLOWTABLE_HOOK_NUM] ||
8013                     !tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
8014                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8015                         return -ENOENT;
8016                 }
8017
8018                 hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
8019                 if (hooknum != NF_NETDEV_INGRESS)
8020                         return -EOPNOTSUPP;
8021
8022                 priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
8023
8024                 flowtable_hook->priority        = priority;
8025                 flowtable_hook->num             = hooknum;
8026         } else {
8027                 if (tb[NFTA_FLOWTABLE_HOOK_NUM]) {
8028                         hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
8029                         if (hooknum != flowtable->hooknum)
8030                                 return -EOPNOTSUPP;
8031                 }
8032
8033                 if (tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
8034                         priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
8035                         if (priority != flowtable->data.priority)
8036                                 return -EOPNOTSUPP;
8037                 }
8038
8039                 flowtable_hook->priority        = flowtable->data.priority;
8040                 flowtable_hook->num             = flowtable->hooknum;
8041         }
8042
8043         if (tb[NFTA_FLOWTABLE_HOOK_DEVS]) {
8044                 err = nf_tables_parse_netdev_hooks(ctx->net,
8045                                                    tb[NFTA_FLOWTABLE_HOOK_DEVS],
8046                                                    &flowtable_hook->list,
8047                                                    extack);
8048                 if (err < 0)
8049                         return err;
8050         }
8051
8052         list_for_each_entry(hook, &flowtable_hook->list, list) {
8053                 hook->ops.pf            = NFPROTO_NETDEV;
8054                 hook->ops.hooknum       = flowtable_hook->num;
8055                 hook->ops.priority      = flowtable_hook->priority;
8056                 hook->ops.priv          = &flowtable->data;
8057                 hook->ops.hook          = flowtable->data.type->hook;
8058         }
8059
8060         return err;
8061 }
8062
8063 static const struct nf_flowtable_type *__nft_flowtable_type_get(u8 family)
8064 {
8065         const struct nf_flowtable_type *type;
8066
8067         list_for_each_entry(type, &nf_tables_flowtables, list) {
8068                 if (family == type->family)
8069                         return type;
8070         }
8071         return NULL;
8072 }
8073
8074 static const struct nf_flowtable_type *
8075 nft_flowtable_type_get(struct net *net, u8 family)
8076 {
8077         const struct nf_flowtable_type *type;
8078
8079         type = __nft_flowtable_type_get(family);
8080         if (type != NULL && try_module_get(type->owner))
8081                 return type;
8082
8083         lockdep_nfnl_nft_mutex_not_held();
8084 #ifdef CONFIG_MODULES
8085         if (type == NULL) {
8086                 if (nft_request_module(net, "nf-flowtable-%u", family) == -EAGAIN)
8087                         return ERR_PTR(-EAGAIN);
8088         }
8089 #endif
8090         return ERR_PTR(-ENOENT);
8091 }
8092
8093 /* Only called from error and netdev event paths. */
8094 static void nft_unregister_flowtable_hook(struct net *net,
8095                                           struct nft_flowtable *flowtable,
8096                                           struct nft_hook *hook)
8097 {
8098         nf_unregister_net_hook(net, &hook->ops);
8099         flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
8100                                     FLOW_BLOCK_UNBIND);
8101 }
8102
8103 static void __nft_unregister_flowtable_net_hooks(struct net *net,
8104                                                  struct list_head *hook_list,
8105                                                  bool release_netdev)
8106 {
8107         struct nft_hook *hook, *next;
8108
8109         list_for_each_entry_safe(hook, next, hook_list, list) {
8110                 nf_unregister_net_hook(net, &hook->ops);
8111                 if (release_netdev) {
8112                         list_del(&hook->list);
8113                         kfree_rcu(hook, rcu);
8114                 }
8115         }
8116 }
8117
8118 static void nft_unregister_flowtable_net_hooks(struct net *net,
8119                                                struct list_head *hook_list)
8120 {
8121         __nft_unregister_flowtable_net_hooks(net, hook_list, false);
8122 }
8123
8124 static int nft_register_flowtable_net_hooks(struct net *net,
8125                                             struct nft_table *table,
8126                                             struct list_head *hook_list,
8127                                             struct nft_flowtable *flowtable)
8128 {
8129         struct nft_hook *hook, *hook2, *next;
8130         struct nft_flowtable *ft;
8131         int err, i = 0;
8132
8133         list_for_each_entry(hook, hook_list, list) {
8134                 list_for_each_entry(ft, &table->flowtables, list) {
8135                         if (!nft_is_active_next(net, ft))
8136                                 continue;
8137
8138                         list_for_each_entry(hook2, &ft->hook_list, list) {
8139                                 if (hook->ops.dev == hook2->ops.dev &&
8140                                     hook->ops.pf == hook2->ops.pf) {
8141                                         err = -EEXIST;
8142                                         goto err_unregister_net_hooks;
8143                                 }
8144                         }
8145                 }
8146
8147                 err = flowtable->data.type->setup(&flowtable->data,
8148                                                   hook->ops.dev,
8149                                                   FLOW_BLOCK_BIND);
8150                 if (err < 0)
8151                         goto err_unregister_net_hooks;
8152
8153                 err = nf_register_net_hook(net, &hook->ops);
8154                 if (err < 0) {
8155                         flowtable->data.type->setup(&flowtable->data,
8156                                                     hook->ops.dev,
8157                                                     FLOW_BLOCK_UNBIND);
8158                         goto err_unregister_net_hooks;
8159                 }
8160
8161                 i++;
8162         }
8163
8164         return 0;
8165
8166 err_unregister_net_hooks:
8167         list_for_each_entry_safe(hook, next, hook_list, list) {
8168                 if (i-- <= 0)
8169                         break;
8170
8171                 nft_unregister_flowtable_hook(net, flowtable, hook);
8172                 list_del_rcu(&hook->list);
8173                 kfree_rcu(hook, rcu);
8174         }
8175
8176         return err;
8177 }
8178
8179 static void nft_hooks_destroy(struct list_head *hook_list)
8180 {
8181         struct nft_hook *hook, *next;
8182
8183         list_for_each_entry_safe(hook, next, hook_list, list) {
8184                 list_del_rcu(&hook->list);
8185                 kfree_rcu(hook, rcu);
8186         }
8187 }
8188
8189 static int nft_flowtable_update(struct nft_ctx *ctx, const struct nlmsghdr *nlh,
8190                                 struct nft_flowtable *flowtable,
8191                                 struct netlink_ext_ack *extack)
8192 {
8193         const struct nlattr * const *nla = ctx->nla;
8194         struct nft_flowtable_hook flowtable_hook;
8195         struct nft_hook *hook, *next;
8196         struct nft_trans *trans;
8197         bool unregister = false;
8198         u32 flags;
8199         int err;
8200
8201         err = nft_flowtable_parse_hook(ctx, nla, &flowtable_hook, flowtable,
8202                                        extack, false);
8203         if (err < 0)
8204                 return err;
8205
8206         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
8207                 if (nft_hook_list_find(&flowtable->hook_list, hook)) {
8208                         list_del(&hook->list);
8209                         kfree(hook);
8210                 }
8211         }
8212
8213         if (nla[NFTA_FLOWTABLE_FLAGS]) {
8214                 flags = ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
8215                 if (flags & ~NFT_FLOWTABLE_MASK) {
8216                         err = -EOPNOTSUPP;
8217                         goto err_flowtable_update_hook;
8218                 }
8219                 if ((flowtable->data.flags & NFT_FLOWTABLE_HW_OFFLOAD) ^
8220                     (flags & NFT_FLOWTABLE_HW_OFFLOAD)) {
8221                         err = -EOPNOTSUPP;
8222                         goto err_flowtable_update_hook;
8223                 }
8224         } else {
8225                 flags = flowtable->data.flags;
8226         }
8227
8228         err = nft_register_flowtable_net_hooks(ctx->net, ctx->table,
8229                                                &flowtable_hook.list, flowtable);
8230         if (err < 0)
8231                 goto err_flowtable_update_hook;
8232
8233         trans = nft_trans_alloc(ctx, NFT_MSG_NEWFLOWTABLE,
8234                                 sizeof(struct nft_trans_flowtable));
8235         if (!trans) {
8236                 unregister = true;
8237                 err = -ENOMEM;
8238                 goto err_flowtable_update_hook;
8239         }
8240
8241         nft_trans_flowtable_flags(trans) = flags;
8242         nft_trans_flowtable(trans) = flowtable;
8243         nft_trans_flowtable_update(trans) = true;
8244         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
8245         list_splice(&flowtable_hook.list, &nft_trans_flowtable_hooks(trans));
8246
8247         nft_trans_commit_list_add_tail(ctx->net, trans);
8248
8249         return 0;
8250
8251 err_flowtable_update_hook:
8252         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
8253                 if (unregister)
8254                         nft_unregister_flowtable_hook(ctx->net, flowtable, hook);
8255                 list_del_rcu(&hook->list);
8256                 kfree_rcu(hook, rcu);
8257         }
8258
8259         return err;
8260
8261 }
8262
8263 static int nf_tables_newflowtable(struct sk_buff *skb,
8264                                   const struct nfnl_info *info,
8265                                   const struct nlattr * const nla[])
8266 {
8267         struct netlink_ext_ack *extack = info->extack;
8268         struct nft_flowtable_hook flowtable_hook;
8269         u8 genmask = nft_genmask_next(info->net);
8270         u8 family = info->nfmsg->nfgen_family;
8271         const struct nf_flowtable_type *type;
8272         struct nft_flowtable *flowtable;
8273         struct nft_hook *hook, *next;
8274         struct net *net = info->net;
8275         struct nft_table *table;
8276         struct nft_ctx ctx;
8277         int err;
8278
8279         if (!nla[NFTA_FLOWTABLE_TABLE] ||
8280             !nla[NFTA_FLOWTABLE_NAME] ||
8281             !nla[NFTA_FLOWTABLE_HOOK])
8282                 return -EINVAL;
8283
8284         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8285                                  genmask, NETLINK_CB(skb).portid);
8286         if (IS_ERR(table)) {
8287                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
8288                 return PTR_ERR(table);
8289         }
8290
8291         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
8292                                          genmask);
8293         if (IS_ERR(flowtable)) {
8294                 err = PTR_ERR(flowtable);
8295                 if (err != -ENOENT) {
8296                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8297                         return err;
8298                 }
8299         } else {
8300                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
8301                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8302                         return -EEXIST;
8303                 }
8304
8305                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8306
8307                 return nft_flowtable_update(&ctx, info->nlh, flowtable, extack);
8308         }
8309
8310         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8311
8312         if (!nft_use_inc(&table->use))
8313                 return -EMFILE;
8314
8315         flowtable = kzalloc(sizeof(*flowtable), GFP_KERNEL_ACCOUNT);
8316         if (!flowtable) {
8317                 err = -ENOMEM;
8318                 goto flowtable_alloc;
8319         }
8320
8321         flowtable->table = table;
8322         flowtable->handle = nf_tables_alloc_handle(table);
8323         INIT_LIST_HEAD(&flowtable->hook_list);
8324
8325         flowtable->name = nla_strdup(nla[NFTA_FLOWTABLE_NAME], GFP_KERNEL_ACCOUNT);
8326         if (!flowtable->name) {
8327                 err = -ENOMEM;
8328                 goto err1;
8329         }
8330
8331         type = nft_flowtable_type_get(net, family);
8332         if (IS_ERR(type)) {
8333                 err = PTR_ERR(type);
8334                 goto err2;
8335         }
8336
8337         if (nla[NFTA_FLOWTABLE_FLAGS]) {
8338                 flowtable->data.flags =
8339                         ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
8340                 if (flowtable->data.flags & ~NFT_FLOWTABLE_MASK) {
8341                         err = -EOPNOTSUPP;
8342                         goto err3;
8343                 }
8344         }
8345
8346         write_pnet(&flowtable->data.net, net);
8347         flowtable->data.type = type;
8348         err = type->init(&flowtable->data);
8349         if (err < 0)
8350                 goto err3;
8351
8352         err = nft_flowtable_parse_hook(&ctx, nla, &flowtable_hook, flowtable,
8353                                        extack, true);
8354         if (err < 0)
8355                 goto err4;
8356
8357         list_splice(&flowtable_hook.list, &flowtable->hook_list);
8358         flowtable->data.priority = flowtable_hook.priority;
8359         flowtable->hooknum = flowtable_hook.num;
8360
8361         err = nft_register_flowtable_net_hooks(ctx.net, table,
8362                                                &flowtable->hook_list,
8363                                                flowtable);
8364         if (err < 0) {
8365                 nft_hooks_destroy(&flowtable->hook_list);
8366                 goto err4;
8367         }
8368
8369         err = nft_trans_flowtable_add(&ctx, NFT_MSG_NEWFLOWTABLE, flowtable);
8370         if (err < 0)
8371                 goto err5;
8372
8373         list_add_tail_rcu(&flowtable->list, &table->flowtables);
8374
8375         return 0;
8376 err5:
8377         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
8378                 nft_unregister_flowtable_hook(net, flowtable, hook);
8379                 list_del_rcu(&hook->list);
8380                 kfree_rcu(hook, rcu);
8381         }
8382 err4:
8383         flowtable->data.type->free(&flowtable->data);
8384 err3:
8385         module_put(type->owner);
8386 err2:
8387         kfree(flowtable->name);
8388 err1:
8389         kfree(flowtable);
8390 flowtable_alloc:
8391         nft_use_dec_restore(&table->use);
8392
8393         return err;
8394 }
8395
8396 static void nft_flowtable_hook_release(struct nft_flowtable_hook *flowtable_hook)
8397 {
8398         struct nft_hook *this, *next;
8399
8400         list_for_each_entry_safe(this, next, &flowtable_hook->list, list) {
8401                 list_del(&this->list);
8402                 kfree(this);
8403         }
8404 }
8405
8406 static int nft_delflowtable_hook(struct nft_ctx *ctx,
8407                                  struct nft_flowtable *flowtable,
8408                                  struct netlink_ext_ack *extack)
8409 {
8410         const struct nlattr * const *nla = ctx->nla;
8411         struct nft_flowtable_hook flowtable_hook;
8412         LIST_HEAD(flowtable_del_list);
8413         struct nft_hook *this, *hook;
8414         struct nft_trans *trans;
8415         int err;
8416
8417         err = nft_flowtable_parse_hook(ctx, nla, &flowtable_hook, flowtable,
8418                                        extack, false);
8419         if (err < 0)
8420                 return err;
8421
8422         list_for_each_entry(this, &flowtable_hook.list, list) {
8423                 hook = nft_hook_list_find(&flowtable->hook_list, this);
8424                 if (!hook) {
8425                         err = -ENOENT;
8426                         goto err_flowtable_del_hook;
8427                 }
8428                 list_move(&hook->list, &flowtable_del_list);
8429         }
8430
8431         trans = nft_trans_alloc(ctx, NFT_MSG_DELFLOWTABLE,
8432                                 sizeof(struct nft_trans_flowtable));
8433         if (!trans) {
8434                 err = -ENOMEM;
8435                 goto err_flowtable_del_hook;
8436         }
8437
8438         nft_trans_flowtable(trans) = flowtable;
8439         nft_trans_flowtable_update(trans) = true;
8440         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
8441         list_splice(&flowtable_del_list, &nft_trans_flowtable_hooks(trans));
8442         nft_flowtable_hook_release(&flowtable_hook);
8443
8444         nft_trans_commit_list_add_tail(ctx->net, trans);
8445
8446         return 0;
8447
8448 err_flowtable_del_hook:
8449         list_splice(&flowtable_del_list, &flowtable->hook_list);
8450         nft_flowtable_hook_release(&flowtable_hook);
8451
8452         return err;
8453 }
8454
8455 static int nf_tables_delflowtable(struct sk_buff *skb,
8456                                   const struct nfnl_info *info,
8457                                   const struct nlattr * const nla[])
8458 {
8459         struct netlink_ext_ack *extack = info->extack;
8460         u8 genmask = nft_genmask_next(info->net);
8461         u8 family = info->nfmsg->nfgen_family;
8462         struct nft_flowtable *flowtable;
8463         struct net *net = info->net;
8464         const struct nlattr *attr;
8465         struct nft_table *table;
8466         struct nft_ctx ctx;
8467
8468         if (!nla[NFTA_FLOWTABLE_TABLE] ||
8469             (!nla[NFTA_FLOWTABLE_NAME] &&
8470              !nla[NFTA_FLOWTABLE_HANDLE]))
8471                 return -EINVAL;
8472
8473         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8474                                  genmask, NETLINK_CB(skb).portid);
8475         if (IS_ERR(table)) {
8476                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
8477                 return PTR_ERR(table);
8478         }
8479
8480         if (nla[NFTA_FLOWTABLE_HANDLE]) {
8481                 attr = nla[NFTA_FLOWTABLE_HANDLE];
8482                 flowtable = nft_flowtable_lookup_byhandle(table, attr, genmask);
8483         } else {
8484                 attr = nla[NFTA_FLOWTABLE_NAME];
8485                 flowtable = nft_flowtable_lookup(table, attr, genmask);
8486         }
8487
8488         if (IS_ERR(flowtable)) {
8489                 if (PTR_ERR(flowtable) == -ENOENT &&
8490                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYFLOWTABLE)
8491                         return 0;
8492
8493                 NL_SET_BAD_ATTR(extack, attr);
8494                 return PTR_ERR(flowtable);
8495         }
8496
8497         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8498
8499         if (nla[NFTA_FLOWTABLE_HOOK])
8500                 return nft_delflowtable_hook(&ctx, flowtable, extack);
8501
8502         if (flowtable->use > 0) {
8503                 NL_SET_BAD_ATTR(extack, attr);
8504                 return -EBUSY;
8505         }
8506
8507         return nft_delflowtable(&ctx, flowtable);
8508 }
8509
8510 static int nf_tables_fill_flowtable_info(struct sk_buff *skb, struct net *net,
8511                                          u32 portid, u32 seq, int event,
8512                                          u32 flags, int family,
8513                                          struct nft_flowtable *flowtable,
8514                                          struct list_head *hook_list)
8515 {
8516         struct nlattr *nest, *nest_devs;
8517         struct nft_hook *hook;
8518         struct nlmsghdr *nlh;
8519
8520         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
8521         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
8522                            NFNETLINK_V0, nft_base_seq(net));
8523         if (!nlh)
8524                 goto nla_put_failure;
8525
8526         if (nla_put_string(skb, NFTA_FLOWTABLE_TABLE, flowtable->table->name) ||
8527             nla_put_string(skb, NFTA_FLOWTABLE_NAME, flowtable->name) ||
8528             nla_put_be64(skb, NFTA_FLOWTABLE_HANDLE, cpu_to_be64(flowtable->handle),
8529                          NFTA_FLOWTABLE_PAD))
8530                 goto nla_put_failure;
8531
8532         if (event == NFT_MSG_DELFLOWTABLE && !hook_list) {
8533                 nlmsg_end(skb, nlh);
8534                 return 0;
8535         }
8536
8537         if (nla_put_be32(skb, NFTA_FLOWTABLE_USE, htonl(flowtable->use)) ||
8538             nla_put_be32(skb, NFTA_FLOWTABLE_FLAGS, htonl(flowtable->data.flags)))
8539                 goto nla_put_failure;
8540
8541         nest = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK);
8542         if (!nest)
8543                 goto nla_put_failure;
8544         if (nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_NUM, htonl(flowtable->hooknum)) ||
8545             nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_PRIORITY, htonl(flowtable->data.priority)))
8546                 goto nla_put_failure;
8547
8548         nest_devs = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK_DEVS);
8549         if (!nest_devs)
8550                 goto nla_put_failure;
8551
8552         if (!hook_list)
8553                 hook_list = &flowtable->hook_list;
8554
8555         list_for_each_entry_rcu(hook, hook_list, list) {
8556                 if (nla_put_string(skb, NFTA_DEVICE_NAME, hook->ops.dev->name))
8557                         goto nla_put_failure;
8558         }
8559         nla_nest_end(skb, nest_devs);
8560         nla_nest_end(skb, nest);
8561
8562         nlmsg_end(skb, nlh);
8563         return 0;
8564
8565 nla_put_failure:
8566         nlmsg_trim(skb, nlh);
8567         return -1;
8568 }
8569
8570 struct nft_flowtable_filter {
8571         char            *table;
8572 };
8573
8574 static int nf_tables_dump_flowtable(struct sk_buff *skb,
8575                                     struct netlink_callback *cb)
8576 {
8577         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
8578         struct nft_flowtable_filter *filter = cb->data;
8579         unsigned int idx = 0, s_idx = cb->args[0];
8580         struct net *net = sock_net(skb->sk);
8581         int family = nfmsg->nfgen_family;
8582         struct nft_flowtable *flowtable;
8583         struct nftables_pernet *nft_net;
8584         const struct nft_table *table;
8585
8586         rcu_read_lock();
8587         nft_net = nft_pernet(net);
8588         cb->seq = READ_ONCE(nft_net->base_seq);
8589
8590         list_for_each_entry_rcu(table, &nft_net->tables, list) {
8591                 if (family != NFPROTO_UNSPEC && family != table->family)
8592                         continue;
8593
8594                 list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
8595                         if (!nft_is_active(net, flowtable))
8596                                 goto cont;
8597                         if (idx < s_idx)
8598                                 goto cont;
8599                         if (idx > s_idx)
8600                                 memset(&cb->args[1], 0,
8601                                        sizeof(cb->args) - sizeof(cb->args[0]));
8602                         if (filter && filter->table &&
8603                             strcmp(filter->table, table->name))
8604                                 goto cont;
8605
8606                         if (nf_tables_fill_flowtable_info(skb, net, NETLINK_CB(cb->skb).portid,
8607                                                           cb->nlh->nlmsg_seq,
8608                                                           NFT_MSG_NEWFLOWTABLE,
8609                                                           NLM_F_MULTI | NLM_F_APPEND,
8610                                                           table->family,
8611                                                           flowtable, NULL) < 0)
8612                                 goto done;
8613
8614                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
8615 cont:
8616                         idx++;
8617                 }
8618         }
8619 done:
8620         rcu_read_unlock();
8621
8622         cb->args[0] = idx;
8623         return skb->len;
8624 }
8625
8626 static int nf_tables_dump_flowtable_start(struct netlink_callback *cb)
8627 {
8628         const struct nlattr * const *nla = cb->data;
8629         struct nft_flowtable_filter *filter = NULL;
8630
8631         if (nla[NFTA_FLOWTABLE_TABLE]) {
8632                 filter = kzalloc(sizeof(*filter), GFP_ATOMIC);
8633                 if (!filter)
8634                         return -ENOMEM;
8635
8636                 filter->table = nla_strdup(nla[NFTA_FLOWTABLE_TABLE],
8637                                            GFP_ATOMIC);
8638                 if (!filter->table) {
8639                         kfree(filter);
8640                         return -ENOMEM;
8641                 }
8642         }
8643
8644         cb->data = filter;
8645         return 0;
8646 }
8647
8648 static int nf_tables_dump_flowtable_done(struct netlink_callback *cb)
8649 {
8650         struct nft_flowtable_filter *filter = cb->data;
8651
8652         if (!filter)
8653                 return 0;
8654
8655         kfree(filter->table);
8656         kfree(filter);
8657
8658         return 0;
8659 }
8660
8661 /* called with rcu_read_lock held */
8662 static int nf_tables_getflowtable(struct sk_buff *skb,
8663                                   const struct nfnl_info *info,
8664                                   const struct nlattr * const nla[])
8665 {
8666         u8 genmask = nft_genmask_cur(info->net);
8667         u8 family = info->nfmsg->nfgen_family;
8668         struct nft_flowtable *flowtable;
8669         const struct nft_table *table;
8670         struct net *net = info->net;
8671         struct sk_buff *skb2;
8672         int err;
8673
8674         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
8675                 struct netlink_dump_control c = {
8676                         .start = nf_tables_dump_flowtable_start,
8677                         .dump = nf_tables_dump_flowtable,
8678                         .done = nf_tables_dump_flowtable_done,
8679                         .module = THIS_MODULE,
8680                         .data = (void *)nla,
8681                 };
8682
8683                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
8684         }
8685
8686         if (!nla[NFTA_FLOWTABLE_NAME])
8687                 return -EINVAL;
8688
8689         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8690                                  genmask, 0);
8691         if (IS_ERR(table))
8692                 return PTR_ERR(table);
8693
8694         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
8695                                          genmask);
8696         if (IS_ERR(flowtable))
8697                 return PTR_ERR(flowtable);
8698
8699         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
8700         if (!skb2)
8701                 return -ENOMEM;
8702
8703         err = nf_tables_fill_flowtable_info(skb2, net, NETLINK_CB(skb).portid,
8704                                             info->nlh->nlmsg_seq,
8705                                             NFT_MSG_NEWFLOWTABLE, 0, family,
8706                                             flowtable, NULL);
8707         if (err < 0)
8708                 goto err_fill_flowtable_info;
8709
8710         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
8711
8712 err_fill_flowtable_info:
8713         kfree_skb(skb2);
8714         return err;
8715 }
8716
8717 static void nf_tables_flowtable_notify(struct nft_ctx *ctx,
8718                                        struct nft_flowtable *flowtable,
8719                                        struct list_head *hook_list, int event)
8720 {
8721         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
8722         struct sk_buff *skb;
8723         u16 flags = 0;
8724         int err;
8725
8726         if (!ctx->report &&
8727             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
8728                 return;
8729
8730         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
8731         if (skb == NULL)
8732                 goto err;
8733
8734         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
8735                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
8736
8737         err = nf_tables_fill_flowtable_info(skb, ctx->net, ctx->portid,
8738                                             ctx->seq, event, flags,
8739                                             ctx->family, flowtable, hook_list);
8740         if (err < 0) {
8741                 kfree_skb(skb);
8742                 goto err;
8743         }
8744
8745         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
8746         return;
8747 err:
8748         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
8749 }
8750
8751 static void nf_tables_flowtable_destroy(struct nft_flowtable *flowtable)
8752 {
8753         struct nft_hook *hook, *next;
8754
8755         flowtable->data.type->free(&flowtable->data);
8756         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
8757                 flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
8758                                             FLOW_BLOCK_UNBIND);
8759                 list_del_rcu(&hook->list);
8760                 kfree(hook);
8761         }
8762         kfree(flowtable->name);
8763         module_put(flowtable->data.type->owner);
8764         kfree(flowtable);
8765 }
8766
8767 static int nf_tables_fill_gen_info(struct sk_buff *skb, struct net *net,
8768                                    u32 portid, u32 seq)
8769 {
8770         struct nftables_pernet *nft_net = nft_pernet(net);
8771         struct nlmsghdr *nlh;
8772         char buf[TASK_COMM_LEN];
8773         int event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWGEN);
8774
8775         nlh = nfnl_msg_put(skb, portid, seq, event, 0, AF_UNSPEC,
8776                            NFNETLINK_V0, nft_base_seq(net));
8777         if (!nlh)
8778                 goto nla_put_failure;
8779
8780         if (nla_put_be32(skb, NFTA_GEN_ID, htonl(nft_net->base_seq)) ||
8781             nla_put_be32(skb, NFTA_GEN_PROC_PID, htonl(task_pid_nr(current))) ||
8782             nla_put_string(skb, NFTA_GEN_PROC_NAME, get_task_comm(buf, current)))
8783                 goto nla_put_failure;
8784
8785         nlmsg_end(skb, nlh);
8786         return 0;
8787
8788 nla_put_failure:
8789         nlmsg_trim(skb, nlh);
8790         return -EMSGSIZE;
8791 }
8792
8793 static void nft_flowtable_event(unsigned long event, struct net_device *dev,
8794                                 struct nft_flowtable *flowtable)
8795 {
8796         struct nft_hook *hook;
8797
8798         list_for_each_entry(hook, &flowtable->hook_list, list) {
8799                 if (hook->ops.dev != dev)
8800                         continue;
8801
8802                 /* flow_offload_netdev_event() cleans up entries for us. */
8803                 nft_unregister_flowtable_hook(dev_net(dev), flowtable, hook);
8804                 list_del_rcu(&hook->list);
8805                 kfree_rcu(hook, rcu);
8806                 break;
8807         }
8808 }
8809
8810 static int nf_tables_flowtable_event(struct notifier_block *this,
8811                                      unsigned long event, void *ptr)
8812 {
8813         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
8814         struct nft_flowtable *flowtable;
8815         struct nftables_pernet *nft_net;
8816         struct nft_table *table;
8817         struct net *net;
8818
8819         if (event != NETDEV_UNREGISTER)
8820                 return 0;
8821
8822         net = dev_net(dev);
8823         nft_net = nft_pernet(net);
8824         mutex_lock(&nft_net->commit_mutex);
8825         list_for_each_entry(table, &nft_net->tables, list) {
8826                 list_for_each_entry(flowtable, &table->flowtables, list) {
8827                         nft_flowtable_event(event, dev, flowtable);
8828                 }
8829         }
8830         mutex_unlock(&nft_net->commit_mutex);
8831
8832         return NOTIFY_DONE;
8833 }
8834
8835 static struct notifier_block nf_tables_flowtable_notifier = {
8836         .notifier_call  = nf_tables_flowtable_event,
8837 };
8838
8839 static void nf_tables_gen_notify(struct net *net, struct sk_buff *skb,
8840                                  int event)
8841 {
8842         struct nlmsghdr *nlh = nlmsg_hdr(skb);
8843         struct sk_buff *skb2;
8844         int err;
8845
8846         if (!nlmsg_report(nlh) &&
8847             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
8848                 return;
8849
8850         skb2 = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
8851         if (skb2 == NULL)
8852                 goto err;
8853
8854         err = nf_tables_fill_gen_info(skb2, net, NETLINK_CB(skb).portid,
8855                                       nlh->nlmsg_seq);
8856         if (err < 0) {
8857                 kfree_skb(skb2);
8858                 goto err;
8859         }
8860
8861         nfnetlink_send(skb2, net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
8862                        nlmsg_report(nlh), GFP_KERNEL);
8863         return;
8864 err:
8865         nfnetlink_set_err(net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
8866                           -ENOBUFS);
8867 }
8868
8869 static int nf_tables_getgen(struct sk_buff *skb, const struct nfnl_info *info,
8870                             const struct nlattr * const nla[])
8871 {
8872         struct sk_buff *skb2;
8873         int err;
8874
8875         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
8876         if (skb2 == NULL)
8877                 return -ENOMEM;
8878
8879         err = nf_tables_fill_gen_info(skb2, info->net, NETLINK_CB(skb).portid,
8880                                       info->nlh->nlmsg_seq);
8881         if (err < 0)
8882                 goto err_fill_gen_info;
8883
8884         return nfnetlink_unicast(skb2, info->net, NETLINK_CB(skb).portid);
8885
8886 err_fill_gen_info:
8887         kfree_skb(skb2);
8888         return err;
8889 }
8890
8891 static const struct nfnl_callback nf_tables_cb[NFT_MSG_MAX] = {
8892         [NFT_MSG_NEWTABLE] = {
8893                 .call           = nf_tables_newtable,
8894                 .type           = NFNL_CB_BATCH,
8895                 .attr_count     = NFTA_TABLE_MAX,
8896                 .policy         = nft_table_policy,
8897         },
8898         [NFT_MSG_GETTABLE] = {
8899                 .call           = nf_tables_gettable,
8900                 .type           = NFNL_CB_RCU,
8901                 .attr_count     = NFTA_TABLE_MAX,
8902                 .policy         = nft_table_policy,
8903         },
8904         [NFT_MSG_DELTABLE] = {
8905                 .call           = nf_tables_deltable,
8906                 .type           = NFNL_CB_BATCH,
8907                 .attr_count     = NFTA_TABLE_MAX,
8908                 .policy         = nft_table_policy,
8909         },
8910         [NFT_MSG_DESTROYTABLE] = {
8911                 .call           = nf_tables_deltable,
8912                 .type           = NFNL_CB_BATCH,
8913                 .attr_count     = NFTA_TABLE_MAX,
8914                 .policy         = nft_table_policy,
8915         },
8916         [NFT_MSG_NEWCHAIN] = {
8917                 .call           = nf_tables_newchain,
8918                 .type           = NFNL_CB_BATCH,
8919                 .attr_count     = NFTA_CHAIN_MAX,
8920                 .policy         = nft_chain_policy,
8921         },
8922         [NFT_MSG_GETCHAIN] = {
8923                 .call           = nf_tables_getchain,
8924                 .type           = NFNL_CB_RCU,
8925                 .attr_count     = NFTA_CHAIN_MAX,
8926                 .policy         = nft_chain_policy,
8927         },
8928         [NFT_MSG_DELCHAIN] = {
8929                 .call           = nf_tables_delchain,
8930                 .type           = NFNL_CB_BATCH,
8931                 .attr_count     = NFTA_CHAIN_MAX,
8932                 .policy         = nft_chain_policy,
8933         },
8934         [NFT_MSG_DESTROYCHAIN] = {
8935                 .call           = nf_tables_delchain,
8936                 .type           = NFNL_CB_BATCH,
8937                 .attr_count     = NFTA_CHAIN_MAX,
8938                 .policy         = nft_chain_policy,
8939         },
8940         [NFT_MSG_NEWRULE] = {
8941                 .call           = nf_tables_newrule,
8942                 .type           = NFNL_CB_BATCH,
8943                 .attr_count     = NFTA_RULE_MAX,
8944                 .policy         = nft_rule_policy,
8945         },
8946         [NFT_MSG_GETRULE] = {
8947                 .call           = nf_tables_getrule,
8948                 .type           = NFNL_CB_RCU,
8949                 .attr_count     = NFTA_RULE_MAX,
8950                 .policy         = nft_rule_policy,
8951         },
8952         [NFT_MSG_GETRULE_RESET] = {
8953                 .call           = nf_tables_getrule,
8954                 .type           = NFNL_CB_RCU,
8955                 .attr_count     = NFTA_RULE_MAX,
8956                 .policy         = nft_rule_policy,
8957         },
8958         [NFT_MSG_DELRULE] = {
8959                 .call           = nf_tables_delrule,
8960                 .type           = NFNL_CB_BATCH,
8961                 .attr_count     = NFTA_RULE_MAX,
8962                 .policy         = nft_rule_policy,
8963         },
8964         [NFT_MSG_DESTROYRULE] = {
8965                 .call           = nf_tables_delrule,
8966                 .type           = NFNL_CB_BATCH,
8967                 .attr_count     = NFTA_RULE_MAX,
8968                 .policy         = nft_rule_policy,
8969         },
8970         [NFT_MSG_NEWSET] = {
8971                 .call           = nf_tables_newset,
8972                 .type           = NFNL_CB_BATCH,
8973                 .attr_count     = NFTA_SET_MAX,
8974                 .policy         = nft_set_policy,
8975         },
8976         [NFT_MSG_GETSET] = {
8977                 .call           = nf_tables_getset,
8978                 .type           = NFNL_CB_RCU,
8979                 .attr_count     = NFTA_SET_MAX,
8980                 .policy         = nft_set_policy,
8981         },
8982         [NFT_MSG_DELSET] = {
8983                 .call           = nf_tables_delset,
8984                 .type           = NFNL_CB_BATCH,
8985                 .attr_count     = NFTA_SET_MAX,
8986                 .policy         = nft_set_policy,
8987         },
8988         [NFT_MSG_DESTROYSET] = {
8989                 .call           = nf_tables_delset,
8990                 .type           = NFNL_CB_BATCH,
8991                 .attr_count     = NFTA_SET_MAX,
8992                 .policy         = nft_set_policy,
8993         },
8994         [NFT_MSG_NEWSETELEM] = {
8995                 .call           = nf_tables_newsetelem,
8996                 .type           = NFNL_CB_BATCH,
8997                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
8998                 .policy         = nft_set_elem_list_policy,
8999         },
9000         [NFT_MSG_GETSETELEM] = {
9001                 .call           = nf_tables_getsetelem,
9002                 .type           = NFNL_CB_RCU,
9003                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9004                 .policy         = nft_set_elem_list_policy,
9005         },
9006         [NFT_MSG_GETSETELEM_RESET] = {
9007                 .call           = nf_tables_getsetelem,
9008                 .type           = NFNL_CB_RCU,
9009                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9010                 .policy         = nft_set_elem_list_policy,
9011         },
9012         [NFT_MSG_DELSETELEM] = {
9013                 .call           = nf_tables_delsetelem,
9014                 .type           = NFNL_CB_BATCH,
9015                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9016                 .policy         = nft_set_elem_list_policy,
9017         },
9018         [NFT_MSG_DESTROYSETELEM] = {
9019                 .call           = nf_tables_delsetelem,
9020                 .type           = NFNL_CB_BATCH,
9021                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9022                 .policy         = nft_set_elem_list_policy,
9023         },
9024         [NFT_MSG_GETGEN] = {
9025                 .call           = nf_tables_getgen,
9026                 .type           = NFNL_CB_RCU,
9027         },
9028         [NFT_MSG_NEWOBJ] = {
9029                 .call           = nf_tables_newobj,
9030                 .type           = NFNL_CB_BATCH,
9031                 .attr_count     = NFTA_OBJ_MAX,
9032                 .policy         = nft_obj_policy,
9033         },
9034         [NFT_MSG_GETOBJ] = {
9035                 .call           = nf_tables_getobj,
9036                 .type           = NFNL_CB_RCU,
9037                 .attr_count     = NFTA_OBJ_MAX,
9038                 .policy         = nft_obj_policy,
9039         },
9040         [NFT_MSG_DELOBJ] = {
9041                 .call           = nf_tables_delobj,
9042                 .type           = NFNL_CB_BATCH,
9043                 .attr_count     = NFTA_OBJ_MAX,
9044                 .policy         = nft_obj_policy,
9045         },
9046         [NFT_MSG_DESTROYOBJ] = {
9047                 .call           = nf_tables_delobj,
9048                 .type           = NFNL_CB_BATCH,
9049                 .attr_count     = NFTA_OBJ_MAX,
9050                 .policy         = nft_obj_policy,
9051         },
9052         [NFT_MSG_GETOBJ_RESET] = {
9053                 .call           = nf_tables_getobj,
9054                 .type           = NFNL_CB_RCU,
9055                 .attr_count     = NFTA_OBJ_MAX,
9056                 .policy         = nft_obj_policy,
9057         },
9058         [NFT_MSG_NEWFLOWTABLE] = {
9059                 .call           = nf_tables_newflowtable,
9060                 .type           = NFNL_CB_BATCH,
9061                 .attr_count     = NFTA_FLOWTABLE_MAX,
9062                 .policy         = nft_flowtable_policy,
9063         },
9064         [NFT_MSG_GETFLOWTABLE] = {
9065                 .call           = nf_tables_getflowtable,
9066                 .type           = NFNL_CB_RCU,
9067                 .attr_count     = NFTA_FLOWTABLE_MAX,
9068                 .policy         = nft_flowtable_policy,
9069         },
9070         [NFT_MSG_DELFLOWTABLE] = {
9071                 .call           = nf_tables_delflowtable,
9072                 .type           = NFNL_CB_BATCH,
9073                 .attr_count     = NFTA_FLOWTABLE_MAX,
9074                 .policy         = nft_flowtable_policy,
9075         },
9076         [NFT_MSG_DESTROYFLOWTABLE] = {
9077                 .call           = nf_tables_delflowtable,
9078                 .type           = NFNL_CB_BATCH,
9079                 .attr_count     = NFTA_FLOWTABLE_MAX,
9080                 .policy         = nft_flowtable_policy,
9081         },
9082 };
9083
9084 static int nf_tables_validate(struct net *net)
9085 {
9086         struct nftables_pernet *nft_net = nft_pernet(net);
9087         struct nft_table *table;
9088
9089         list_for_each_entry(table, &nft_net->tables, list) {
9090                 switch (table->validate_state) {
9091                 case NFT_VALIDATE_SKIP:
9092                         continue;
9093                 case NFT_VALIDATE_NEED:
9094                         nft_validate_state_update(table, NFT_VALIDATE_DO);
9095                         fallthrough;
9096                 case NFT_VALIDATE_DO:
9097                         if (nft_table_validate(net, table) < 0)
9098                                 return -EAGAIN;
9099
9100                         nft_validate_state_update(table, NFT_VALIDATE_SKIP);
9101                         break;
9102                 }
9103         }
9104
9105         return 0;
9106 }
9107
9108 /* a drop policy has to be deferred until all rules have been activated,
9109  * otherwise a large ruleset that contains a drop-policy base chain will
9110  * cause all packets to get dropped until the full transaction has been
9111  * processed.
9112  *
9113  * We defer the drop policy until the transaction has been finalized.
9114  */
9115 static void nft_chain_commit_drop_policy(struct nft_trans *trans)
9116 {
9117         struct nft_base_chain *basechain;
9118
9119         if (nft_trans_chain_policy(trans) != NF_DROP)
9120                 return;
9121
9122         if (!nft_is_base_chain(trans->ctx.chain))
9123                 return;
9124
9125         basechain = nft_base_chain(trans->ctx.chain);
9126         basechain->policy = NF_DROP;
9127 }
9128
9129 static void nft_chain_commit_update(struct nft_trans *trans)
9130 {
9131         struct nft_base_chain *basechain;
9132
9133         if (nft_trans_chain_name(trans)) {
9134                 rhltable_remove(&trans->ctx.table->chains_ht,
9135                                 &trans->ctx.chain->rhlhead,
9136                                 nft_chain_ht_params);
9137                 swap(trans->ctx.chain->name, nft_trans_chain_name(trans));
9138                 rhltable_insert_key(&trans->ctx.table->chains_ht,
9139                                     trans->ctx.chain->name,
9140                                     &trans->ctx.chain->rhlhead,
9141                                     nft_chain_ht_params);
9142         }
9143
9144         if (!nft_is_base_chain(trans->ctx.chain))
9145                 return;
9146
9147         nft_chain_stats_replace(trans);
9148
9149         basechain = nft_base_chain(trans->ctx.chain);
9150
9151         switch (nft_trans_chain_policy(trans)) {
9152         case NF_DROP:
9153         case NF_ACCEPT:
9154                 basechain->policy = nft_trans_chain_policy(trans);
9155                 break;
9156         }
9157 }
9158
9159 static void nft_obj_commit_update(struct nft_trans *trans)
9160 {
9161         struct nft_object *newobj;
9162         struct nft_object *obj;
9163
9164         obj = nft_trans_obj(trans);
9165         newobj = nft_trans_obj_newobj(trans);
9166
9167         if (obj->ops->update)
9168                 obj->ops->update(obj, newobj);
9169
9170         nft_obj_destroy(&trans->ctx, newobj);
9171 }
9172
9173 static void nft_commit_release(struct nft_trans *trans)
9174 {
9175         switch (trans->msg_type) {
9176         case NFT_MSG_DELTABLE:
9177         case NFT_MSG_DESTROYTABLE:
9178                 nf_tables_table_destroy(&trans->ctx);
9179                 break;
9180         case NFT_MSG_NEWCHAIN:
9181                 free_percpu(nft_trans_chain_stats(trans));
9182                 kfree(nft_trans_chain_name(trans));
9183                 break;
9184         case NFT_MSG_DELCHAIN:
9185         case NFT_MSG_DESTROYCHAIN:
9186                 if (nft_trans_chain_update(trans))
9187                         nft_hooks_destroy(&nft_trans_chain_hooks(trans));
9188                 else
9189                         nf_tables_chain_destroy(&trans->ctx);
9190                 break;
9191         case NFT_MSG_DELRULE:
9192         case NFT_MSG_DESTROYRULE:
9193                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
9194                 break;
9195         case NFT_MSG_DELSET:
9196         case NFT_MSG_DESTROYSET:
9197                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
9198                 break;
9199         case NFT_MSG_DELSETELEM:
9200         case NFT_MSG_DESTROYSETELEM:
9201                 nf_tables_set_elem_destroy(&trans->ctx,
9202                                            nft_trans_elem_set(trans),
9203                                            nft_trans_elem(trans).priv);
9204                 break;
9205         case NFT_MSG_DELOBJ:
9206         case NFT_MSG_DESTROYOBJ:
9207                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
9208                 break;
9209         case NFT_MSG_DELFLOWTABLE:
9210         case NFT_MSG_DESTROYFLOWTABLE:
9211                 if (nft_trans_flowtable_update(trans))
9212                         nft_hooks_destroy(&nft_trans_flowtable_hooks(trans));
9213                 else
9214                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
9215                 break;
9216         }
9217
9218         if (trans->put_net)
9219                 put_net(trans->ctx.net);
9220
9221         kfree(trans);
9222 }
9223
9224 static void nf_tables_trans_destroy_work(struct work_struct *w)
9225 {
9226         struct nft_trans *trans, *next;
9227         LIST_HEAD(head);
9228
9229         spin_lock(&nf_tables_destroy_list_lock);
9230         list_splice_init(&nf_tables_destroy_list, &head);
9231         spin_unlock(&nf_tables_destroy_list_lock);
9232
9233         if (list_empty(&head))
9234                 return;
9235
9236         synchronize_rcu();
9237
9238         list_for_each_entry_safe(trans, next, &head, list) {
9239                 nft_trans_list_del(trans);
9240                 nft_commit_release(trans);
9241         }
9242 }
9243
9244 void nf_tables_trans_destroy_flush_work(void)
9245 {
9246         flush_work(&trans_destroy_work);
9247 }
9248 EXPORT_SYMBOL_GPL(nf_tables_trans_destroy_flush_work);
9249
9250 static bool nft_expr_reduce(struct nft_regs_track *track,
9251                             const struct nft_expr *expr)
9252 {
9253         return false;
9254 }
9255
9256 static int nf_tables_commit_chain_prepare(struct net *net, struct nft_chain *chain)
9257 {
9258         const struct nft_expr *expr, *last;
9259         struct nft_regs_track track = {};
9260         unsigned int size, data_size;
9261         void *data, *data_boundary;
9262         struct nft_rule_dp *prule;
9263         struct nft_rule *rule;
9264
9265         /* already handled or inactive chain? */
9266         if (chain->blob_next || !nft_is_active_next(net, chain))
9267                 return 0;
9268
9269         data_size = 0;
9270         list_for_each_entry(rule, &chain->rules, list) {
9271                 if (nft_is_active_next(net, rule)) {
9272                         data_size += sizeof(*prule) + rule->dlen;
9273                         if (data_size > INT_MAX)
9274                                 return -ENOMEM;
9275                 }
9276         }
9277
9278         chain->blob_next = nf_tables_chain_alloc_rules(chain, data_size);
9279         if (!chain->blob_next)
9280                 return -ENOMEM;
9281
9282         data = (void *)chain->blob_next->data;
9283         data_boundary = data + data_size;
9284         size = 0;
9285
9286         list_for_each_entry(rule, &chain->rules, list) {
9287                 if (!nft_is_active_next(net, rule))
9288                         continue;
9289
9290                 prule = (struct nft_rule_dp *)data;
9291                 data += offsetof(struct nft_rule_dp, data);
9292                 if (WARN_ON_ONCE(data > data_boundary))
9293                         return -ENOMEM;
9294
9295                 size = 0;
9296                 track.last = nft_expr_last(rule);
9297                 nft_rule_for_each_expr(expr, last, rule) {
9298                         track.cur = expr;
9299
9300                         if (nft_expr_reduce(&track, expr)) {
9301                                 expr = track.cur;
9302                                 continue;
9303                         }
9304
9305                         if (WARN_ON_ONCE(data + size + expr->ops->size > data_boundary))
9306                                 return -ENOMEM;
9307
9308                         memcpy(data + size, expr, expr->ops->size);
9309                         size += expr->ops->size;
9310                 }
9311                 if (WARN_ON_ONCE(size >= 1 << 12))
9312                         return -ENOMEM;
9313
9314                 prule->handle = rule->handle;
9315                 prule->dlen = size;
9316                 prule->is_last = 0;
9317
9318                 data += size;
9319                 size = 0;
9320                 chain->blob_next->size += (unsigned long)(data - (void *)prule);
9321         }
9322
9323         if (WARN_ON_ONCE(data > data_boundary))
9324                 return -ENOMEM;
9325
9326         prule = (struct nft_rule_dp *)data;
9327         nft_last_rule(chain, prule);
9328
9329         return 0;
9330 }
9331
9332 static void nf_tables_commit_chain_prepare_cancel(struct net *net)
9333 {
9334         struct nftables_pernet *nft_net = nft_pernet(net);
9335         struct nft_trans *trans, *next;
9336
9337         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
9338                 struct nft_chain *chain = trans->ctx.chain;
9339
9340                 if (trans->msg_type == NFT_MSG_NEWRULE ||
9341                     trans->msg_type == NFT_MSG_DELRULE) {
9342                         kvfree(chain->blob_next);
9343                         chain->blob_next = NULL;
9344                 }
9345         }
9346 }
9347
9348 static void __nf_tables_commit_chain_free_rules(struct rcu_head *h)
9349 {
9350         struct nft_rule_dp_last *l = container_of(h, struct nft_rule_dp_last, h);
9351
9352         kvfree(l->blob);
9353 }
9354
9355 static void nf_tables_commit_chain_free_rules_old(struct nft_rule_blob *blob)
9356 {
9357         struct nft_rule_dp_last *last;
9358
9359         /* last rule trailer is after end marker */
9360         last = (void *)blob + sizeof(*blob) + blob->size;
9361         last->blob = blob;
9362
9363         call_rcu(&last->h, __nf_tables_commit_chain_free_rules);
9364 }
9365
9366 static void nf_tables_commit_chain(struct net *net, struct nft_chain *chain)
9367 {
9368         struct nft_rule_blob *g0, *g1;
9369         bool next_genbit;
9370
9371         next_genbit = nft_gencursor_next(net);
9372
9373         g0 = rcu_dereference_protected(chain->blob_gen_0,
9374                                        lockdep_commit_lock_is_held(net));
9375         g1 = rcu_dereference_protected(chain->blob_gen_1,
9376                                        lockdep_commit_lock_is_held(net));
9377
9378         /* No changes to this chain? */
9379         if (chain->blob_next == NULL) {
9380                 /* chain had no change in last or next generation */
9381                 if (g0 == g1)
9382                         return;
9383                 /*
9384                  * chain had no change in this generation; make sure next
9385                  * one uses same rules as current generation.
9386                  */
9387                 if (next_genbit) {
9388                         rcu_assign_pointer(chain->blob_gen_1, g0);
9389                         nf_tables_commit_chain_free_rules_old(g1);
9390                 } else {
9391                         rcu_assign_pointer(chain->blob_gen_0, g1);
9392                         nf_tables_commit_chain_free_rules_old(g0);
9393                 }
9394
9395                 return;
9396         }
9397
9398         if (next_genbit)
9399                 rcu_assign_pointer(chain->blob_gen_1, chain->blob_next);
9400         else
9401                 rcu_assign_pointer(chain->blob_gen_0, chain->blob_next);
9402
9403         chain->blob_next = NULL;
9404
9405         if (g0 == g1)
9406                 return;
9407
9408         if (next_genbit)
9409                 nf_tables_commit_chain_free_rules_old(g1);
9410         else
9411                 nf_tables_commit_chain_free_rules_old(g0);
9412 }
9413
9414 static void nft_obj_del(struct nft_object *obj)
9415 {
9416         rhltable_remove(&nft_objname_ht, &obj->rhlhead, nft_objname_ht_params);
9417         list_del_rcu(&obj->list);
9418 }
9419
9420 void nft_chain_del(struct nft_chain *chain)
9421 {
9422         struct nft_table *table = chain->table;
9423
9424         WARN_ON_ONCE(rhltable_remove(&table->chains_ht, &chain->rhlhead,
9425                                      nft_chain_ht_params));
9426         list_del_rcu(&chain->list);
9427 }
9428
9429 static void nft_trans_gc_setelem_remove(struct nft_ctx *ctx,
9430                                         struct nft_trans_gc *trans)
9431 {
9432         void **priv = trans->priv;
9433         unsigned int i;
9434
9435         for (i = 0; i < trans->count; i++) {
9436                 struct nft_set_elem elem = {
9437                         .priv = priv[i],
9438                 };
9439
9440                 nft_setelem_data_deactivate(ctx->net, trans->set, &elem);
9441                 nft_setelem_remove(ctx->net, trans->set, &elem);
9442         }
9443 }
9444
9445 void nft_trans_gc_destroy(struct nft_trans_gc *trans)
9446 {
9447         nft_set_put(trans->set);
9448         put_net(trans->net);
9449         kfree(trans);
9450 }
9451
9452 static void nft_trans_gc_trans_free(struct rcu_head *rcu)
9453 {
9454         struct nft_set_elem elem = {};
9455         struct nft_trans_gc *trans;
9456         struct nft_ctx ctx = {};
9457         unsigned int i;
9458
9459         trans = container_of(rcu, struct nft_trans_gc, rcu);
9460         ctx.net = read_pnet(&trans->set->net);
9461
9462         for (i = 0; i < trans->count; i++) {
9463                 elem.priv = trans->priv[i];
9464                 if (!nft_setelem_is_catchall(trans->set, &elem))
9465                         atomic_dec(&trans->set->nelems);
9466
9467                 nf_tables_set_elem_destroy(&ctx, trans->set, elem.priv);
9468         }
9469
9470         nft_trans_gc_destroy(trans);
9471 }
9472
9473 static bool nft_trans_gc_work_done(struct nft_trans_gc *trans)
9474 {
9475         struct nftables_pernet *nft_net;
9476         struct nft_ctx ctx = {};
9477
9478         nft_net = nft_pernet(trans->net);
9479
9480         mutex_lock(&nft_net->commit_mutex);
9481
9482         /* Check for race with transaction, otherwise this batch refers to
9483          * stale objects that might not be there anymore. Skip transaction if
9484          * set has been destroyed from control plane transaction in case gc
9485          * worker loses race.
9486          */
9487         if (READ_ONCE(nft_net->gc_seq) != trans->seq || trans->set->dead) {
9488                 mutex_unlock(&nft_net->commit_mutex);
9489                 return false;
9490         }
9491
9492         ctx.net = trans->net;
9493         ctx.table = trans->set->table;
9494
9495         nft_trans_gc_setelem_remove(&ctx, trans);
9496         mutex_unlock(&nft_net->commit_mutex);
9497
9498         return true;
9499 }
9500
9501 static void nft_trans_gc_work(struct work_struct *work)
9502 {
9503         struct nft_trans_gc *trans, *next;
9504         LIST_HEAD(trans_gc_list);
9505
9506         spin_lock(&nf_tables_gc_list_lock);
9507         list_splice_init(&nf_tables_gc_list, &trans_gc_list);
9508         spin_unlock(&nf_tables_gc_list_lock);
9509
9510         list_for_each_entry_safe(trans, next, &trans_gc_list, list) {
9511                 list_del(&trans->list);
9512                 if (!nft_trans_gc_work_done(trans)) {
9513                         nft_trans_gc_destroy(trans);
9514                         continue;
9515                 }
9516                 call_rcu(&trans->rcu, nft_trans_gc_trans_free);
9517         }
9518 }
9519
9520 struct nft_trans_gc *nft_trans_gc_alloc(struct nft_set *set,
9521                                         unsigned int gc_seq, gfp_t gfp)
9522 {
9523         struct net *net = read_pnet(&set->net);
9524         struct nft_trans_gc *trans;
9525
9526         trans = kzalloc(sizeof(*trans), gfp);
9527         if (!trans)
9528                 return NULL;
9529
9530         trans->net = maybe_get_net(net);
9531         if (!trans->net) {
9532                 kfree(trans);
9533                 return NULL;
9534         }
9535
9536         refcount_inc(&set->refs);
9537         trans->set = set;
9538         trans->seq = gc_seq;
9539
9540         return trans;
9541 }
9542
9543 void nft_trans_gc_elem_add(struct nft_trans_gc *trans, void *priv)
9544 {
9545         trans->priv[trans->count++] = priv;
9546 }
9547
9548 static void nft_trans_gc_queue_work(struct nft_trans_gc *trans)
9549 {
9550         spin_lock(&nf_tables_gc_list_lock);
9551         list_add_tail(&trans->list, &nf_tables_gc_list);
9552         spin_unlock(&nf_tables_gc_list_lock);
9553
9554         schedule_work(&trans_gc_work);
9555 }
9556
9557 static int nft_trans_gc_space(struct nft_trans_gc *trans)
9558 {
9559         return NFT_TRANS_GC_BATCHCOUNT - trans->count;
9560 }
9561
9562 struct nft_trans_gc *nft_trans_gc_queue_async(struct nft_trans_gc *gc,
9563                                               unsigned int gc_seq, gfp_t gfp)
9564 {
9565         if (nft_trans_gc_space(gc))
9566                 return gc;
9567
9568         nft_trans_gc_queue_work(gc);
9569
9570         return nft_trans_gc_alloc(gc->set, gc_seq, gfp);
9571 }
9572
9573 void nft_trans_gc_queue_async_done(struct nft_trans_gc *trans)
9574 {
9575         if (trans->count == 0) {
9576                 nft_trans_gc_destroy(trans);
9577                 return;
9578         }
9579
9580         nft_trans_gc_queue_work(trans);
9581 }
9582
9583 struct nft_trans_gc *nft_trans_gc_queue_sync(struct nft_trans_gc *gc, gfp_t gfp)
9584 {
9585         if (WARN_ON_ONCE(!lockdep_commit_lock_is_held(gc->net)))
9586                 return NULL;
9587
9588         if (nft_trans_gc_space(gc))
9589                 return gc;
9590
9591         call_rcu(&gc->rcu, nft_trans_gc_trans_free);
9592
9593         return nft_trans_gc_alloc(gc->set, 0, gfp);
9594 }
9595
9596 void nft_trans_gc_queue_sync_done(struct nft_trans_gc *trans)
9597 {
9598         WARN_ON_ONCE(!lockdep_commit_lock_is_held(trans->net));
9599
9600         if (trans->count == 0) {
9601                 nft_trans_gc_destroy(trans);
9602                 return;
9603         }
9604
9605         call_rcu(&trans->rcu, nft_trans_gc_trans_free);
9606 }
9607
9608 struct nft_trans_gc *nft_trans_gc_catchall(struct nft_trans_gc *gc,
9609                                            unsigned int gc_seq)
9610 {
9611         struct nft_set_elem_catchall *catchall;
9612         const struct nft_set *set = gc->set;
9613         struct nft_set_ext *ext;
9614
9615         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
9616                 ext = nft_set_elem_ext(set, catchall->elem);
9617
9618                 if (!nft_set_elem_expired(ext))
9619                         continue;
9620                 if (nft_set_elem_is_dead(ext))
9621                         goto dead_elem;
9622
9623                 nft_set_elem_dead(ext);
9624 dead_elem:
9625                 gc = nft_trans_gc_queue_async(gc, gc_seq, GFP_ATOMIC);
9626                 if (!gc)
9627                         return NULL;
9628
9629                 nft_trans_gc_elem_add(gc, catchall->elem);
9630         }
9631
9632         return gc;
9633 }
9634
9635 static void nf_tables_module_autoload_cleanup(struct net *net)
9636 {
9637         struct nftables_pernet *nft_net = nft_pernet(net);
9638         struct nft_module_request *req, *next;
9639
9640         WARN_ON_ONCE(!list_empty(&nft_net->commit_list));
9641         list_for_each_entry_safe(req, next, &nft_net->module_list, list) {
9642                 WARN_ON_ONCE(!req->done);
9643                 list_del(&req->list);
9644                 kfree(req);
9645         }
9646 }
9647
9648 static void nf_tables_commit_release(struct net *net)
9649 {
9650         struct nftables_pernet *nft_net = nft_pernet(net);
9651         struct nft_trans *trans;
9652
9653         /* all side effects have to be made visible.
9654          * For example, if a chain named 'foo' has been deleted, a
9655          * new transaction must not find it anymore.
9656          *
9657          * Memory reclaim happens asynchronously from work queue
9658          * to prevent expensive synchronize_rcu() in commit phase.
9659          */
9660         if (list_empty(&nft_net->commit_list)) {
9661                 nf_tables_module_autoload_cleanup(net);
9662                 mutex_unlock(&nft_net->commit_mutex);
9663                 return;
9664         }
9665
9666         trans = list_last_entry(&nft_net->commit_list,
9667                                 struct nft_trans, list);
9668         get_net(trans->ctx.net);
9669         WARN_ON_ONCE(trans->put_net);
9670
9671         trans->put_net = true;
9672         spin_lock(&nf_tables_destroy_list_lock);
9673         list_splice_tail_init(&nft_net->commit_list, &nf_tables_destroy_list);
9674         spin_unlock(&nf_tables_destroy_list_lock);
9675
9676         nf_tables_module_autoload_cleanup(net);
9677         schedule_work(&trans_destroy_work);
9678
9679         mutex_unlock(&nft_net->commit_mutex);
9680 }
9681
9682 static void nft_commit_notify(struct net *net, u32 portid)
9683 {
9684         struct nftables_pernet *nft_net = nft_pernet(net);
9685         struct sk_buff *batch_skb = NULL, *nskb, *skb;
9686         unsigned char *data;
9687         int len;
9688
9689         list_for_each_entry_safe(skb, nskb, &nft_net->notify_list, list) {
9690                 if (!batch_skb) {
9691 new_batch:
9692                         batch_skb = skb;
9693                         len = NLMSG_GOODSIZE - skb->len;
9694                         list_del(&skb->list);
9695                         continue;
9696                 }
9697                 len -= skb->len;
9698                 if (len > 0 && NFT_CB(skb).report == NFT_CB(batch_skb).report) {
9699                         data = skb_put(batch_skb, skb->len);
9700                         memcpy(data, skb->data, skb->len);
9701                         list_del(&skb->list);
9702                         kfree_skb(skb);
9703                         continue;
9704                 }
9705                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
9706                                NFT_CB(batch_skb).report, GFP_KERNEL);
9707                 goto new_batch;
9708         }
9709
9710         if (batch_skb) {
9711                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
9712                                NFT_CB(batch_skb).report, GFP_KERNEL);
9713         }
9714
9715         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
9716 }
9717
9718 static int nf_tables_commit_audit_alloc(struct list_head *adl,
9719                                         struct nft_table *table)
9720 {
9721         struct nft_audit_data *adp;
9722
9723         list_for_each_entry(adp, adl, list) {
9724                 if (adp->table == table)
9725                         return 0;
9726         }
9727         adp = kzalloc(sizeof(*adp), GFP_KERNEL);
9728         if (!adp)
9729                 return -ENOMEM;
9730         adp->table = table;
9731         list_add(&adp->list, adl);
9732         return 0;
9733 }
9734
9735 static void nf_tables_commit_audit_free(struct list_head *adl)
9736 {
9737         struct nft_audit_data *adp, *adn;
9738
9739         list_for_each_entry_safe(adp, adn, adl, list) {
9740                 list_del(&adp->list);
9741                 kfree(adp);
9742         }
9743 }
9744
9745 static void nf_tables_commit_audit_collect(struct list_head *adl,
9746                                            struct nft_table *table, u32 op)
9747 {
9748         struct nft_audit_data *adp;
9749
9750         list_for_each_entry(adp, adl, list) {
9751                 if (adp->table == table)
9752                         goto found;
9753         }
9754         WARN_ONCE(1, "table=%s not expected in commit list", table->name);
9755         return;
9756 found:
9757         adp->entries++;
9758         if (!adp->op || adp->op > op)
9759                 adp->op = op;
9760 }
9761
9762 #define AUNFTABLENAMELEN (NFT_TABLE_MAXNAMELEN + 22)
9763
9764 static void nf_tables_commit_audit_log(struct list_head *adl, u32 generation)
9765 {
9766         struct nft_audit_data *adp, *adn;
9767         char aubuf[AUNFTABLENAMELEN];
9768
9769         list_for_each_entry_safe(adp, adn, adl, list) {
9770                 snprintf(aubuf, AUNFTABLENAMELEN, "%s:%u", adp->table->name,
9771                          generation);
9772                 audit_log_nfcfg(aubuf, adp->table->family, adp->entries,
9773                                 nft2audit_op[adp->op], GFP_KERNEL);
9774                 list_del(&adp->list);
9775                 kfree(adp);
9776         }
9777 }
9778
9779 static void nft_set_commit_update(struct list_head *set_update_list)
9780 {
9781         struct nft_set *set, *next;
9782
9783         list_for_each_entry_safe(set, next, set_update_list, pending_update) {
9784                 list_del_init(&set->pending_update);
9785
9786                 if (!set->ops->commit)
9787                         continue;
9788
9789                 set->ops->commit(set);
9790         }
9791 }
9792
9793 static unsigned int nft_gc_seq_begin(struct nftables_pernet *nft_net)
9794 {
9795         unsigned int gc_seq;
9796
9797         /* Bump gc counter, it becomes odd, this is the busy mark. */
9798         gc_seq = READ_ONCE(nft_net->gc_seq);
9799         WRITE_ONCE(nft_net->gc_seq, ++gc_seq);
9800
9801         return gc_seq;
9802 }
9803
9804 static void nft_gc_seq_end(struct nftables_pernet *nft_net, unsigned int gc_seq)
9805 {
9806         WRITE_ONCE(nft_net->gc_seq, ++gc_seq);
9807 }
9808
9809 static int nf_tables_commit(struct net *net, struct sk_buff *skb)
9810 {
9811         struct nftables_pernet *nft_net = nft_pernet(net);
9812         struct nft_trans *trans, *next;
9813         unsigned int base_seq, gc_seq;
9814         LIST_HEAD(set_update_list);
9815         struct nft_trans_elem *te;
9816         struct nft_chain *chain;
9817         struct nft_table *table;
9818         LIST_HEAD(adl);
9819         int err;
9820
9821         if (list_empty(&nft_net->commit_list)) {
9822                 mutex_unlock(&nft_net->commit_mutex);
9823                 return 0;
9824         }
9825
9826         list_for_each_entry(trans, &nft_net->binding_list, binding_list) {
9827                 switch (trans->msg_type) {
9828                 case NFT_MSG_NEWSET:
9829                         if (!nft_trans_set_update(trans) &&
9830                             nft_set_is_anonymous(nft_trans_set(trans)) &&
9831                             !nft_trans_set_bound(trans)) {
9832                                 pr_warn_once("nftables ruleset with unbound set\n");
9833                                 return -EINVAL;
9834                         }
9835                         break;
9836                 case NFT_MSG_NEWCHAIN:
9837                         if (!nft_trans_chain_update(trans) &&
9838                             nft_chain_binding(nft_trans_chain(trans)) &&
9839                             !nft_trans_chain_bound(trans)) {
9840                                 pr_warn_once("nftables ruleset with unbound chain\n");
9841                                 return -EINVAL;
9842                         }
9843                         break;
9844                 }
9845         }
9846
9847         /* 0. Validate ruleset, otherwise roll back for error reporting. */
9848         if (nf_tables_validate(net) < 0) {
9849                 nft_net->validate_state = NFT_VALIDATE_DO;
9850                 return -EAGAIN;
9851         }
9852
9853         err = nft_flow_rule_offload_commit(net);
9854         if (err < 0)
9855                 return err;
9856
9857         /* 1.  Allocate space for next generation rules_gen_X[] */
9858         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
9859                 int ret;
9860
9861                 ret = nf_tables_commit_audit_alloc(&adl, trans->ctx.table);
9862                 if (ret) {
9863                         nf_tables_commit_chain_prepare_cancel(net);
9864                         nf_tables_commit_audit_free(&adl);
9865                         return ret;
9866                 }
9867                 if (trans->msg_type == NFT_MSG_NEWRULE ||
9868                     trans->msg_type == NFT_MSG_DELRULE) {
9869                         chain = trans->ctx.chain;
9870
9871                         ret = nf_tables_commit_chain_prepare(net, chain);
9872                         if (ret < 0) {
9873                                 nf_tables_commit_chain_prepare_cancel(net);
9874                                 nf_tables_commit_audit_free(&adl);
9875                                 return ret;
9876                         }
9877                 }
9878         }
9879
9880         /* step 2.  Make rules_gen_X visible to packet path */
9881         list_for_each_entry(table, &nft_net->tables, list) {
9882                 list_for_each_entry(chain, &table->chains, list)
9883                         nf_tables_commit_chain(net, chain);
9884         }
9885
9886         /*
9887          * Bump generation counter, invalidate any dump in progress.
9888          * Cannot fail after this point.
9889          */
9890         base_seq = READ_ONCE(nft_net->base_seq);
9891         while (++base_seq == 0)
9892                 ;
9893
9894         WRITE_ONCE(nft_net->base_seq, base_seq);
9895
9896         gc_seq = nft_gc_seq_begin(nft_net);
9897
9898         /* step 3. Start new generation, rules_gen_X now in use. */
9899         net->nft.gencursor = nft_gencursor_next(net);
9900
9901         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
9902                 nf_tables_commit_audit_collect(&adl, trans->ctx.table,
9903                                                trans->msg_type);
9904                 switch (trans->msg_type) {
9905                 case NFT_MSG_NEWTABLE:
9906                         if (nft_trans_table_update(trans)) {
9907                                 if (!(trans->ctx.table->flags & __NFT_TABLE_F_UPDATE)) {
9908                                         nft_trans_destroy(trans);
9909                                         break;
9910                                 }
9911                                 if (trans->ctx.table->flags & NFT_TABLE_F_DORMANT)
9912                                         nf_tables_table_disable(net, trans->ctx.table);
9913
9914                                 trans->ctx.table->flags &= ~__NFT_TABLE_F_UPDATE;
9915                         } else {
9916                                 nft_clear(net, trans->ctx.table);
9917                         }
9918                         nf_tables_table_notify(&trans->ctx, NFT_MSG_NEWTABLE);
9919                         nft_trans_destroy(trans);
9920                         break;
9921                 case NFT_MSG_DELTABLE:
9922                 case NFT_MSG_DESTROYTABLE:
9923                         list_del_rcu(&trans->ctx.table->list);
9924                         nf_tables_table_notify(&trans->ctx, trans->msg_type);
9925                         break;
9926                 case NFT_MSG_NEWCHAIN:
9927                         if (nft_trans_chain_update(trans)) {
9928                                 nft_chain_commit_update(trans);
9929                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN,
9930                                                        &nft_trans_chain_hooks(trans));
9931                                 list_splice(&nft_trans_chain_hooks(trans),
9932                                             &nft_trans_basechain(trans)->hook_list);
9933                                 /* trans destroyed after rcu grace period */
9934                         } else {
9935                                 nft_chain_commit_drop_policy(trans);
9936                                 nft_clear(net, trans->ctx.chain);
9937                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN, NULL);
9938                                 nft_trans_destroy(trans);
9939                         }
9940                         break;
9941                 case NFT_MSG_DELCHAIN:
9942                 case NFT_MSG_DESTROYCHAIN:
9943                         if (nft_trans_chain_update(trans)) {
9944                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_DELCHAIN,
9945                                                        &nft_trans_chain_hooks(trans));
9946                                 nft_netdev_unregister_hooks(net,
9947                                                             &nft_trans_chain_hooks(trans),
9948                                                             true);
9949                         } else {
9950                                 nft_chain_del(trans->ctx.chain);
9951                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_DELCHAIN,
9952                                                        NULL);
9953                                 nf_tables_unregister_hook(trans->ctx.net,
9954                                                           trans->ctx.table,
9955                                                           trans->ctx.chain);
9956                         }
9957                         break;
9958                 case NFT_MSG_NEWRULE:
9959                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
9960                         nf_tables_rule_notify(&trans->ctx,
9961                                               nft_trans_rule(trans),
9962                                               NFT_MSG_NEWRULE);
9963                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
9964                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
9965
9966                         nft_trans_destroy(trans);
9967                         break;
9968                 case NFT_MSG_DELRULE:
9969                 case NFT_MSG_DESTROYRULE:
9970                         list_del_rcu(&nft_trans_rule(trans)->list);
9971                         nf_tables_rule_notify(&trans->ctx,
9972                                               nft_trans_rule(trans),
9973                                               trans->msg_type);
9974                         nft_rule_expr_deactivate(&trans->ctx,
9975                                                  nft_trans_rule(trans),
9976                                                  NFT_TRANS_COMMIT);
9977
9978                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
9979                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
9980                         break;
9981                 case NFT_MSG_NEWSET:
9982                         if (nft_trans_set_update(trans)) {
9983                                 struct nft_set *set = nft_trans_set(trans);
9984
9985                                 WRITE_ONCE(set->timeout, nft_trans_set_timeout(trans));
9986                                 WRITE_ONCE(set->gc_int, nft_trans_set_gc_int(trans));
9987
9988                                 if (nft_trans_set_size(trans))
9989                                         WRITE_ONCE(set->size, nft_trans_set_size(trans));
9990                         } else {
9991                                 nft_clear(net, nft_trans_set(trans));
9992                                 /* This avoids hitting -EBUSY when deleting the table
9993                                  * from the transaction.
9994                                  */
9995                                 if (nft_set_is_anonymous(nft_trans_set(trans)) &&
9996                                     !list_empty(&nft_trans_set(trans)->bindings))
9997                                         nft_use_dec(&trans->ctx.table->use);
9998                         }
9999                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
10000                                              NFT_MSG_NEWSET, GFP_KERNEL);
10001                         nft_trans_destroy(trans);
10002                         break;
10003                 case NFT_MSG_DELSET:
10004                 case NFT_MSG_DESTROYSET:
10005                         nft_trans_set(trans)->dead = 1;
10006                         list_del_rcu(&nft_trans_set(trans)->list);
10007                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
10008                                              trans->msg_type, GFP_KERNEL);
10009                         break;
10010                 case NFT_MSG_NEWSETELEM:
10011                         te = (struct nft_trans_elem *)trans->data;
10012
10013                         nft_setelem_activate(net, te->set, &te->elem);
10014                         nf_tables_setelem_notify(&trans->ctx, te->set,
10015                                                  &te->elem,
10016                                                  NFT_MSG_NEWSETELEM);
10017                         if (te->set->ops->commit &&
10018                             list_empty(&te->set->pending_update)) {
10019                                 list_add_tail(&te->set->pending_update,
10020                                               &set_update_list);
10021                         }
10022                         nft_trans_destroy(trans);
10023                         break;
10024                 case NFT_MSG_DELSETELEM:
10025                 case NFT_MSG_DESTROYSETELEM:
10026                         te = (struct nft_trans_elem *)trans->data;
10027
10028                         nf_tables_setelem_notify(&trans->ctx, te->set,
10029                                                  &te->elem,
10030                                                  trans->msg_type);
10031                         nft_setelem_remove(net, te->set, &te->elem);
10032                         if (!nft_setelem_is_catchall(te->set, &te->elem)) {
10033                                 atomic_dec(&te->set->nelems);
10034                                 te->set->ndeact--;
10035                         }
10036                         if (te->set->ops->commit &&
10037                             list_empty(&te->set->pending_update)) {
10038                                 list_add_tail(&te->set->pending_update,
10039                                               &set_update_list);
10040                         }
10041                         break;
10042                 case NFT_MSG_NEWOBJ:
10043                         if (nft_trans_obj_update(trans)) {
10044                                 nft_obj_commit_update(trans);
10045                                 nf_tables_obj_notify(&trans->ctx,
10046                                                      nft_trans_obj(trans),
10047                                                      NFT_MSG_NEWOBJ);
10048                         } else {
10049                                 nft_clear(net, nft_trans_obj(trans));
10050                                 nf_tables_obj_notify(&trans->ctx,
10051                                                      nft_trans_obj(trans),
10052                                                      NFT_MSG_NEWOBJ);
10053                                 nft_trans_destroy(trans);
10054                         }
10055                         break;
10056                 case NFT_MSG_DELOBJ:
10057                 case NFT_MSG_DESTROYOBJ:
10058                         nft_obj_del(nft_trans_obj(trans));
10059                         nf_tables_obj_notify(&trans->ctx, nft_trans_obj(trans),
10060                                              trans->msg_type);
10061                         break;
10062                 case NFT_MSG_NEWFLOWTABLE:
10063                         if (nft_trans_flowtable_update(trans)) {
10064                                 nft_trans_flowtable(trans)->data.flags =
10065                                         nft_trans_flowtable_flags(trans);
10066                                 nf_tables_flowtable_notify(&trans->ctx,
10067                                                            nft_trans_flowtable(trans),
10068                                                            &nft_trans_flowtable_hooks(trans),
10069                                                            NFT_MSG_NEWFLOWTABLE);
10070                                 list_splice(&nft_trans_flowtable_hooks(trans),
10071                                             &nft_trans_flowtable(trans)->hook_list);
10072                         } else {
10073                                 nft_clear(net, nft_trans_flowtable(trans));
10074                                 nf_tables_flowtable_notify(&trans->ctx,
10075                                                            nft_trans_flowtable(trans),
10076                                                            NULL,
10077                                                            NFT_MSG_NEWFLOWTABLE);
10078                         }
10079                         nft_trans_destroy(trans);
10080                         break;
10081                 case NFT_MSG_DELFLOWTABLE:
10082                 case NFT_MSG_DESTROYFLOWTABLE:
10083                         if (nft_trans_flowtable_update(trans)) {
10084                                 nf_tables_flowtable_notify(&trans->ctx,
10085                                                            nft_trans_flowtable(trans),
10086                                                            &nft_trans_flowtable_hooks(trans),
10087                                                            trans->msg_type);
10088                                 nft_unregister_flowtable_net_hooks(net,
10089                                                                    &nft_trans_flowtable_hooks(trans));
10090                         } else {
10091                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
10092                                 nf_tables_flowtable_notify(&trans->ctx,
10093                                                            nft_trans_flowtable(trans),
10094                                                            NULL,
10095                                                            trans->msg_type);
10096                                 nft_unregister_flowtable_net_hooks(net,
10097                                                 &nft_trans_flowtable(trans)->hook_list);
10098                         }
10099                         break;
10100                 }
10101         }
10102
10103         nft_set_commit_update(&set_update_list);
10104
10105         nft_commit_notify(net, NETLINK_CB(skb).portid);
10106         nf_tables_gen_notify(net, skb, NFT_MSG_NEWGEN);
10107         nf_tables_commit_audit_log(&adl, nft_net->base_seq);
10108
10109         nft_gc_seq_end(nft_net, gc_seq);
10110         nft_net->validate_state = NFT_VALIDATE_SKIP;
10111         nf_tables_commit_release(net);
10112
10113         return 0;
10114 }
10115
10116 static void nf_tables_module_autoload(struct net *net)
10117 {
10118         struct nftables_pernet *nft_net = nft_pernet(net);
10119         struct nft_module_request *req, *next;
10120         LIST_HEAD(module_list);
10121
10122         list_splice_init(&nft_net->module_list, &module_list);
10123         mutex_unlock(&nft_net->commit_mutex);
10124         list_for_each_entry_safe(req, next, &module_list, list) {
10125                 request_module("%s", req->module);
10126                 req->done = true;
10127         }
10128         mutex_lock(&nft_net->commit_mutex);
10129         list_splice(&module_list, &nft_net->module_list);
10130 }
10131
10132 static void nf_tables_abort_release(struct nft_trans *trans)
10133 {
10134         switch (trans->msg_type) {
10135         case NFT_MSG_NEWTABLE:
10136                 nf_tables_table_destroy(&trans->ctx);
10137                 break;
10138         case NFT_MSG_NEWCHAIN:
10139                 if (nft_trans_chain_update(trans))
10140                         nft_hooks_destroy(&nft_trans_chain_hooks(trans));
10141                 else
10142                         nf_tables_chain_destroy(&trans->ctx);
10143                 break;
10144         case NFT_MSG_NEWRULE:
10145                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
10146                 break;
10147         case NFT_MSG_NEWSET:
10148                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
10149                 break;
10150         case NFT_MSG_NEWSETELEM:
10151                 nft_set_elem_destroy(nft_trans_elem_set(trans),
10152                                      nft_trans_elem(trans).priv, true);
10153                 break;
10154         case NFT_MSG_NEWOBJ:
10155                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
10156                 break;
10157         case NFT_MSG_NEWFLOWTABLE:
10158                 if (nft_trans_flowtable_update(trans))
10159                         nft_hooks_destroy(&nft_trans_flowtable_hooks(trans));
10160                 else
10161                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
10162                 break;
10163         }
10164         kfree(trans);
10165 }
10166
10167 static void nft_set_abort_update(struct list_head *set_update_list)
10168 {
10169         struct nft_set *set, *next;
10170
10171         list_for_each_entry_safe(set, next, set_update_list, pending_update) {
10172                 list_del_init(&set->pending_update);
10173
10174                 if (!set->ops->abort)
10175                         continue;
10176
10177                 set->ops->abort(set);
10178         }
10179 }
10180
10181 static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action)
10182 {
10183         struct nftables_pernet *nft_net = nft_pernet(net);
10184         struct nft_trans *trans, *next;
10185         LIST_HEAD(set_update_list);
10186         struct nft_trans_elem *te;
10187
10188         if (action == NFNL_ABORT_VALIDATE &&
10189             nf_tables_validate(net) < 0)
10190                 return -EAGAIN;
10191
10192         list_for_each_entry_safe_reverse(trans, next, &nft_net->commit_list,
10193                                          list) {
10194                 switch (trans->msg_type) {
10195                 case NFT_MSG_NEWTABLE:
10196                         if (nft_trans_table_update(trans)) {
10197                                 if (!(trans->ctx.table->flags & __NFT_TABLE_F_UPDATE)) {
10198                                         nft_trans_destroy(trans);
10199                                         break;
10200                                 }
10201                                 if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_DORMANT) {
10202                                         nf_tables_table_disable(net, trans->ctx.table);
10203                                         trans->ctx.table->flags |= NFT_TABLE_F_DORMANT;
10204                                 } else if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_AWAKEN) {
10205                                         trans->ctx.table->flags &= ~NFT_TABLE_F_DORMANT;
10206                                 }
10207                                 trans->ctx.table->flags &= ~__NFT_TABLE_F_UPDATE;
10208                                 nft_trans_destroy(trans);
10209                         } else {
10210                                 list_del_rcu(&trans->ctx.table->list);
10211                         }
10212                         break;
10213                 case NFT_MSG_DELTABLE:
10214                 case NFT_MSG_DESTROYTABLE:
10215                         nft_clear(trans->ctx.net, trans->ctx.table);
10216                         nft_trans_destroy(trans);
10217                         break;
10218                 case NFT_MSG_NEWCHAIN:
10219                         if (nft_trans_chain_update(trans)) {
10220                                 nft_netdev_unregister_hooks(net,
10221                                                             &nft_trans_chain_hooks(trans),
10222                                                             true);
10223                                 free_percpu(nft_trans_chain_stats(trans));
10224                                 kfree(nft_trans_chain_name(trans));
10225                                 nft_trans_destroy(trans);
10226                         } else {
10227                                 if (nft_trans_chain_bound(trans)) {
10228                                         nft_trans_destroy(trans);
10229                                         break;
10230                                 }
10231                                 nft_use_dec_restore(&trans->ctx.table->use);
10232                                 nft_chain_del(trans->ctx.chain);
10233                                 nf_tables_unregister_hook(trans->ctx.net,
10234                                                           trans->ctx.table,
10235                                                           trans->ctx.chain);
10236                         }
10237                         break;
10238                 case NFT_MSG_DELCHAIN:
10239                 case NFT_MSG_DESTROYCHAIN:
10240                         if (nft_trans_chain_update(trans)) {
10241                                 list_splice(&nft_trans_chain_hooks(trans),
10242                                             &nft_trans_basechain(trans)->hook_list);
10243                         } else {
10244                                 nft_use_inc_restore(&trans->ctx.table->use);
10245                                 nft_clear(trans->ctx.net, trans->ctx.chain);
10246                         }
10247                         nft_trans_destroy(trans);
10248                         break;
10249                 case NFT_MSG_NEWRULE:
10250                         if (nft_trans_rule_bound(trans)) {
10251                                 nft_trans_destroy(trans);
10252                                 break;
10253                         }
10254                         nft_use_dec_restore(&trans->ctx.chain->use);
10255                         list_del_rcu(&nft_trans_rule(trans)->list);
10256                         nft_rule_expr_deactivate(&trans->ctx,
10257                                                  nft_trans_rule(trans),
10258                                                  NFT_TRANS_ABORT);
10259                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
10260                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10261                         break;
10262                 case NFT_MSG_DELRULE:
10263                 case NFT_MSG_DESTROYRULE:
10264                         nft_use_inc_restore(&trans->ctx.chain->use);
10265                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
10266                         nft_rule_expr_activate(&trans->ctx, nft_trans_rule(trans));
10267                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
10268                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10269
10270                         nft_trans_destroy(trans);
10271                         break;
10272                 case NFT_MSG_NEWSET:
10273                         if (nft_trans_set_update(trans)) {
10274                                 nft_trans_destroy(trans);
10275                                 break;
10276                         }
10277                         nft_use_dec_restore(&trans->ctx.table->use);
10278                         if (nft_trans_set_bound(trans)) {
10279                                 nft_trans_destroy(trans);
10280                                 break;
10281                         }
10282                         list_del_rcu(&nft_trans_set(trans)->list);
10283                         break;
10284                 case NFT_MSG_DELSET:
10285                 case NFT_MSG_DESTROYSET:
10286                         nft_use_inc_restore(&trans->ctx.table->use);
10287                         nft_clear(trans->ctx.net, nft_trans_set(trans));
10288                         if (nft_trans_set(trans)->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
10289                                 nft_map_activate(&trans->ctx, nft_trans_set(trans));
10290
10291                         nft_trans_destroy(trans);
10292                         break;
10293                 case NFT_MSG_NEWSETELEM:
10294                         if (nft_trans_elem_set_bound(trans)) {
10295                                 nft_trans_destroy(trans);
10296                                 break;
10297                         }
10298                         te = (struct nft_trans_elem *)trans->data;
10299                         nft_setelem_remove(net, te->set, &te->elem);
10300                         if (!nft_setelem_is_catchall(te->set, &te->elem))
10301                                 atomic_dec(&te->set->nelems);
10302
10303                         if (te->set->ops->abort &&
10304                             list_empty(&te->set->pending_update)) {
10305                                 list_add_tail(&te->set->pending_update,
10306                                               &set_update_list);
10307                         }
10308                         break;
10309                 case NFT_MSG_DELSETELEM:
10310                 case NFT_MSG_DESTROYSETELEM:
10311                         te = (struct nft_trans_elem *)trans->data;
10312
10313                         nft_setelem_data_activate(net, te->set, &te->elem);
10314                         nft_setelem_activate(net, te->set, &te->elem);
10315                         if (!nft_setelem_is_catchall(te->set, &te->elem))
10316                                 te->set->ndeact--;
10317
10318                         if (te->set->ops->abort &&
10319                             list_empty(&te->set->pending_update)) {
10320                                 list_add_tail(&te->set->pending_update,
10321                                               &set_update_list);
10322                         }
10323                         nft_trans_destroy(trans);
10324                         break;
10325                 case NFT_MSG_NEWOBJ:
10326                         if (nft_trans_obj_update(trans)) {
10327                                 nft_obj_destroy(&trans->ctx, nft_trans_obj_newobj(trans));
10328                                 nft_trans_destroy(trans);
10329                         } else {
10330                                 nft_use_dec_restore(&trans->ctx.table->use);
10331                                 nft_obj_del(nft_trans_obj(trans));
10332                         }
10333                         break;
10334                 case NFT_MSG_DELOBJ:
10335                 case NFT_MSG_DESTROYOBJ:
10336                         nft_use_inc_restore(&trans->ctx.table->use);
10337                         nft_clear(trans->ctx.net, nft_trans_obj(trans));
10338                         nft_trans_destroy(trans);
10339                         break;
10340                 case NFT_MSG_NEWFLOWTABLE:
10341                         if (nft_trans_flowtable_update(trans)) {
10342                                 nft_unregister_flowtable_net_hooks(net,
10343                                                 &nft_trans_flowtable_hooks(trans));
10344                         } else {
10345                                 nft_use_dec_restore(&trans->ctx.table->use);
10346                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
10347                                 nft_unregister_flowtable_net_hooks(net,
10348                                                 &nft_trans_flowtable(trans)->hook_list);
10349                         }
10350                         break;
10351                 case NFT_MSG_DELFLOWTABLE:
10352                 case NFT_MSG_DESTROYFLOWTABLE:
10353                         if (nft_trans_flowtable_update(trans)) {
10354                                 list_splice(&nft_trans_flowtable_hooks(trans),
10355                                             &nft_trans_flowtable(trans)->hook_list);
10356                         } else {
10357                                 nft_use_inc_restore(&trans->ctx.table->use);
10358                                 nft_clear(trans->ctx.net, nft_trans_flowtable(trans));
10359                         }
10360                         nft_trans_destroy(trans);
10361                         break;
10362                 }
10363         }
10364
10365         nft_set_abort_update(&set_update_list);
10366
10367         synchronize_rcu();
10368
10369         list_for_each_entry_safe_reverse(trans, next,
10370                                          &nft_net->commit_list, list) {
10371                 nft_trans_list_del(trans);
10372                 nf_tables_abort_release(trans);
10373         }
10374
10375         if (action == NFNL_ABORT_AUTOLOAD)
10376                 nf_tables_module_autoload(net);
10377         else
10378                 nf_tables_module_autoload_cleanup(net);
10379
10380         return 0;
10381 }
10382
10383 static int nf_tables_abort(struct net *net, struct sk_buff *skb,
10384                            enum nfnl_abort_action action)
10385 {
10386         struct nftables_pernet *nft_net = nft_pernet(net);
10387         unsigned int gc_seq;
10388         int ret;
10389
10390         gc_seq = nft_gc_seq_begin(nft_net);
10391         ret = __nf_tables_abort(net, action);
10392         nft_gc_seq_end(nft_net, gc_seq);
10393         mutex_unlock(&nft_net->commit_mutex);
10394
10395         return ret;
10396 }
10397
10398 static bool nf_tables_valid_genid(struct net *net, u32 genid)
10399 {
10400         struct nftables_pernet *nft_net = nft_pernet(net);
10401         bool genid_ok;
10402
10403         mutex_lock(&nft_net->commit_mutex);
10404
10405         genid_ok = genid == 0 || nft_net->base_seq == genid;
10406         if (!genid_ok)
10407                 mutex_unlock(&nft_net->commit_mutex);
10408
10409         /* else, commit mutex has to be released by commit or abort function */
10410         return genid_ok;
10411 }
10412
10413 static const struct nfnetlink_subsystem nf_tables_subsys = {
10414         .name           = "nf_tables",
10415         .subsys_id      = NFNL_SUBSYS_NFTABLES,
10416         .cb_count       = NFT_MSG_MAX,
10417         .cb             = nf_tables_cb,
10418         .commit         = nf_tables_commit,
10419         .abort          = nf_tables_abort,
10420         .valid_genid    = nf_tables_valid_genid,
10421         .owner          = THIS_MODULE,
10422 };
10423
10424 int nft_chain_validate_dependency(const struct nft_chain *chain,
10425                                   enum nft_chain_types type)
10426 {
10427         const struct nft_base_chain *basechain;
10428
10429         if (nft_is_base_chain(chain)) {
10430                 basechain = nft_base_chain(chain);
10431                 if (basechain->type->type != type)
10432                         return -EOPNOTSUPP;
10433         }
10434         return 0;
10435 }
10436 EXPORT_SYMBOL_GPL(nft_chain_validate_dependency);
10437
10438 int nft_chain_validate_hooks(const struct nft_chain *chain,
10439                              unsigned int hook_flags)
10440 {
10441         struct nft_base_chain *basechain;
10442
10443         if (nft_is_base_chain(chain)) {
10444                 basechain = nft_base_chain(chain);
10445
10446                 if ((1 << basechain->ops.hooknum) & hook_flags)
10447                         return 0;
10448
10449                 return -EOPNOTSUPP;
10450         }
10451
10452         return 0;
10453 }
10454 EXPORT_SYMBOL_GPL(nft_chain_validate_hooks);
10455
10456 /*
10457  * Loop detection - walk through the ruleset beginning at the destination chain
10458  * of a new jump until either the source chain is reached (loop) or all
10459  * reachable chains have been traversed.
10460  *
10461  * The loop check is performed whenever a new jump verdict is added to an
10462  * expression or verdict map or a verdict map is bound to a new chain.
10463  */
10464
10465 static int nf_tables_check_loops(const struct nft_ctx *ctx,
10466                                  const struct nft_chain *chain);
10467
10468 static int nft_check_loops(const struct nft_ctx *ctx,
10469                            const struct nft_set_ext *ext)
10470 {
10471         const struct nft_data *data;
10472         int ret;
10473
10474         data = nft_set_ext_data(ext);
10475         switch (data->verdict.code) {
10476         case NFT_JUMP:
10477         case NFT_GOTO:
10478                 ret = nf_tables_check_loops(ctx, data->verdict.chain);
10479                 break;
10480         default:
10481                 ret = 0;
10482                 break;
10483         }
10484
10485         return ret;
10486 }
10487
10488 static int nf_tables_loop_check_setelem(const struct nft_ctx *ctx,
10489                                         struct nft_set *set,
10490                                         const struct nft_set_iter *iter,
10491                                         struct nft_set_elem *elem)
10492 {
10493         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
10494
10495         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
10496             *nft_set_ext_flags(ext) & NFT_SET_ELEM_INTERVAL_END)
10497                 return 0;
10498
10499         return nft_check_loops(ctx, ext);
10500 }
10501
10502 static int nft_set_catchall_loops(const struct nft_ctx *ctx,
10503                                   struct nft_set *set)
10504 {
10505         u8 genmask = nft_genmask_next(ctx->net);
10506         struct nft_set_elem_catchall *catchall;
10507         struct nft_set_ext *ext;
10508         int ret = 0;
10509
10510         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
10511                 ext = nft_set_elem_ext(set, catchall->elem);
10512                 if (!nft_set_elem_active(ext, genmask))
10513                         continue;
10514
10515                 ret = nft_check_loops(ctx, ext);
10516                 if (ret < 0)
10517                         return ret;
10518         }
10519
10520         return ret;
10521 }
10522
10523 static int nf_tables_check_loops(const struct nft_ctx *ctx,
10524                                  const struct nft_chain *chain)
10525 {
10526         const struct nft_rule *rule;
10527         const struct nft_expr *expr, *last;
10528         struct nft_set *set;
10529         struct nft_set_binding *binding;
10530         struct nft_set_iter iter;
10531
10532         if (ctx->chain == chain)
10533                 return -ELOOP;
10534
10535         if (fatal_signal_pending(current))
10536                 return -EINTR;
10537
10538         list_for_each_entry(rule, &chain->rules, list) {
10539                 nft_rule_for_each_expr(expr, last, rule) {
10540                         struct nft_immediate_expr *priv;
10541                         const struct nft_data *data;
10542                         int err;
10543
10544                         if (strcmp(expr->ops->type->name, "immediate"))
10545                                 continue;
10546
10547                         priv = nft_expr_priv(expr);
10548                         if (priv->dreg != NFT_REG_VERDICT)
10549                                 continue;
10550
10551                         data = &priv->data;
10552                         switch (data->verdict.code) {
10553                         case NFT_JUMP:
10554                         case NFT_GOTO:
10555                                 err = nf_tables_check_loops(ctx,
10556                                                         data->verdict.chain);
10557                                 if (err < 0)
10558                                         return err;
10559                                 break;
10560                         default:
10561                                 break;
10562                         }
10563                 }
10564         }
10565
10566         list_for_each_entry(set, &ctx->table->sets, list) {
10567                 if (!nft_is_active_next(ctx->net, set))
10568                         continue;
10569                 if (!(set->flags & NFT_SET_MAP) ||
10570                     set->dtype != NFT_DATA_VERDICT)
10571                         continue;
10572
10573                 list_for_each_entry(binding, &set->bindings, list) {
10574                         if (!(binding->flags & NFT_SET_MAP) ||
10575                             binding->chain != chain)
10576                                 continue;
10577
10578                         iter.genmask    = nft_genmask_next(ctx->net);
10579                         iter.skip       = 0;
10580                         iter.count      = 0;
10581                         iter.err        = 0;
10582                         iter.fn         = nf_tables_loop_check_setelem;
10583
10584                         set->ops->walk(ctx, set, &iter);
10585                         if (!iter.err)
10586                                 iter.err = nft_set_catchall_loops(ctx, set);
10587
10588                         if (iter.err < 0)
10589                                 return iter.err;
10590                 }
10591         }
10592
10593         return 0;
10594 }
10595
10596 /**
10597  *      nft_parse_u32_check - fetch u32 attribute and check for maximum value
10598  *
10599  *      @attr: netlink attribute to fetch value from
10600  *      @max: maximum value to be stored in dest
10601  *      @dest: pointer to the variable
10602  *
10603  *      Parse, check and store a given u32 netlink attribute into variable.
10604  *      This function returns -ERANGE if the value goes over maximum value.
10605  *      Otherwise a 0 is returned and the attribute value is stored in the
10606  *      destination variable.
10607  */
10608 int nft_parse_u32_check(const struct nlattr *attr, int max, u32 *dest)
10609 {
10610         u32 val;
10611
10612         val = ntohl(nla_get_be32(attr));
10613         if (val > max)
10614                 return -ERANGE;
10615
10616         *dest = val;
10617         return 0;
10618 }
10619 EXPORT_SYMBOL_GPL(nft_parse_u32_check);
10620
10621 static int nft_parse_register(const struct nlattr *attr, u32 *preg)
10622 {
10623         unsigned int reg;
10624
10625         reg = ntohl(nla_get_be32(attr));
10626         switch (reg) {
10627         case NFT_REG_VERDICT...NFT_REG_4:
10628                 *preg = reg * NFT_REG_SIZE / NFT_REG32_SIZE;
10629                 break;
10630         case NFT_REG32_00...NFT_REG32_15:
10631                 *preg = reg + NFT_REG_SIZE / NFT_REG32_SIZE - NFT_REG32_00;
10632                 break;
10633         default:
10634                 return -ERANGE;
10635         }
10636
10637         return 0;
10638 }
10639
10640 /**
10641  *      nft_dump_register - dump a register value to a netlink attribute
10642  *
10643  *      @skb: socket buffer
10644  *      @attr: attribute number
10645  *      @reg: register number
10646  *
10647  *      Construct a netlink attribute containing the register number. For
10648  *      compatibility reasons, register numbers being a multiple of 4 are
10649  *      translated to the corresponding 128 bit register numbers.
10650  */
10651 int nft_dump_register(struct sk_buff *skb, unsigned int attr, unsigned int reg)
10652 {
10653         if (reg % (NFT_REG_SIZE / NFT_REG32_SIZE) == 0)
10654                 reg = reg / (NFT_REG_SIZE / NFT_REG32_SIZE);
10655         else
10656                 reg = reg - NFT_REG_SIZE / NFT_REG32_SIZE + NFT_REG32_00;
10657
10658         return nla_put_be32(skb, attr, htonl(reg));
10659 }
10660 EXPORT_SYMBOL_GPL(nft_dump_register);
10661
10662 static int nft_validate_register_load(enum nft_registers reg, unsigned int len)
10663 {
10664         if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
10665                 return -EINVAL;
10666         if (len == 0)
10667                 return -EINVAL;
10668         if (reg * NFT_REG32_SIZE + len > sizeof_field(struct nft_regs, data))
10669                 return -ERANGE;
10670
10671         return 0;
10672 }
10673
10674 int nft_parse_register_load(const struct nlattr *attr, u8 *sreg, u32 len)
10675 {
10676         u32 reg;
10677         int err;
10678
10679         err = nft_parse_register(attr, &reg);
10680         if (err < 0)
10681                 return err;
10682
10683         err = nft_validate_register_load(reg, len);
10684         if (err < 0)
10685                 return err;
10686
10687         *sreg = reg;
10688         return 0;
10689 }
10690 EXPORT_SYMBOL_GPL(nft_parse_register_load);
10691
10692 static int nft_validate_register_store(const struct nft_ctx *ctx,
10693                                        enum nft_registers reg,
10694                                        const struct nft_data *data,
10695                                        enum nft_data_types type,
10696                                        unsigned int len)
10697 {
10698         int err;
10699
10700         switch (reg) {
10701         case NFT_REG_VERDICT:
10702                 if (type != NFT_DATA_VERDICT)
10703                         return -EINVAL;
10704
10705                 if (data != NULL &&
10706                     (data->verdict.code == NFT_GOTO ||
10707                      data->verdict.code == NFT_JUMP)) {
10708                         err = nf_tables_check_loops(ctx, data->verdict.chain);
10709                         if (err < 0)
10710                                 return err;
10711                 }
10712
10713                 return 0;
10714         default:
10715                 if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
10716                         return -EINVAL;
10717                 if (len == 0)
10718                         return -EINVAL;
10719                 if (reg * NFT_REG32_SIZE + len >
10720                     sizeof_field(struct nft_regs, data))
10721                         return -ERANGE;
10722
10723                 if (data != NULL && type != NFT_DATA_VALUE)
10724                         return -EINVAL;
10725                 return 0;
10726         }
10727 }
10728
10729 int nft_parse_register_store(const struct nft_ctx *ctx,
10730                              const struct nlattr *attr, u8 *dreg,
10731                              const struct nft_data *data,
10732                              enum nft_data_types type, unsigned int len)
10733 {
10734         int err;
10735         u32 reg;
10736
10737         err = nft_parse_register(attr, &reg);
10738         if (err < 0)
10739                 return err;
10740
10741         err = nft_validate_register_store(ctx, reg, data, type, len);
10742         if (err < 0)
10743                 return err;
10744
10745         *dreg = reg;
10746         return 0;
10747 }
10748 EXPORT_SYMBOL_GPL(nft_parse_register_store);
10749
10750 static const struct nla_policy nft_verdict_policy[NFTA_VERDICT_MAX + 1] = {
10751         [NFTA_VERDICT_CODE]     = { .type = NLA_U32 },
10752         [NFTA_VERDICT_CHAIN]    = { .type = NLA_STRING,
10753                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
10754         [NFTA_VERDICT_CHAIN_ID] = { .type = NLA_U32 },
10755 };
10756
10757 static int nft_verdict_init(const struct nft_ctx *ctx, struct nft_data *data,
10758                             struct nft_data_desc *desc, const struct nlattr *nla)
10759 {
10760         u8 genmask = nft_genmask_next(ctx->net);
10761         struct nlattr *tb[NFTA_VERDICT_MAX + 1];
10762         struct nft_chain *chain;
10763         int err;
10764
10765         err = nla_parse_nested_deprecated(tb, NFTA_VERDICT_MAX, nla,
10766                                           nft_verdict_policy, NULL);
10767         if (err < 0)
10768                 return err;
10769
10770         if (!tb[NFTA_VERDICT_CODE])
10771                 return -EINVAL;
10772
10773         /* zero padding hole for memcmp */
10774         memset(data, 0, sizeof(*data));
10775         data->verdict.code = ntohl(nla_get_be32(tb[NFTA_VERDICT_CODE]));
10776
10777         switch (data->verdict.code) {
10778         default:
10779                 switch (data->verdict.code & NF_VERDICT_MASK) {
10780                 case NF_ACCEPT:
10781                 case NF_DROP:
10782                 case NF_QUEUE:
10783                         break;
10784                 default:
10785                         return -EINVAL;
10786                 }
10787                 fallthrough;
10788         case NFT_CONTINUE:
10789         case NFT_BREAK:
10790         case NFT_RETURN:
10791                 break;
10792         case NFT_JUMP:
10793         case NFT_GOTO:
10794                 if (tb[NFTA_VERDICT_CHAIN]) {
10795                         chain = nft_chain_lookup(ctx->net, ctx->table,
10796                                                  tb[NFTA_VERDICT_CHAIN],
10797                                                  genmask);
10798                 } else if (tb[NFTA_VERDICT_CHAIN_ID]) {
10799                         chain = nft_chain_lookup_byid(ctx->net, ctx->table,
10800                                                       tb[NFTA_VERDICT_CHAIN_ID],
10801                                                       genmask);
10802                         if (IS_ERR(chain))
10803                                 return PTR_ERR(chain);
10804                 } else {
10805                         return -EINVAL;
10806                 }
10807
10808                 if (IS_ERR(chain))
10809                         return PTR_ERR(chain);
10810                 if (nft_is_base_chain(chain))
10811                         return -EOPNOTSUPP;
10812                 if (nft_chain_is_bound(chain))
10813                         return -EINVAL;
10814                 if (desc->flags & NFT_DATA_DESC_SETELEM &&
10815                     chain->flags & NFT_CHAIN_BINDING)
10816                         return -EINVAL;
10817                 if (!nft_use_inc(&chain->use))
10818                         return -EMFILE;
10819
10820                 data->verdict.chain = chain;
10821                 break;
10822         }
10823
10824         desc->len = sizeof(data->verdict);
10825
10826         return 0;
10827 }
10828
10829 static void nft_verdict_uninit(const struct nft_data *data)
10830 {
10831         struct nft_chain *chain;
10832
10833         switch (data->verdict.code) {
10834         case NFT_JUMP:
10835         case NFT_GOTO:
10836                 chain = data->verdict.chain;
10837                 nft_use_dec(&chain->use);
10838                 break;
10839         }
10840 }
10841
10842 int nft_verdict_dump(struct sk_buff *skb, int type, const struct nft_verdict *v)
10843 {
10844         struct nlattr *nest;
10845
10846         nest = nla_nest_start_noflag(skb, type);
10847         if (!nest)
10848                 goto nla_put_failure;
10849
10850         if (nla_put_be32(skb, NFTA_VERDICT_CODE, htonl(v->code)))
10851                 goto nla_put_failure;
10852
10853         switch (v->code) {
10854         case NFT_JUMP:
10855         case NFT_GOTO:
10856                 if (nla_put_string(skb, NFTA_VERDICT_CHAIN,
10857                                    v->chain->name))
10858                         goto nla_put_failure;
10859         }
10860         nla_nest_end(skb, nest);
10861         return 0;
10862
10863 nla_put_failure:
10864         return -1;
10865 }
10866
10867 static int nft_value_init(const struct nft_ctx *ctx,
10868                           struct nft_data *data, struct nft_data_desc *desc,
10869                           const struct nlattr *nla)
10870 {
10871         unsigned int len;
10872
10873         len = nla_len(nla);
10874         if (len == 0)
10875                 return -EINVAL;
10876         if (len > desc->size)
10877                 return -EOVERFLOW;
10878         if (desc->len) {
10879                 if (len != desc->len)
10880                         return -EINVAL;
10881         } else {
10882                 desc->len = len;
10883         }
10884
10885         nla_memcpy(data->data, nla, len);
10886
10887         return 0;
10888 }
10889
10890 static int nft_value_dump(struct sk_buff *skb, const struct nft_data *data,
10891                           unsigned int len)
10892 {
10893         return nla_put(skb, NFTA_DATA_VALUE, len, data->data);
10894 }
10895
10896 static const struct nla_policy nft_data_policy[NFTA_DATA_MAX + 1] = {
10897         [NFTA_DATA_VALUE]       = { .type = NLA_BINARY },
10898         [NFTA_DATA_VERDICT]     = { .type = NLA_NESTED },
10899 };
10900
10901 /**
10902  *      nft_data_init - parse nf_tables data netlink attributes
10903  *
10904  *      @ctx: context of the expression using the data
10905  *      @data: destination struct nft_data
10906  *      @desc: data description
10907  *      @nla: netlink attribute containing data
10908  *
10909  *      Parse the netlink data attributes and initialize a struct nft_data.
10910  *      The type and length of data are returned in the data description.
10911  *
10912  *      The caller can indicate that it only wants to accept data of type
10913  *      NFT_DATA_VALUE by passing NULL for the ctx argument.
10914  */
10915 int nft_data_init(const struct nft_ctx *ctx, struct nft_data *data,
10916                   struct nft_data_desc *desc, const struct nlattr *nla)
10917 {
10918         struct nlattr *tb[NFTA_DATA_MAX + 1];
10919         int err;
10920
10921         if (WARN_ON_ONCE(!desc->size))
10922                 return -EINVAL;
10923
10924         err = nla_parse_nested_deprecated(tb, NFTA_DATA_MAX, nla,
10925                                           nft_data_policy, NULL);
10926         if (err < 0)
10927                 return err;
10928
10929         if (tb[NFTA_DATA_VALUE]) {
10930                 if (desc->type != NFT_DATA_VALUE)
10931                         return -EINVAL;
10932
10933                 err = nft_value_init(ctx, data, desc, tb[NFTA_DATA_VALUE]);
10934         } else if (tb[NFTA_DATA_VERDICT] && ctx != NULL) {
10935                 if (desc->type != NFT_DATA_VERDICT)
10936                         return -EINVAL;
10937
10938                 err = nft_verdict_init(ctx, data, desc, tb[NFTA_DATA_VERDICT]);
10939         } else {
10940                 err = -EINVAL;
10941         }
10942
10943         return err;
10944 }
10945 EXPORT_SYMBOL_GPL(nft_data_init);
10946
10947 /**
10948  *      nft_data_release - release a nft_data item
10949  *
10950  *      @data: struct nft_data to release
10951  *      @type: type of data
10952  *
10953  *      Release a nft_data item. NFT_DATA_VALUE types can be silently discarded,
10954  *      all others need to be released by calling this function.
10955  */
10956 void nft_data_release(const struct nft_data *data, enum nft_data_types type)
10957 {
10958         if (type < NFT_DATA_VERDICT)
10959                 return;
10960         switch (type) {
10961         case NFT_DATA_VERDICT:
10962                 return nft_verdict_uninit(data);
10963         default:
10964                 WARN_ON(1);
10965         }
10966 }
10967 EXPORT_SYMBOL_GPL(nft_data_release);
10968
10969 int nft_data_dump(struct sk_buff *skb, int attr, const struct nft_data *data,
10970                   enum nft_data_types type, unsigned int len)
10971 {
10972         struct nlattr *nest;
10973         int err;
10974
10975         nest = nla_nest_start_noflag(skb, attr);
10976         if (nest == NULL)
10977                 return -1;
10978
10979         switch (type) {
10980         case NFT_DATA_VALUE:
10981                 err = nft_value_dump(skb, data, len);
10982                 break;
10983         case NFT_DATA_VERDICT:
10984                 err = nft_verdict_dump(skb, NFTA_DATA_VERDICT, &data->verdict);
10985                 break;
10986         default:
10987                 err = -EINVAL;
10988                 WARN_ON(1);
10989         }
10990
10991         nla_nest_end(skb, nest);
10992         return err;
10993 }
10994 EXPORT_SYMBOL_GPL(nft_data_dump);
10995
10996 int __nft_release_basechain(struct nft_ctx *ctx)
10997 {
10998         struct nft_rule *rule, *nr;
10999
11000         if (WARN_ON(!nft_is_base_chain(ctx->chain)))
11001                 return 0;
11002
11003         nf_tables_unregister_hook(ctx->net, ctx->chain->table, ctx->chain);
11004         list_for_each_entry_safe(rule, nr, &ctx->chain->rules, list) {
11005                 list_del(&rule->list);
11006                 nft_use_dec(&ctx->chain->use);
11007                 nf_tables_rule_release(ctx, rule);
11008         }
11009         nft_chain_del(ctx->chain);
11010         nft_use_dec(&ctx->table->use);
11011         nf_tables_chain_destroy(ctx);
11012
11013         return 0;
11014 }
11015 EXPORT_SYMBOL_GPL(__nft_release_basechain);
11016
11017 static void __nft_release_hook(struct net *net, struct nft_table *table)
11018 {
11019         struct nft_flowtable *flowtable;
11020         struct nft_chain *chain;
11021
11022         list_for_each_entry(chain, &table->chains, list)
11023                 __nf_tables_unregister_hook(net, table, chain, true);
11024         list_for_each_entry(flowtable, &table->flowtables, list)
11025                 __nft_unregister_flowtable_net_hooks(net, &flowtable->hook_list,
11026                                                      true);
11027 }
11028
11029 static void __nft_release_hooks(struct net *net)
11030 {
11031         struct nftables_pernet *nft_net = nft_pernet(net);
11032         struct nft_table *table;
11033
11034         list_for_each_entry(table, &nft_net->tables, list) {
11035                 if (nft_table_has_owner(table))
11036                         continue;
11037
11038                 __nft_release_hook(net, table);
11039         }
11040 }
11041
11042 static void __nft_release_table(struct net *net, struct nft_table *table)
11043 {
11044         struct nft_flowtable *flowtable, *nf;
11045         struct nft_chain *chain, *nc;
11046         struct nft_object *obj, *ne;
11047         struct nft_rule *rule, *nr;
11048         struct nft_set *set, *ns;
11049         struct nft_ctx ctx = {
11050                 .net    = net,
11051                 .family = NFPROTO_NETDEV,
11052         };
11053
11054         ctx.family = table->family;
11055         ctx.table = table;
11056         list_for_each_entry(chain, &table->chains, list) {
11057                 if (nft_chain_is_bound(chain))
11058                         continue;
11059
11060                 ctx.chain = chain;
11061                 list_for_each_entry_safe(rule, nr, &chain->rules, list) {
11062                         list_del(&rule->list);
11063                         nft_use_dec(&chain->use);
11064                         nf_tables_rule_release(&ctx, rule);
11065                 }
11066         }
11067         list_for_each_entry_safe(flowtable, nf, &table->flowtables, list) {
11068                 list_del(&flowtable->list);
11069                 nft_use_dec(&table->use);
11070                 nf_tables_flowtable_destroy(flowtable);
11071         }
11072         list_for_each_entry_safe(set, ns, &table->sets, list) {
11073                 list_del(&set->list);
11074                 nft_use_dec(&table->use);
11075                 if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
11076                         nft_map_deactivate(&ctx, set);
11077
11078                 nft_set_destroy(&ctx, set);
11079         }
11080         list_for_each_entry_safe(obj, ne, &table->objects, list) {
11081                 nft_obj_del(obj);
11082                 nft_use_dec(&table->use);
11083                 nft_obj_destroy(&ctx, obj);
11084         }
11085         list_for_each_entry_safe(chain, nc, &table->chains, list) {
11086                 ctx.chain = chain;
11087                 nft_chain_del(chain);
11088                 nft_use_dec(&table->use);
11089                 nf_tables_chain_destroy(&ctx);
11090         }
11091         nf_tables_table_destroy(&ctx);
11092 }
11093
11094 static void __nft_release_tables(struct net *net)
11095 {
11096         struct nftables_pernet *nft_net = nft_pernet(net);
11097         struct nft_table *table, *nt;
11098
11099         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
11100                 if (nft_table_has_owner(table))
11101                         continue;
11102
11103                 list_del(&table->list);
11104
11105                 __nft_release_table(net, table);
11106         }
11107 }
11108
11109 static int nft_rcv_nl_event(struct notifier_block *this, unsigned long event,
11110                             void *ptr)
11111 {
11112         struct nft_table *table, *to_delete[8];
11113         struct nftables_pernet *nft_net;
11114         struct netlink_notify *n = ptr;
11115         struct net *net = n->net;
11116         unsigned int deleted;
11117         bool restart = false;
11118         unsigned int gc_seq;
11119
11120         if (event != NETLINK_URELEASE || n->protocol != NETLINK_NETFILTER)
11121                 return NOTIFY_DONE;
11122
11123         nft_net = nft_pernet(net);
11124         deleted = 0;
11125         mutex_lock(&nft_net->commit_mutex);
11126
11127         gc_seq = nft_gc_seq_begin(nft_net);
11128
11129         if (!list_empty(&nf_tables_destroy_list))
11130                 nf_tables_trans_destroy_flush_work();
11131 again:
11132         list_for_each_entry(table, &nft_net->tables, list) {
11133                 if (nft_table_has_owner(table) &&
11134                     n->portid == table->nlpid) {
11135                         __nft_release_hook(net, table);
11136                         list_del_rcu(&table->list);
11137                         to_delete[deleted++] = table;
11138                         if (deleted >= ARRAY_SIZE(to_delete))
11139                                 break;
11140                 }
11141         }
11142         if (deleted) {
11143                 restart = deleted >= ARRAY_SIZE(to_delete);
11144                 synchronize_rcu();
11145                 while (deleted)
11146                         __nft_release_table(net, to_delete[--deleted]);
11147
11148                 if (restart)
11149                         goto again;
11150         }
11151         nft_gc_seq_end(nft_net, gc_seq);
11152
11153         mutex_unlock(&nft_net->commit_mutex);
11154
11155         return NOTIFY_DONE;
11156 }
11157
11158 static struct notifier_block nft_nl_notifier = {
11159         .notifier_call  = nft_rcv_nl_event,
11160 };
11161
11162 static int __net_init nf_tables_init_net(struct net *net)
11163 {
11164         struct nftables_pernet *nft_net = nft_pernet(net);
11165
11166         INIT_LIST_HEAD(&nft_net->tables);
11167         INIT_LIST_HEAD(&nft_net->commit_list);
11168         INIT_LIST_HEAD(&nft_net->binding_list);
11169         INIT_LIST_HEAD(&nft_net->module_list);
11170         INIT_LIST_HEAD(&nft_net->notify_list);
11171         mutex_init(&nft_net->commit_mutex);
11172         nft_net->base_seq = 1;
11173         nft_net->gc_seq = 0;
11174         nft_net->validate_state = NFT_VALIDATE_SKIP;
11175
11176         return 0;
11177 }
11178
11179 static void __net_exit nf_tables_pre_exit_net(struct net *net)
11180 {
11181         struct nftables_pernet *nft_net = nft_pernet(net);
11182
11183         mutex_lock(&nft_net->commit_mutex);
11184         __nft_release_hooks(net);
11185         mutex_unlock(&nft_net->commit_mutex);
11186 }
11187
11188 static void __net_exit nf_tables_exit_net(struct net *net)
11189 {
11190         struct nftables_pernet *nft_net = nft_pernet(net);
11191         unsigned int gc_seq;
11192
11193         mutex_lock(&nft_net->commit_mutex);
11194
11195         gc_seq = nft_gc_seq_begin(nft_net);
11196
11197         if (!list_empty(&nft_net->commit_list) ||
11198             !list_empty(&nft_net->module_list))
11199                 __nf_tables_abort(net, NFNL_ABORT_NONE);
11200
11201         __nft_release_tables(net);
11202
11203         nft_gc_seq_end(nft_net, gc_seq);
11204
11205         mutex_unlock(&nft_net->commit_mutex);
11206         WARN_ON_ONCE(!list_empty(&nft_net->tables));
11207         WARN_ON_ONCE(!list_empty(&nft_net->module_list));
11208         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
11209 }
11210
11211 static void nf_tables_exit_batch(struct list_head *net_exit_list)
11212 {
11213         flush_work(&trans_gc_work);
11214 }
11215
11216 static struct pernet_operations nf_tables_net_ops = {
11217         .init           = nf_tables_init_net,
11218         .pre_exit       = nf_tables_pre_exit_net,
11219         .exit           = nf_tables_exit_net,
11220         .exit_batch     = nf_tables_exit_batch,
11221         .id             = &nf_tables_net_id,
11222         .size           = sizeof(struct nftables_pernet),
11223 };
11224
11225 static int __init nf_tables_module_init(void)
11226 {
11227         int err;
11228
11229         err = register_pernet_subsys(&nf_tables_net_ops);
11230         if (err < 0)
11231                 return err;
11232
11233         err = nft_chain_filter_init();
11234         if (err < 0)
11235                 goto err_chain_filter;
11236
11237         err = nf_tables_core_module_init();
11238         if (err < 0)
11239                 goto err_core_module;
11240
11241         err = register_netdevice_notifier(&nf_tables_flowtable_notifier);
11242         if (err < 0)
11243                 goto err_netdev_notifier;
11244
11245         err = rhltable_init(&nft_objname_ht, &nft_objname_ht_params);
11246         if (err < 0)
11247                 goto err_rht_objname;
11248
11249         err = nft_offload_init();
11250         if (err < 0)
11251                 goto err_offload;
11252
11253         err = netlink_register_notifier(&nft_nl_notifier);
11254         if (err < 0)
11255                 goto err_netlink_notifier;
11256
11257         /* must be last */
11258         err = nfnetlink_subsys_register(&nf_tables_subsys);
11259         if (err < 0)
11260                 goto err_nfnl_subsys;
11261
11262         nft_chain_route_init();
11263
11264         return err;
11265
11266 err_nfnl_subsys:
11267         netlink_unregister_notifier(&nft_nl_notifier);
11268 err_netlink_notifier:
11269         nft_offload_exit();
11270 err_offload:
11271         rhltable_destroy(&nft_objname_ht);
11272 err_rht_objname:
11273         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
11274 err_netdev_notifier:
11275         nf_tables_core_module_exit();
11276 err_core_module:
11277         nft_chain_filter_fini();
11278 err_chain_filter:
11279         unregister_pernet_subsys(&nf_tables_net_ops);
11280         return err;
11281 }
11282
11283 static void __exit nf_tables_module_exit(void)
11284 {
11285         nfnetlink_subsys_unregister(&nf_tables_subsys);
11286         netlink_unregister_notifier(&nft_nl_notifier);
11287         nft_offload_exit();
11288         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
11289         nft_chain_filter_fini();
11290         nft_chain_route_fini();
11291         unregister_pernet_subsys(&nf_tables_net_ops);
11292         cancel_work_sync(&trans_gc_work);
11293         cancel_work_sync(&trans_destroy_work);
11294         rcu_barrier();
11295         rhltable_destroy(&nft_objname_ht);
11296         nf_tables_core_module_exit();
11297 }
11298
11299 module_init(nf_tables_module_init);
11300 module_exit(nf_tables_module_exit);
11301
11302 MODULE_LICENSE("GPL");
11303 MODULE_AUTHOR("Patrick McHardy <kaber@trash.net>");
11304 MODULE_ALIAS_NFNL_SUBSYS(NFNL_SUBSYS_NFTABLES);