netfilter: nf_tables: asynchronous release

Release the committed transaction log from a work queue, moving
expensive synchronize_rcu out of the locked section and providing
opportunity to batch this.

On my test machine this cuts runtime of nft-test.py in half.
Based on earlier patch from Pablo Neira Ayuso.

Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: Pablo Neira Ayuso <pablo@netfilter.org>

diff --git a/include/net/netfilter/nf_tables.h b/include/net/netfilter/nf_tables.h
index 2c33958..841835a 100644 (file)
--- a/include/net/netfilter/nf_tables.h
+++ b/include/net/netfilter/nf_tables.h
@@ -1298,12 +1298,14 @@ static inline void nft_set_elem_clear_busy(struct nft_set_ext *ext)
  *
  *     @list: used internally
  *     @msg_type: message type
+ *     @put_net: ctx->net needs to be put
  *     @ctx: transaction context
  *     @data: internal information related to the transaction
  */
 struct nft_trans {
        struct list_head                list;
        int                             msg_type;
+       bool                            put_net;
        struct nft_ctx                  ctx;
        char                            data[0];
 };

diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c
index d4c531e..72dbdb1 100644 (file)
--- a/net/netfilter/nf_tables_api.c
+++ b/net/netfilter/nf_tables_api.c
@@ -27,6 +27,8 @@
 static LIST_HEAD(nf_tables_expressions);
 static LIST_HEAD(nf_tables_objects);
 static LIST_HEAD(nf_tables_flowtables);
+static LIST_HEAD(nf_tables_destroy_list);
+static DEFINE_SPINLOCK(nf_tables_destroy_list_lock);
 static u64 table_handle;
 
 enum {
@@ -64,6 +66,8 @@ static void nft_validate_state_update(struct net *net, u8 new_validate_state)
 
        net->nft.validate_state = new_validate_state;
 }
+static void nf_tables_trans_destroy_work(struct work_struct *w);
+static DECLARE_WORK(trans_destroy_work, nf_tables_trans_destroy_work);
 
 static void nft_ctx_init(struct nft_ctx *ctx,
                         struct net *net,
@@ -2453,7 +2457,6 @@ static void nf_tables_rule_destroy(const struct nft_ctx *ctx,
 {
        struct nft_expr *expr;
 
-       lockdep_assert_held(&ctx->net->nft.commit_mutex);
        /*
         * Careful: some expressions might not be initialized in case this
         * is called on error from nf_tables_newrule().
@@ -6224,19 +6227,28 @@ static void nft_commit_release(struct nft_trans *trans)
                nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
                break;
        }
+
+       if (trans->put_net)
+               put_net(trans->ctx.net);
+
        kfree(trans);
 }
 
-static void nf_tables_commit_release(struct net *net)
+static void nf_tables_trans_destroy_work(struct work_struct *w)
 {
        struct nft_trans *trans, *next;
+       LIST_HEAD(head);
+
+       spin_lock(&nf_tables_destroy_list_lock);
+       list_splice_init(&nf_tables_destroy_list, &head);
+       spin_unlock(&nf_tables_destroy_list_lock);
 
-       if (list_empty(&net->nft.commit_list))
+       if (list_empty(&head))
                return;
 
        synchronize_rcu();
 
-       list_for_each_entry_safe(trans, next, &net->nft.commit_list, list) {
+       list_for_each_entry_safe(trans, next, &head, list) {
                list_del(&trans->list);
                nft_commit_release(trans);
        }
@@ -6367,6 +6379,37 @@ static void nft_chain_del(struct nft_chain *chain)
        list_del_rcu(&chain->list);
 }
 
+static void nf_tables_commit_release(struct net *net)
+{
+       struct nft_trans *trans;
+
+       /* all side effects have to be made visible.
+        * For example, if a chain named 'foo' has been deleted, a
+        * new transaction must not find it anymore.
+        *
+        * Memory reclaim happens asynchronously from work queue
+        * to prevent expensive synchronize_rcu() in commit phase.
+        */
+       if (list_empty(&net->nft.commit_list)) {
+               mutex_unlock(&net->nft.commit_mutex);
+               return;
+       }
+
+       trans = list_last_entry(&net->nft.commit_list,
+                               struct nft_trans, list);
+       get_net(trans->ctx.net);
+       WARN_ON_ONCE(trans->put_net);
+
+       trans->put_net = true;
+       spin_lock(&nf_tables_destroy_list_lock);
+       list_splice_tail_init(&net->nft.commit_list, &nf_tables_destroy_list);
+       spin_unlock(&nf_tables_destroy_list_lock);
+
+       mutex_unlock(&net->nft.commit_mutex);
+
+       schedule_work(&trans_destroy_work);
+}
+
 static int nf_tables_commit(struct net *net, struct sk_buff *skb)
 {
        struct nft_trans *trans, *next;
@@ -6528,9 +6571,8 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb)
                }
        }
 
-       nf_tables_commit_release(net);
        nf_tables_gen_notify(net, skb, NFT_MSG_NEWGEN);
-       mutex_unlock(&net->nft.commit_mutex);
+       nf_tables_commit_release(net);
 
        return 0;
 }
@@ -7304,6 +7346,7 @@ static int __init nf_tables_module_init(void)
 {
        int err;
 
+       spin_lock_init(&nf_tables_destroy_list_lock);
        err = register_pernet_subsys(&nf_tables_net_ops);
        if (err < 0)
                return err;
@@ -7343,6 +7386,7 @@ static void __exit nf_tables_module_exit(void)
        unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
        nft_chain_filter_fini();
        unregister_pernet_subsys(&nf_tables_net_ops);
+       cancel_work_sync(&trans_destroy_work);
        rcu_barrier();
        nf_tables_core_module_exit();
 }