netfilter: x_tables: improve limit_mt scalability

We've seen this spin_lock show up high in profiles. Let's introduce a
lockless version. I've tested this using pktgen_sample01_simple.sh.

Signed-off-by: Jason Baron <jbaron@akamai.com>
Signed-off-by: Pablo Neira Ayuso <pablo@netfilter.org>

diff --git a/net/netfilter/xt_limit.c b/net/netfilter/xt_limit.c
index 24d4afb..8b4fd27 100644 (file)
--- a/net/netfilter/xt_limit.c
+++ b/net/netfilter/xt_limit.c
@@ -8,16 +8,14 @@
 #include <linux/slab.h>
 #include <linux/module.h>
 #include <linux/skbuff.h>
-#include <linux/spinlock.h>
 #include <linux/interrupt.h>
 
 #include <linux/netfilter/x_tables.h>
 #include <linux/netfilter/xt_limit.h>
 
 struct xt_limit_priv {
-       spinlock_t lock;
        unsigned long prev;
-       uint32_t credit;
+       u32 credit;
 };
 
 MODULE_LICENSE("GPL");
@@ -66,22 +64,31 @@ limit_mt(const struct sk_buff *skb, struct xt_action_param *par)
 {
        const struct xt_rateinfo *r = par->matchinfo;
        struct xt_limit_priv *priv = r->master;
-       unsigned long now = jiffies;
-
-       spin_lock_bh(&priv->lock);
-       priv->credit += (now - xchg(&priv->prev, now)) * CREDITS_PER_JIFFY;
-       if (priv->credit > r->credit_cap)
-               priv->credit = r->credit_cap;
-
-       if (priv->credit >= r->cost) {
-               /* We're not limited. */
-               priv->credit -= r->cost;
-               spin_unlock_bh(&priv->lock);
-               return true;
-       }
-
-       spin_unlock_bh(&priv->lock);
-       return false;
+       unsigned long now;
+       u32 old_credit, new_credit, credit_increase = 0;
+       bool ret;
+
+       /* fastpath if there is nothing to update */
+       if ((READ_ONCE(priv->credit) < r->cost) && (READ_ONCE(priv->prev) == jiffies))
+               return false;
+
+       do {
+               now = jiffies;
+               credit_increase += (now - xchg(&priv->prev, now)) * CREDITS_PER_JIFFY;
+               old_credit = READ_ONCE(priv->credit);
+               new_credit = old_credit;
+               new_credit += credit_increase;
+               if (new_credit > r->credit_cap)
+                       new_credit = r->credit_cap;
+               if (new_credit >= r->cost) {
+                       ret = true;
+                       new_credit -= r->cost;
+               } else {
+                       ret = false;
+               }
+       } while (cmpxchg(&priv->credit, old_credit, new_credit) != old_credit);
+
+       return ret;
 }
 
 /* Precision saver. */
@@ -122,7 +129,6 @@ static int limit_mt_check(const struct xt_mtchk_param *par)
                r->credit_cap = priv->credit; /* Credits full. */
                r->cost = user2credits(r->avg);
        }
-       spin_lock_init(&priv->lock);
 
        return 0;
 }