ipv4: Add fib_nh_common to fib_result

Most of the ipv4 code only needs data from fib_nh_common. Add
fib_nh_common selection to fib_result and update users to use it.

Right now, fib_nh_common in fib_result will point to a fib_nh struct
that is embedded within a fib_info:

        fib_info  --> fib_nh
                      fib_nh
                      ...
                      fib_nh
                        ^
    fib_result->nhc ----+

Later, nhc can point to a fib_nh within a nexthop struct:

        fib_info --> nexthop --> fib_nh
                                   ^
    fib_result->nhc ---------------+

or for a nexthop group:

        fib_info --> nexthop --> nexthop --> fib_nh
                                 nexthop --> fib_nh
                                 ...
                                 nexthop --> fib_nh
                                               ^
    fib_result->nhc ---------------------------+

In all cases nhsel within fib_result will point to which leg in the
multipath route is used.

Signed-off-by: David Ahern <dsahern@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/net/ip_fib.h b/include/net/ip_fib.h
index 12a6d75..1f4a3b8 100644 (file)
--- a/include/net/ip_fib.h
+++ b/include/net/ip_fib.h
@@ -156,15 +156,16 @@ struct fib_rule;
 
 struct fib_table;
 struct fib_result {
-       __be32          prefix;
-       unsigned char   prefixlen;
-       unsigned char   nh_sel;
-       unsigned char   type;
-       unsigned char   scope;
-       u32             tclassid;
-       struct fib_info *fi;
-       struct fib_table *table;
-       struct hlist_head *fa_head;
+       __be32                  prefix;
+       unsigned char           prefixlen;
+       unsigned char           nh_sel;
+       unsigned char           type;
+       unsigned char           scope;
+       u32                     tclassid;
+       struct fib_nh_common    *nhc;
+       struct fib_info         *fi;
+       struct fib_table        *table;
+       struct hlist_head       *fa_head;
 };
 
 struct fib_result_nl {
@@ -182,11 +183,10 @@ struct fib_result_nl {
        int             err;
 };
 
-#ifdef CONFIG_IP_ROUTE_MULTIPATH
-#define FIB_RES_NH(res)                ((res).fi->fib_nh[(res).nh_sel])
-#else /* CONFIG_IP_ROUTE_MULTIPATH */
-#define FIB_RES_NH(res)                ((res).fi->fib_nh[0])
-#endif /* CONFIG_IP_ROUTE_MULTIPATH */
+static inline struct fib_nh_common *fib_info_nhc(struct fib_info *fi, int nhsel)
+{
+       return &fi->fib_nh[nhsel].nh_common;
+}
 
 #ifdef CONFIG_IP_MULTIPLE_TABLES
 #define FIB_TABLE_HASHSZ 256
@@ -195,18 +195,11 @@ struct fib_result_nl {
 #endif
 
 __be32 fib_info_update_nh_saddr(struct net *net, struct fib_nh *nh);
+__be32 fib_result_prefsrc(struct net *net, struct fib_result *res);
 
-#define FIB_RES_SADDR(net, res)                                \
-       ((FIB_RES_NH(res).nh_saddr_genid ==             \
-         atomic_read(&(net)->ipv4.dev_addr_genid)) ?   \
-        FIB_RES_NH(res).nh_saddr :                     \
-        fib_info_update_nh_saddr((net), &FIB_RES_NH(res)))
-#define FIB_RES_GW(res)                        (FIB_RES_NH(res).fib_nh_gw4)
-#define FIB_RES_DEV(res)               (FIB_RES_NH(res).fib_nh_dev)
-#define FIB_RES_OIF(res)               (FIB_RES_NH(res).fib_nh_oif)
-
-#define FIB_RES_PREFSRC(net, res)      ((res).fi->fib_prefsrc ? : \
-                                        FIB_RES_SADDR(net, res))
+#define FIB_RES_NHC(res)               ((res).nhc)
+#define FIB_RES_DEV(res)       (FIB_RES_NHC(res)->nhc_dev)
+#define FIB_RES_OIF(res)       (FIB_RES_NHC(res)->nhc_oif)
 
 struct fib_entry_notifier_info {
        struct fib_notifier_info info; /* must be first */
@@ -453,10 +446,12 @@ struct fib_table *fib_trie_table(u32 id, struct fib_table *alias);
 static inline void fib_combine_itag(u32 *itag, const struct fib_result *res)
 {
 #ifdef CONFIG_IP_ROUTE_CLASSID
+       struct fib_nh_common *nhc = res->nhc;
+       struct fib_nh *nh = container_of(nhc, struct fib_nh, nh_common);
 #ifdef CONFIG_IP_MULTIPLE_TABLES
        u32 rtag;
 #endif
-       *itag = FIB_RES_NH(*res).nh_tclassid<<16;
+       *itag = nh->nh_tclassid << 16;
 #ifdef CONFIG_IP_MULTIPLE_TABLES
        rtag = res->tclassid;
        if (*itag == 0)

diff --git a/net/core/filter.c b/net/core/filter.c
index cdaafa3..08b53af 100644 (file)
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -4555,11 +4555,11 @@ static int bpf_fib_set_fwd_params(struct bpf_fib_lookup *params,
 static int bpf_ipv4_fib_lookup(struct net *net, struct bpf_fib_lookup *params,
                               u32 flags, bool check_mtu)
 {
+       struct fib_nh_common *nhc;
        struct in_device *in_dev;
        struct neighbour *neigh;
        struct net_device *dev;
        struct fib_result res;
-       struct fib_nh *nh;
        struct flowi4 fl4;
        int err;
        u32 mtu;
@@ -4632,15 +4632,15 @@ static int bpf_ipv4_fib_lookup(struct net *net, struct bpf_fib_lookup *params,
                        return BPF_FIB_LKUP_RET_FRAG_NEEDED;
        }
 
-       nh = &res.fi->fib_nh[res.nh_sel];
+       nhc = res.nhc;
 
        /* do not handle lwt encaps right now */
-       if (nh->fib_nh_lws)
+       if (nhc->nhc_lwtstate)
                return BPF_FIB_LKUP_RET_UNSUPP_LWT;
 
-       dev = nh->fib_nh_dev;
-       if (nh->fib_nh_gw4)
-               params->ipv4_dst = nh->fib_nh_gw4;
+       dev = nhc->nhc_dev;
+       if (nhc->nhc_has_gw)
+               params->ipv4_dst = nhc->nhc_gw.ipv4;
 
        params->rt_metric = res.fi->fib_priority;
 

diff --git a/net/ipv4/fib_frontend.c b/net/ipv4/fib_frontend.c
index ffbe243..15f779b 100644 (file)
--- a/net/ipv4/fib_frontend.c
+++ b/net/ipv4/fib_frontend.c
@@ -307,7 +307,7 @@ __be32 fib_compute_spec_dst(struct sk_buff *skb)
                        .flowi4_mark = vmark ? skb->mark : 0,
                };
                if (!fib_lookup(net, &fl4, &res, 0))
-                       return FIB_RES_PREFSRC(net, res);
+                       return fib_result_prefsrc(net, &res);
        } else {
                scope = RT_SCOPE_LINK;
        }
@@ -390,7 +390,7 @@ static int __fib_validate_source(struct sk_buff *skb, __be32 src, __be32 dst,
 
        dev_match = fib_info_nh_uses_dev(res.fi, dev);
        if (dev_match) {
-               ret = FIB_RES_NH(res).fib_nh_scope >= RT_SCOPE_HOST;
+               ret = FIB_RES_NHC(res)->nhc_scope >= RT_SCOPE_HOST;
                return ret;
        }
        if (no_addr)
@@ -402,7 +402,7 @@ static int __fib_validate_source(struct sk_buff *skb, __be32 src, __be32 dst,
        ret = 0;
        if (fib_lookup(net, &fl4, &res, FIB_LOOKUP_IGNORE_LINKSTATE) == 0) {
                if (res.type == RTN_UNICAST)
-                       ret = FIB_RES_NH(res).fib_nh_scope >= RT_SCOPE_HOST;
+                       ret = FIB_RES_NHC(res)->nhc_scope >= RT_SCOPE_HOST;
        }
        return ret;
 

diff --git a/net/ipv4/fib_lookup.h b/net/ipv4/fib_lookup.h
index e6ff282..7945f05 100644 (file)
--- a/net/ipv4/fib_lookup.h
+++ b/net/ipv4/fib_lookup.h
@@ -45,6 +45,7 @@ static inline void fib_result_assign(struct fib_result *res,
 {
        /* we used to play games with refcounts, but we now use RCU */
        res->fi = fi;
+       res->nhc = fib_info_nhc(fi, 0);
 }
 
 struct fib_prop {

diff --git a/net/ipv4/fib_semantics.c b/net/ipv4/fib_semantics.c
index df777af..42666a4 100644 (file)
--- a/net/ipv4/fib_semantics.c
+++ b/net/ipv4/fib_semantics.c
@@ -1075,6 +1075,21 @@ __be32 fib_info_update_nh_saddr(struct net *net, struct fib_nh *nh)
        return nh->nh_saddr;
 }
 
+__be32 fib_result_prefsrc(struct net *net, struct fib_result *res)
+{
+       struct fib_nh_common *nhc = res->nhc;
+       struct fib_nh *nh;
+
+       if (res->fi->fib_prefsrc)
+               return res->fi->fib_prefsrc;
+
+       nh = container_of(nhc, struct fib_nh, nh_common);
+       if (nh->nh_saddr_genid == atomic_read(&net->ipv4.dev_addr_genid))
+               return nh->nh_saddr;
+
+       return fib_info_update_nh_saddr(net, nh);
+}
+
 static bool fib_valid_prefsrc(struct fib_config *cfg, __be32 fib_prefsrc)
 {
        if (cfg->fc_type != RTN_LOCAL || !cfg->fc_dst ||
@@ -1762,20 +1777,22 @@ void fib_select_multipath(struct fib_result *res, int hash)
        struct net *net = fi->fib_net;
        bool first = false;
 
-       for_nexthops(fi) {
+       change_nexthops(fi) {
                if (net->ipv4.sysctl_fib_multipath_use_neigh) {
-                       if (!fib_good_nh(nh))
+                       if (!fib_good_nh(nexthop_nh))
                                continue;
                        if (!first) {
                                res->nh_sel = nhsel;
+                               res->nhc = &nexthop_nh->nh_common;
                                first = true;
                        }
                }
 
-               if (hash > atomic_read(&nh->fib_nh_upper_bound))
+               if (hash > atomic_read(&nexthop_nh->fib_nh_upper_bound))
                        continue;
 
                res->nh_sel = nhsel;
+               res->nhc = &nexthop_nh->nh_common;
                return;
        } endfor_nexthops(fi);
 }
@@ -1802,5 +1819,5 @@ void fib_select_path(struct net *net, struct fib_result *res,
 
 check_saddr:
        if (!fl4->saddr)
-               fl4->saddr = FIB_RES_PREFSRC(net, *res);
+               fl4->saddr = fib_result_prefsrc(net, res);
 }

diff --git a/net/ipv4/fib_trie.c b/net/ipv4/fib_trie.c
index 13b3327..334f723 100644 (file)
--- a/net/ipv4/fib_trie.c
+++ b/net/ipv4/fib_trie.c
@@ -1470,17 +1470,17 @@ found:
                if (fi->fib_flags & RTNH_F_DEAD)
                        continue;
                for (nhsel = 0; nhsel < fi->fib_nhs; nhsel++) {
-                       const struct fib_nh *nh = &fi->fib_nh[nhsel];
+                       struct fib_nh_common *nhc = fib_info_nhc(fi, nhsel);
 
-                       if (nh->fib_nh_flags & RTNH_F_DEAD)
+                       if (nhc->nhc_flags & RTNH_F_DEAD)
                                continue;
-                       if (ip_ignore_linkdown(nh->fib_nh_dev) &&
-                           nh->fib_nh_flags & RTNH_F_LINKDOWN &&
+                       if (ip_ignore_linkdown(nhc->nhc_dev) &&
+                           nhc->nhc_flags & RTNH_F_LINKDOWN &&
                            !(fib_flags & FIB_LOOKUP_IGNORE_LINKSTATE))
                                continue;
                        if (!(flp->flowi4_flags & FLOWI_FLAG_SKIP_NH_OIF)) {
                                if (flp->flowi4_oif &&
-                                   flp->flowi4_oif != nh->fib_nh_oif)
+                                   flp->flowi4_oif != nhc->nhc_oif)
                                        continue;
                        }
 
@@ -1490,6 +1490,7 @@ found:
                        res->prefix = htonl(n->key);
                        res->prefixlen = KEYLENGTH - fa->fa_slen;
                        res->nh_sel = nhsel;
+                       res->nhc = nhc;
                        res->type = fa->fa_type;
                        res->scope = fi->fib_scope;
                        res->fi = fi;
@@ -1498,7 +1499,7 @@ found:
 #ifdef CONFIG_IP_FIB_TRIE_STATS
                        this_cpu_inc(stats->semantic_match_passed);
 #endif
-                       trace_fib_table_lookup(tb->tb_id, flp, &nh->nh_common, err);
+                       trace_fib_table_lookup(tb->tb_id, flp, nhc, err);
 
                        return err;
                }

diff --git a/net/ipv4/route.c b/net/ipv4/route.c
index 7977514..f3f2adf 100644 (file)
--- a/net/ipv4/route.c
+++ b/net/ipv4/route.c
@@ -778,8 +778,10 @@ static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flow
                        neigh_event_send(n, NULL);
                } else {
                        if (fib_lookup(net, fl4, &res, 0) == 0) {
-                               struct fib_nh *nh = &FIB_RES_NH(res);
+                               struct fib_nh_common *nhc = FIB_RES_NHC(res);
+                               struct fib_nh *nh;
 
+                               nh = container_of(nhc, struct fib_nh, nh_common);
                                update_or_create_fnhe(nh, fl4->daddr, new_gw,
                                                0, false,
                                                jiffies + ip_rt_gc_timeout);
@@ -1027,8 +1029,10 @@ static void __ip_rt_update_pmtu(struct rtable *rt, struct flowi4 *fl4, u32 mtu)
 
        rcu_read_lock();
        if (fib_lookup(dev_net(dst->dev), fl4, &res, 0) == 0) {
-               struct fib_nh *nh = &FIB_RES_NH(res);
+               struct fib_nh_common *nhc = FIB_RES_NHC(res);
+               struct fib_nh *nh;
 
+               nh = container_of(nhc, struct fib_nh, nh_common);
                update_or_create_fnhe(nh, fl4->daddr, 0, mtu, lock,
                                      jiffies + ip_rt_mtu_expires);
        }
@@ -1235,7 +1239,7 @@ void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt)
 
                rcu_read_lock();
                if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res, 0) == 0)
-                       src = FIB_RES_PREFSRC(dev_net(rt->dst.dev), res);
+                       src = fib_result_prefsrc(dev_net(rt->dst.dev), &res);
                else
                        src = inet_select_addr(rt->dst.dev,
                                               rt_nexthop(rt, iph->daddr),
@@ -1354,9 +1358,9 @@ static struct fib_nh_exception *find_exception(struct fib_nh *nh, __be32 daddr)
 
 u32 ip_mtu_from_fib_result(struct fib_result *res, __be32 daddr)
 {
+       struct fib_nh_common *nhc = res->nhc;
+       struct net_device *dev = nhc->nhc_dev;
        struct fib_info *fi = res->fi;
-       struct fib_nh *nh = &fi->fib_nh[res->nh_sel];
-       struct net_device *dev = nh->fib_nh_dev;
        u32 mtu = 0;
 
        if (dev_net(dev)->ipv4.sysctl_ip_fwd_use_pmtu ||
@@ -1364,6 +1368,7 @@ u32 ip_mtu_from_fib_result(struct fib_result *res, __be32 daddr)
                mtu = fi->fib_mtu;
 
        if (likely(!mtu)) {
+               struct fib_nh *nh = container_of(nhc, struct fib_nh, nh_common);
                struct fib_nh_exception *fnhe;
 
                fnhe = find_exception(nh, daddr);
@@ -1374,7 +1379,7 @@ u32 ip_mtu_from_fib_result(struct fib_result *res, __be32 daddr)
        if (likely(!mtu))
                mtu = min(READ_ONCE(dev->mtu), IP_MAX_MTU);
 
-       return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu);
+       return mtu - lwtunnel_headroom(nhc->nhc_lwtstate, mtu);
 }
 
 static bool rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe,
@@ -1529,7 +1534,8 @@ static void rt_set_nexthop(struct rtable *rt, __be32 daddr,
        bool cached = false;
 
        if (fi) {
-               struct fib_nh *nh = &FIB_RES_NH(*res);
+               struct fib_nh_common *nhc = FIB_RES_NHC(*res);
+               struct fib_nh *nh = container_of(nhc, struct fib_nh, nh_common);
 
                if (nh->fib_nh_gw4 && nh->fib_nh_scope == RT_SCOPE_LINK) {
                        rt->rt_gateway = nh->fib_nh_gw4;
@@ -1699,15 +1705,18 @@ static int __mkroute_input(struct sk_buff *skb,
                           struct in_device *in_dev,
                           __be32 daddr, __be32 saddr, u32 tos)
 {
+       struct fib_nh_common *nhc = FIB_RES_NHC(*res);
+       struct net_device *dev = nhc->nhc_dev;
        struct fib_nh_exception *fnhe;
        struct rtable *rth;
+       struct fib_nh *nh;
        int err;
        struct in_device *out_dev;
        bool do_cache;
        u32 itag = 0;
 
        /* get a working reference to the output device */
-       out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
+       out_dev = __in_dev_get_rcu(dev);
        if (!out_dev) {
                net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n");
                return -EINVAL;
@@ -1724,10 +1733,13 @@ static int __mkroute_input(struct sk_buff *skb,
 
        do_cache = res->fi && !itag;
        if (out_dev == in_dev && err && IN_DEV_TX_REDIRECTS(out_dev) &&
-           skb->protocol == htons(ETH_P_IP) &&
-           (IN_DEV_SHARED_MEDIA(out_dev) ||
-            inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res))))
-               IPCB(skb)->flags |= IPSKB_DOREDIRECT;
+           skb->protocol == htons(ETH_P_IP)) {
+               __be32 gw = nhc->nhc_family == AF_INET ? nhc->nhc_gw.ipv4 : 0;
+
+               if (IN_DEV_SHARED_MEDIA(out_dev) ||
+                   inet_addr_onlink(out_dev, saddr, gw))
+                       IPCB(skb)->flags |= IPSKB_DOREDIRECT;
+       }
 
        if (skb->protocol != htons(ETH_P_IP)) {
                /* Not IP (i.e. ARP). Do not create route, if it is
@@ -1744,12 +1756,13 @@ static int __mkroute_input(struct sk_buff *skb,
                }
        }
 
-       fnhe = find_exception(&FIB_RES_NH(*res), daddr);
+       nh = container_of(nhc, struct fib_nh, nh_common);
+       fnhe = find_exception(nh, daddr);
        if (do_cache) {
                if (fnhe)
                        rth = rcu_dereference(fnhe->fnhe_rth_input);
                else
-                       rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input);
+                       rth = rcu_dereference(nh->nh_rth_input);
                if (rt_cache_valid(rth)) {
                        skb_dst_set_noref(skb, &rth->dst);
                        goto out;
@@ -2043,7 +2056,11 @@ local_input:
        do_cache = false;
        if (res->fi) {
                if (!itag) {
-                       rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input);
+                       struct fib_nh_common *nhc = FIB_RES_NHC(*res);
+                       struct fib_nh *nh;
+
+                       nh = container_of(nhc, struct fib_nh, nh_common);
+                       rth = rcu_dereference(nh->nh_rth_input);
                        if (rt_cache_valid(rth)) {
                                skb_dst_set_noref(skb, &rth->dst);
                                err = 0;
@@ -2073,15 +2090,17 @@ local_input:
        }
 
        if (do_cache) {
-               struct fib_nh *nh = &FIB_RES_NH(*res);
+               struct fib_nh_common *nhc = FIB_RES_NHC(*res);
+               struct fib_nh *nh;
 
-               rth->dst.lwtstate = lwtstate_get(nh->fib_nh_lws);
+               rth->dst.lwtstate = lwtstate_get(nhc->nhc_lwtstate);
                if (lwtunnel_input_redirect(rth->dst.lwtstate)) {
                        WARN_ON(rth->dst.input == lwtunnel_input);
                        rth->dst.lwtstate->orig_input = rth->dst.input;
                        rth->dst.input = lwtunnel_input;
                }
 
+               nh = container_of(nhc, struct fib_nh, nh_common);
                if (unlikely(!rt_cache_route(nh, rth)))
                        rt_add_uncached_list(rth);
        }
@@ -2253,8 +2272,9 @@ static struct rtable *__mkroute_output(const struct fib_result *res,
        fnhe = NULL;
        do_cache &= fi != NULL;
        if (fi) {
+               struct fib_nh_common *nhc = FIB_RES_NHC(*res);
+               struct fib_nh *nh = container_of(nhc, struct fib_nh, nh_common);
                struct rtable __rcu **prth;
-               struct fib_nh *nh = &FIB_RES_NH(*res);
 
                fnhe = find_exception(nh, fl4->daddr);
                if (!do_cache)
@@ -2264,8 +2284,8 @@ static struct rtable *__mkroute_output(const struct fib_result *res,
                } else {
                        if (unlikely(fl4->flowi4_flags &
                                     FLOWI_FLAG_KNOWN_NH &&
-                                    !(nh->fib_nh_gw4 &&
-                                      nh->fib_nh_scope == RT_SCOPE_LINK))) {
+                                    !(nhc->nhc_has_gw &&
+                                      nhc->nhc_scope == RT_SCOPE_LINK))) {
                                do_cache = false;
                                goto add;
                        }