ipv6: move inet6_sk(sk)->rx_dst_cookie to sk->sk_rx_dst_cookie

Increase cache locality by moving rx_dst_coookie next to sk->sk_rx_dst

This removes one or two cache line misses in IPv6 early demux (TCP/UDP)

Signed-off-by: Eric Dumazet <edumazet@google.com>
Acked-by: Soheil Hassas Yeganeh <soheil@google.com>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/include/linux/ipv6.h b/include/linux/ipv6.h
index ef4a698..c383630 100644 (file)
--- a/include/linux/ipv6.h
+++ b/include/linux/ipv6.h
@@ -282,7 +282,6 @@ struct ipv6_pinfo {
        __be32                  rcv_flowinfo;
 
        __u32                   dst_cookie;
-       __u32                   rx_dst_cookie;
 
        struct ipv6_mc_socklist __rcu *ipv6_mc_list;
        struct ipv6_ac_socklist *ipv6_ac_list;

diff --git a/include/net/sock.h b/include/net/sock.h
index 0bfb3f1..99c4194 100644 (file)
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -260,6 +260,7 @@ struct bpf_local_storage;
   *    @sk_wq: sock wait queue and async head
   *    @sk_rx_dst: receive input route used by early demux
   *    @sk_rx_dst_ifindex: ifindex for @sk_rx_dst
+  *    @sk_rx_dst_cookie: cookie for @sk_rx_dst
   *    @sk_dst_cache: destination cache
   *    @sk_dst_pending_confirm: need to confirm neighbour
   *    @sk_policy: flow policy
@@ -432,6 +433,7 @@ struct sock {
 #endif
        struct dst_entry        *sk_rx_dst;
        int                     sk_rx_dst_ifindex;
+       u32                     sk_rx_dst_cookie;
 
        struct dst_entry __rcu  *sk_dst_cache;
        atomic_t                sk_omem_alloc;

diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c
index 3e8669b..50d9578 100644 (file)
--- a/net/ipv6/tcp_ipv6.c
+++ b/net/ipv6/tcp_ipv6.c
@@ -109,7 +109,7 @@ static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
 
                sk->sk_rx_dst = dst;
                sk->sk_rx_dst_ifindex = skb->skb_iif;
-               tcp_inet6_sk(sk)->rx_dst_cookie = rt6_get_cookie(rt);
+               sk->sk_rx_dst_cookie = rt6_get_cookie(rt);
        }
 }
 
@@ -1511,7 +1511,7 @@ static int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)
                if (dst) {
                        if (sk->sk_rx_dst_ifindex != skb->skb_iif ||
                            INDIRECT_CALL_1(dst->ops->check, ip6_dst_check,
-                                           dst, np->rx_dst_cookie) == NULL) {
+                                           dst, sk->sk_rx_dst_cookie) == NULL) {
                                dst_release(dst);
                                sk->sk_rx_dst = NULL;
                        }
@@ -1872,7 +1872,7 @@ INDIRECT_CALLABLE_SCOPE void tcp_v6_early_demux(struct sk_buff *skb)
                        struct dst_entry *dst = READ_ONCE(sk->sk_rx_dst);
 
                        if (dst)
-                               dst = dst_check(dst, tcp_inet6_sk(sk)->rx_dst_cookie);
+                               dst = dst_check(dst, sk->sk_rx_dst_cookie);
                        if (dst &&
                            sk->sk_rx_dst_ifindex == skb->skb_iif)
                                skb_dst_set_noref(skb, dst);

diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
index 8d78523..14a94cd 100644 (file)
--- a/net/ipv6/udp.c
+++ b/net/ipv6/udp.c
@@ -884,7 +884,7 @@ static void udp6_sk_rx_dst_set(struct sock *sk, struct dst_entry *dst)
        if (udp_sk_rx_dst_set(sk, dst)) {
                const struct rt6_info *rt = (const struct rt6_info *)dst;
 
-               inet6_sk(sk)->rx_dst_cookie = rt6_get_cookie(rt);
+               sk->sk_rx_dst_cookie = rt6_get_cookie(rt);
        }
 }
 
@@ -1073,7 +1073,7 @@ INDIRECT_CALLABLE_SCOPE void udp_v6_early_demux(struct sk_buff *skb)
        dst = READ_ONCE(sk->sk_rx_dst);
 
        if (dst)
-               dst = dst_check(dst, inet6_sk(sk)->rx_dst_cookie);
+               dst = dst_check(dst, sk->sk_rx_dst_cookie);
        if (dst) {
                /* set noref for now.
                 * any place which wants to hold dst has to call