xdp: Move conversion to xdp_frame out of map functions

All map redirect functions except XSK maps convert xdp_buff to xdp_frame
before enqueueing it. So move this conversion of out the map functions
and into xdp_do_redirect(). This removes a bit of duplicated code, but more
importantly it makes it possible to support caller-allocated xdp_frame
structures, which will be added in a subsequent commit.

Signed-off-by: Toke Høiland-Jørgensen <toke@redhat.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Link: https://lore.kernel.org/bpf/20220103150812.87914-5-toke@redhat.com

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 2675313..6e947cd 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1669,17 +1669,17 @@ void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
 struct btf *bpf_get_btf_vmlinux(void);
 
 /* Map specifics */
-struct xdp_buff;
+struct xdp_frame;
 struct sk_buff;
 struct bpf_dtab_netdev;
 struct bpf_cpu_map_entry;
 
 void __dev_flush(void);
-int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp,
+int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
                    struct net_device *dev_rx);
-int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp,
+int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,
                    struct net_device *dev_rx);
-int dev_map_enqueue_multi(struct xdp_buff *xdp, struct net_device *dev_rx,
+int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx,
                          struct bpf_map *map, bool exclude_ingress);
 int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb,
                             struct bpf_prog *xdp_prog);
@@ -1688,7 +1688,7 @@ int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb,
                           bool exclude_ingress);
 
 void __cpu_map_flush(void);
-int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp,
+int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf,
                    struct net_device *dev_rx);
 int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu,
                             struct sk_buff *skb);
@@ -1866,26 +1866,26 @@ static inline void __dev_flush(void)
 {
 }
 
-struct xdp_buff;
+struct xdp_frame;
 struct bpf_dtab_netdev;
 struct bpf_cpu_map_entry;
 
 static inline
-int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp,
+int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
                    struct net_device *dev_rx)
 {
        return 0;
 }
 
 static inline
-int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp,
+int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,
                    struct net_device *dev_rx)
 {
        return 0;
 }
 
 static inline
-int dev_map_enqueue_multi(struct xdp_buff *xdp, struct net_device *dev_rx,
+int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx,
                          struct bpf_map *map, bool exclude_ingress)
 {
        return 0;
@@ -1913,7 +1913,7 @@ static inline void __cpu_map_flush(void)
 }
 
 static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu,
-                                 struct xdp_buff *xdp,
+                                 struct xdp_frame *xdpf,
                                  struct net_device *dev_rx)
 {
        return 0;

diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c
index 0421061..b3e6b94 100644 (file)
--- a/kernel/bpf/cpumap.c
+++ b/kernel/bpf/cpumap.c
@@ -746,15 +746,9 @@ static void bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf)
                list_add(&bq->flush_node, flush_list);
 }
 
-int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp,
+int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf,
                    struct net_device *dev_rx)
 {
-       struct xdp_frame *xdpf;
-
-       xdpf = xdp_convert_buff_to_frame(xdp);
-       if (unlikely(!xdpf))
-               return -EOVERFLOW;
-
        /* Info needed when constructing SKB on remote CPU */
        xdpf->dev_rx = dev_rx;
 

diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c
index 6feea29..fe019db 100644 (file)
--- a/kernel/bpf/devmap.c
+++ b/kernel/bpf/devmap.c
@@ -467,24 +467,19 @@ static void bq_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
        bq->q[bq->count++] = xdpf;
 }
 
-static inline int __xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp,
+static inline int __xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
                                struct net_device *dev_rx,
                                struct bpf_prog *xdp_prog)
 {
-       struct xdp_frame *xdpf;
        int err;
 
        if (!dev->netdev_ops->ndo_xdp_xmit)
                return -EOPNOTSUPP;
 
-       err = xdp_ok_fwd_dev(dev, xdp->data_end - xdp->data);
+       err = xdp_ok_fwd_dev(dev, xdpf->len);
        if (unlikely(err))
                return err;
 
-       xdpf = xdp_convert_buff_to_frame(xdp);
-       if (unlikely(!xdpf))
-               return -EOVERFLOW;
-
        bq_enqueue(dev, xdpf, dev_rx, xdp_prog);
        return 0;
 }
@@ -520,27 +515,27 @@ static u32 dev_map_bpf_prog_run_skb(struct sk_buff *skb, struct bpf_dtab_netdev
        return act;
 }
 
-int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp,
+int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
                    struct net_device *dev_rx)
 {
-       return __xdp_enqueue(dev, xdp, dev_rx, NULL);
+       return __xdp_enqueue(dev, xdpf, dev_rx, NULL);
 }
 
-int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp,
+int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,
                    struct net_device *dev_rx)
 {
        struct net_device *dev = dst->dev;
 
-       return __xdp_enqueue(dev, xdp, dev_rx, dst->xdp_prog);
+       return __xdp_enqueue(dev, xdpf, dev_rx, dst->xdp_prog);
 }
 
-static bool is_valid_dst(struct bpf_dtab_netdev *obj, struct xdp_buff *xdp)
+static bool is_valid_dst(struct bpf_dtab_netdev *obj, struct xdp_frame *xdpf)
 {
        if (!obj ||
            !obj->dev->netdev_ops->ndo_xdp_xmit)
                return false;
 
-       if (xdp_ok_fwd_dev(obj->dev, xdp->data_end - xdp->data))
+       if (xdp_ok_fwd_dev(obj->dev, xdpf->len))
                return false;
 
        return true;
@@ -586,14 +581,13 @@ static int get_upper_ifindexes(struct net_device *dev, int *indexes)
        return n;
 }
 
-int dev_map_enqueue_multi(struct xdp_buff *xdp, struct net_device *dev_rx,
+int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx,
                          struct bpf_map *map, bool exclude_ingress)
 {
        struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
        struct bpf_dtab_netdev *dst, *last_dst = NULL;
        int excluded_devices[1+MAX_NEST_DEV];
        struct hlist_head *head;
-       struct xdp_frame *xdpf;
        int num_excluded = 0;
        unsigned int i;
        int err;
@@ -603,15 +597,11 @@ int dev_map_enqueue_multi(struct xdp_buff *xdp, struct net_device *dev_rx,
                excluded_devices[num_excluded++] = dev_rx->ifindex;
        }
 
-       xdpf = xdp_convert_buff_to_frame(xdp);
-       if (unlikely(!xdpf))
-               return -EOVERFLOW;
-
        if (map->map_type == BPF_MAP_TYPE_DEVMAP) {
                for (i = 0; i < map->max_entries; i++) {
                        dst = rcu_dereference_check(dtab->netdev_map[i],
                                                    rcu_read_lock_bh_held());
-                       if (!is_valid_dst(dst, xdp))
+                       if (!is_valid_dst(dst, xdpf))
                                continue;
 
                        if (is_ifindex_excluded(excluded_devices, num_excluded, dst->dev->ifindex))
@@ -634,7 +624,7 @@ int dev_map_enqueue_multi(struct xdp_buff *xdp, struct net_device *dev_rx,
                        head = dev_map_index_hash(dtab, i);
                        hlist_for_each_entry_rcu(dst, head, index_hlist,
                                                 lockdep_is_held(&dtab->index_lock)) {
-                               if (!is_valid_dst(dst, xdp))
+                               if (!is_valid_dst(dst, xdpf))
                                        continue;
 
                                if (is_ifindex_excluded(excluded_devices, num_excluded,

diff --git a/net/core/filter.c b/net/core/filter.c
index cac2be5..e2b8305 100644 (file)
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -3964,12 +3964,24 @@ int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp,
        enum bpf_map_type map_type = ri->map_type;
        void *fwd = ri->tgt_value;
        u32 map_id = ri->map_id;
+       struct xdp_frame *xdpf;
        struct bpf_map *map;
        int err;
 
        ri->map_id = 0; /* Valid map id idr range: [1,INT_MAX[ */
        ri->map_type = BPF_MAP_TYPE_UNSPEC;
 
+       if (map_type == BPF_MAP_TYPE_XSKMAP) {
+               err = __xsk_map_redirect(fwd, xdp);
+               goto out;
+       }
+
+       xdpf = xdp_convert_buff_to_frame(xdp);
+       if (unlikely(!xdpf)) {
+               err = -EOVERFLOW;
+               goto err;
+       }
+
        switch (map_type) {
        case BPF_MAP_TYPE_DEVMAP:
                fallthrough;
@@ -3977,17 +3989,14 @@ int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp,
                map = READ_ONCE(ri->map);
                if (unlikely(map)) {
                        WRITE_ONCE(ri->map, NULL);
-                       err = dev_map_enqueue_multi(xdp, dev, map,
+                       err = dev_map_enqueue_multi(xdpf, dev, map,
                                                    ri->flags & BPF_F_EXCLUDE_INGRESS);
                } else {
-                       err = dev_map_enqueue(fwd, xdp, dev);
+                       err = dev_map_enqueue(fwd, xdpf, dev);
                }
                break;
        case BPF_MAP_TYPE_CPUMAP:
-               err = cpu_map_enqueue(fwd, xdp, dev);
-               break;
-       case BPF_MAP_TYPE_XSKMAP:
-               err = __xsk_map_redirect(fwd, xdp);
+               err = cpu_map_enqueue(fwd, xdpf, dev);
                break;
        case BPF_MAP_TYPE_UNSPEC:
                if (map_id == INT_MAX) {
@@ -3996,7 +4005,7 @@ int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp,
                                err = -EINVAL;
                                break;
                        }
-                       err = dev_xdp_enqueue(fwd, xdp, dev);
+                       err = dev_xdp_enqueue(fwd, xdpf, dev);
                        break;
                }
                fallthrough;
@@ -4004,6 +4013,7 @@ int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp,
                err = -EBADRQC;
        }
 
+out:
        if (unlikely(err))
                goto err;