dpaa2-eth: add bulking to XDP_TX

Add driver level bulking to the XDP_TX action.

An array of frame descriptors is held for each Tx frame queue and
populated accordingly when the action returned by the XDP program is
XDP_TX. The frames will be actually enqueued only when the array is
filled. At the end of the NAPI cycle a flush on the queued frames is
performed in order to enqueue the remaining FDs.

Signed-off-by: Ioana Ciornei <ioana.ciornei@nxp.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c
index 5fbaa51..fe3806d 100644 (file)
--- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c
+++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c
@@ -273,13 +273,43 @@ static int dpaa2_eth_xdp_flush(struct dpaa2_eth_priv *priv,
        return total_enqueued;
 }
 
-static int xdp_enqueue(struct dpaa2_eth_priv *priv, struct dpaa2_fd *fd,
-                      void *buf_start, u16 queue_id)
+static void xdp_tx_flush(struct dpaa2_eth_priv *priv,
+                        struct dpaa2_eth_channel *ch,
+                        struct dpaa2_eth_fq *fq)
+{
+       struct rtnl_link_stats64 *percpu_stats;
+       struct dpaa2_fd *fds;
+       int enqueued, i;
+
+       percpu_stats = this_cpu_ptr(priv->percpu_stats);
+
+       // enqueue the array of XDP_TX frames
+       enqueued = dpaa2_eth_xdp_flush(priv, fq, &fq->xdp_tx_fds);
+
+       /* update statistics */
+       percpu_stats->tx_packets += enqueued;
+       fds = fq->xdp_tx_fds.fds;
+       for (i = 0; i < enqueued; i++) {
+               percpu_stats->tx_bytes += dpaa2_fd_get_len(&fds[i]);
+               ch->stats.xdp_tx++;
+       }
+       for (i = enqueued; i < fq->xdp_tx_fds.num; i++) {
+               xdp_release_buf(priv, ch, dpaa2_fd_get_addr(&fds[i]));
+               percpu_stats->tx_errors++;
+               ch->stats.xdp_tx_err++;
+       }
+       fq->xdp_tx_fds.num = 0;
+}
+
+static void xdp_enqueue(struct dpaa2_eth_priv *priv,
+                       struct dpaa2_eth_channel *ch,
+                       struct dpaa2_fd *fd,
+                       void *buf_start, u16 queue_id)
 {
-       struct dpaa2_eth_fq *fq;
        struct dpaa2_faead *faead;
+       struct dpaa2_fd *dest_fd;
+       struct dpaa2_eth_fq *fq;
        u32 ctrl, frc;
-       int i, err;
 
        /* Mark the egress frame hardware annotation area as valid */
        frc = dpaa2_fd_get_frc(fd);
@@ -296,13 +326,13 @@ static int xdp_enqueue(struct dpaa2_eth_priv *priv, struct dpaa2_fd *fd,
        faead->conf_fqid = 0;
 
        fq = &priv->fq[queue_id];
-       for (i = 0; i < DPAA2_ETH_ENQUEUE_RETRIES; i++) {
-               err = priv->enqueue(priv, fq, fd, 0, 1, NULL);
-               if (err != -EBUSY)
-                       break;
-       }
+       dest_fd = &fq->xdp_tx_fds.fds[fq->xdp_tx_fds.num++];
+       memcpy(dest_fd, fd, sizeof(*dest_fd));
 
-       return err;
+       if (fq->xdp_tx_fds.num < DEV_MAP_BULK_SIZE)
+               return;
+
+       xdp_tx_flush(priv, ch, fq);
 }
 
 static u32 run_xdp(struct dpaa2_eth_priv *priv,
@@ -311,14 +341,11 @@ static u32 run_xdp(struct dpaa2_eth_priv *priv,
                   struct dpaa2_fd *fd, void *vaddr)
 {
        dma_addr_t addr = dpaa2_fd_get_addr(fd);
-       struct rtnl_link_stats64 *percpu_stats;
        struct bpf_prog *xdp_prog;
        struct xdp_buff xdp;
        u32 xdp_act = XDP_PASS;
        int err;
 
-       percpu_stats = this_cpu_ptr(priv->percpu_stats);
-
        rcu_read_lock();
 
        xdp_prog = READ_ONCE(ch->xdp.prog);
@@ -344,16 +371,7 @@ static u32 run_xdp(struct dpaa2_eth_priv *priv,
        case XDP_PASS:
                break;
        case XDP_TX:
-               err = xdp_enqueue(priv, fd, vaddr, rx_fq->flowid);
-               if (err) {
-                       xdp_release_buf(priv, ch, addr);
-                       percpu_stats->tx_errors++;
-                       ch->stats.xdp_tx_err++;
-               } else {
-                       percpu_stats->tx_packets++;
-                       percpu_stats->tx_bytes += dpaa2_fd_get_len(fd);
-                       ch->stats.xdp_tx++;
-               }
+               xdp_enqueue(priv, ch, fd, vaddr, rx_fq->flowid);
                break;
        default:
                bpf_warn_invalid_xdp_action(xdp_act);
@@ -1175,6 +1193,7 @@ static int dpaa2_eth_poll(struct napi_struct *napi, int budget)
        int store_cleaned, work_done;
        struct list_head rx_list;
        int retries = 0;
+       u16 flowid;
        int err;
 
        ch = container_of(napi, struct dpaa2_eth_channel, napi);
@@ -1197,6 +1216,7 @@ static int dpaa2_eth_poll(struct napi_struct *napi, int budget)
                        break;
                if (fq->type == DPAA2_RX_FQ) {
                        rx_cleaned += store_cleaned;
+                       flowid = fq->flowid;
                } else {
                        txconf_cleaned += store_cleaned;
                        /* We have a single Tx conf FQ on this channel */
@@ -1239,6 +1259,8 @@ out:
 
        if (ch->xdp.res & XDP_REDIRECT)
                xdp_do_flush_map();
+       else if (rx_cleaned && ch->xdp.res & XDP_TX)
+               xdp_tx_flush(priv, ch, &priv->fq[flowid]);
 
        return work_done;
 }

diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.h b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.h
index 42f0a7a..0581fbf 100644 (file)
--- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.h
+++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.h
@@ -334,6 +334,7 @@ struct dpaa2_eth_fq {
        struct dpaa2_eth_fq_stats stats;
 
        struct dpaa2_eth_xdp_fds xdp_redirect_fds;
+       struct dpaa2_eth_xdp_fds xdp_tx_fds;
 };
 
 struct dpaa2_eth_ch_xdp {