ice: xsk: Do not convert to buff to frame for XDP_TX

Let us store pointer to xdp_buff that came from xsk_buff_pool on tx_buf
so that it will be possible to recycle it via xsk_buff_free() on Tx
cleaning side. This way it is not necessary to do expensive copy to
another xdp_buff backed by a newly allocated page.

Signed-off-by: Maciej Fijalkowski <maciej.fijalkowski@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Reviewed-by: Alexander Lobakin <alexandr.lobakin@intel.com>
Link: https://lore.kernel.org/bpf/20230131204506.219292-14-maciej.fijalkowski@intel.com

diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.c b/drivers/net/ethernet/intel/ice/ice_txrx.c
index 3cc63e6..466113c 100644 (file)
--- a/drivers/net/ethernet/intel/ice/ice_txrx.c
+++ b/drivers/net/ethernet/intel/ice/ice_txrx.c
@@ -113,12 +113,16 @@ static void
 ice_unmap_and_free_tx_buf(struct ice_tx_ring *ring, struct ice_tx_buf *tx_buf)
 {
        if (tx_buf->skb) {
-               if (tx_buf->tx_flags & ICE_TX_FLAGS_DUMMY_PKT)
+               if (tx_buf->tx_flags & ICE_TX_FLAGS_DUMMY_PKT) {
                        devm_kfree(ring->dev, tx_buf->raw_buf);
-               else if (ice_ring_is_xdp(ring))
-                       page_frag_free(tx_buf->raw_buf);
-               else
+               } else if (ice_ring_is_xdp(ring)) {
+                       if (ring->xsk_pool)
+                               xsk_buff_free(tx_buf->xdp);
+                       else
+                               page_frag_free(tx_buf->raw_buf);
+               } else {
                        dev_kfree_skb_any(tx_buf->skb);
+               }
                if (dma_unmap_len(tx_buf, len))
                        dma_unmap_single(ring->dev,
                                         dma_unmap_addr(tx_buf, dma),

diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.h b/drivers/net/ethernet/intel/ice/ice_txrx.h
index 7903bb6..efa3d37 100644 (file)
--- a/drivers/net/ethernet/intel/ice/ice_txrx.h
+++ b/drivers/net/ethernet/intel/ice/ice_txrx.h
@@ -157,6 +157,7 @@ struct ice_tx_buf {
        union {
                struct sk_buff *skb;
                void *raw_buf; /* used for XDP */
+               struct xdp_buff *xdp; /* used for XDP_TX ZC */
        };
        unsigned int bytecount;
        union {

diff --git a/drivers/net/ethernet/intel/ice/ice_txrx_lib.c b/drivers/net/ethernet/intel/ice/ice_txrx_lib.c
index 3dcc40b..9bbed3f 100644 (file)
--- a/drivers/net/ethernet/intel/ice/ice_txrx_lib.c
+++ b/drivers/net/ethernet/intel/ice/ice_txrx_lib.c
@@ -275,7 +275,10 @@ static u32 ice_clean_xdp_irq(struct ice_tx_ring *xdp_ring)
                /* count head + frags */
                ready_frames -= frags + 1;
 
-               ice_clean_xdp_tx_buf(xdp_ring, tx_buf);
+               if (xdp_ring->xsk_pool)
+                       xsk_buff_free(tx_buf->xdp);
+               else
+                       ice_clean_xdp_tx_buf(xdp_ring, tx_buf);
                ntc++;
                if (ntc == cnt)
                        ntc = 0;
@@ -408,23 +411,6 @@ int ice_xmit_xdp_ring(struct xdp_frame *xdpf, struct ice_tx_ring *xdp_ring)
 }
 
 /**
- * ice_xmit_xdp_buff - convert an XDP buffer to an XDP frame and send it
- * @xdp: XDP buffer
- * @xdp_ring: XDP Tx ring
- *
- * Returns negative on failure, 0 on success.
- */
-int ice_xmit_xdp_buff(struct xdp_buff *xdp, struct ice_tx_ring *xdp_ring)
-{
-       struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);
-
-       if (unlikely(!xdpf))
-               return ICE_XDP_CONSUMED;
-
-       return ice_xmit_xdp_ring(xdpf, xdp_ring);
-}
-
-/**
  * ice_finalize_xdp_rx - Bump XDP Tx tail and/or flush redirect map
  * @xdp_ring: XDP ring
  * @xdp_res: Result of the receive batch

diff --git a/drivers/net/ethernet/intel/ice/ice_xsk.c b/drivers/net/ethernet/intel/ice/ice_xsk.c
index b1b419c..a25a68c 100644 (file)
--- a/drivers/net/ethernet/intel/ice/ice_xsk.c
+++ b/drivers/net/ethernet/intel/ice/ice_xsk.c
@@ -598,6 +598,107 @@ ice_construct_skb_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp)
 }
 
 /**
+ * ice_clean_xdp_irq_zc - AF_XDP ZC specific Tx cleaning routine
+ * @xdp_ring: XDP Tx ring
+ */
+static void ice_clean_xdp_irq_zc(struct ice_tx_ring *xdp_ring)
+{
+       u16 ntc = xdp_ring->next_to_clean;
+       struct ice_tx_desc *tx_desc;
+       u16 cnt = xdp_ring->count;
+       struct ice_tx_buf *tx_buf;
+       u16 xsk_frames = 0;
+       u16 last_rs;
+       int i;
+
+       last_rs = xdp_ring->next_to_use ? xdp_ring->next_to_use - 1 : cnt - 1;
+       tx_desc = ICE_TX_DESC(xdp_ring, last_rs);
+       if (tx_desc->cmd_type_offset_bsz &
+           cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE)) {
+               if (last_rs >= ntc)
+                       xsk_frames = last_rs - ntc + 1;
+               else
+                       xsk_frames = last_rs + cnt - ntc + 1;
+       }
+
+       if (!xsk_frames)
+               return;
+
+       if (likely(!xdp_ring->xdp_tx_active))
+               goto skip;
+
+       ntc = xdp_ring->next_to_clean;
+       for (i = 0; i < xsk_frames; i++) {
+               tx_buf = &xdp_ring->tx_buf[ntc];
+
+               if (tx_buf->xdp) {
+                       xsk_buff_free(tx_buf->xdp);
+                       xdp_ring->xdp_tx_active--;
+               } else {
+                       xsk_frames++;
+               }
+
+               ntc++;
+               if (ntc == cnt)
+                       ntc = 0;
+       }
+skip:
+       tx_desc->cmd_type_offset_bsz = 0;
+       xdp_ring->next_to_clean += xsk_frames;
+       if (xdp_ring->next_to_clean >= cnt)
+               xdp_ring->next_to_clean -= cnt;
+       if (xsk_frames)
+               xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames);
+}
+
+/**
+ * ice_xmit_xdp_tx_zc - AF_XDP ZC handler for XDP_TX
+ * @xdp: XDP buffer to xmit
+ * @xdp_ring: XDP ring to produce descriptor onto
+ *
+ * note that this function works directly on xdp_buff, no need to convert
+ * it to xdp_frame. xdp_buff pointer is stored to ice_tx_buf so that cleaning
+ * side will be able to xsk_buff_free() it.
+ *
+ * Returns ICE_XDP_TX for successfully produced desc, ICE_XDP_CONSUMED if there
+ * was not enough space on XDP ring
+ */
+static int ice_xmit_xdp_tx_zc(struct xdp_buff *xdp,
+                             struct ice_tx_ring *xdp_ring)
+{
+       u32 size = xdp->data_end - xdp->data;
+       u32 ntu = xdp_ring->next_to_use;
+       struct ice_tx_desc *tx_desc;
+       struct ice_tx_buf *tx_buf;
+       dma_addr_t dma;
+
+       if (ICE_DESC_UNUSED(xdp_ring) < ICE_RING_QUARTER(xdp_ring)) {
+               ice_clean_xdp_irq_zc(xdp_ring);
+               if (!ICE_DESC_UNUSED(xdp_ring)) {
+                       xdp_ring->ring_stats->tx_stats.tx_busy++;
+                       return ICE_XDP_CONSUMED;
+               }
+       }
+
+       dma = xsk_buff_xdp_get_dma(xdp);
+       xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_pool, dma, size);
+
+       tx_buf = &xdp_ring->tx_buf[ntu];
+       tx_buf->xdp = xdp;
+       tx_desc = ICE_TX_DESC(xdp_ring, ntu);
+       tx_desc->buf_addr = cpu_to_le64(dma);
+       tx_desc->cmd_type_offset_bsz = ice_build_ctob(ICE_TX_DESC_CMD_EOP,
+                                                     0, size, 0);
+       xdp_ring->xdp_tx_active++;
+
+       if (++ntu == xdp_ring->count)
+               ntu = 0;
+       xdp_ring->next_to_use = ntu;
+
+       return ICE_XDP_TX;
+}
+
+/**
  * ice_run_xdp_zc - Executes an XDP program in zero-copy path
  * @rx_ring: Rx ring
  * @xdp: xdp_buff used as input to the XDP program
@@ -630,7 +731,7 @@ ice_run_xdp_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp,
        case XDP_PASS:
                break;
        case XDP_TX:
-               result = ice_xmit_xdp_buff(xdp, xdp_ring);
+               result = ice_xmit_xdp_tx_zc(xdp, xdp_ring);
                if (result == ICE_XDP_CONSUMED)
                        goto out_failure;
                break;
@@ -776,75 +877,6 @@ construct_skb:
 }
 
 /**
- * ice_clean_xdp_tx_buf - Free and unmap XDP Tx buffer
- * @xdp_ring: XDP Tx ring
- * @tx_buf: Tx buffer to clean
- */
-static void
-ice_clean_xdp_tx_buf(struct ice_tx_ring *xdp_ring, struct ice_tx_buf *tx_buf)
-{
-       page_frag_free(tx_buf->raw_buf);
-       xdp_ring->xdp_tx_active--;
-       dma_unmap_single(xdp_ring->dev, dma_unmap_addr(tx_buf, dma),
-                        dma_unmap_len(tx_buf, len), DMA_TO_DEVICE);
-       dma_unmap_len_set(tx_buf, len, 0);
-}
-
-/**
- * ice_clean_xdp_irq_zc - produce AF_XDP descriptors to CQ
- * @xdp_ring: XDP Tx ring
- */
-static void ice_clean_xdp_irq_zc(struct ice_tx_ring *xdp_ring)
-{
-       u16 ntc = xdp_ring->next_to_clean;
-       struct ice_tx_desc *tx_desc;
-       u16 cnt = xdp_ring->count;
-       struct ice_tx_buf *tx_buf;
-       u16 xsk_frames = 0;
-       u16 last_rs;
-       int i;
-
-       last_rs = xdp_ring->next_to_use ? xdp_ring->next_to_use - 1 : cnt - 1;
-       tx_desc = ICE_TX_DESC(xdp_ring, last_rs);
-       if ((tx_desc->cmd_type_offset_bsz &
-           cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE))) {
-               if (last_rs >= ntc)
-                       xsk_frames = last_rs - ntc + 1;
-               else
-                       xsk_frames = last_rs + cnt - ntc + 1;
-       }
-
-       if (!xsk_frames)
-               return;
-
-       if (likely(!xdp_ring->xdp_tx_active))
-               goto skip;
-
-       ntc = xdp_ring->next_to_clean;
-       for (i = 0; i < xsk_frames; i++) {
-               tx_buf = &xdp_ring->tx_buf[ntc];
-
-               if (tx_buf->raw_buf) {
-                       ice_clean_xdp_tx_buf(xdp_ring, tx_buf);
-                       tx_buf->raw_buf = NULL;
-               } else {
-                       xsk_frames++;
-               }
-
-               ntc++;
-               if (ntc >= xdp_ring->count)
-                       ntc = 0;
-       }
-skip:
-       tx_desc->cmd_type_offset_bsz = 0;
-       xdp_ring->next_to_clean += xsk_frames;
-       if (xdp_ring->next_to_clean >= cnt)
-               xdp_ring->next_to_clean -= cnt;
-       if (xsk_frames)
-               xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames);
-}
-
-/**
  * ice_xmit_pkt - produce a single HW Tx descriptor out of AF_XDP descriptor
  * @xdp_ring: XDP ring to produce the HW Tx descriptor on
  * @desc: AF_XDP descriptor to pull the DMA address and length from
@@ -1051,8 +1083,8 @@ void ice_xsk_clean_xdp_ring(struct ice_tx_ring *xdp_ring)
        while (ntc != ntu) {
                struct ice_tx_buf *tx_buf = &xdp_ring->tx_buf[ntc];
 
-               if (tx_buf->raw_buf)
-                       ice_clean_xdp_tx_buf(xdp_ring, tx_buf);
+               if (tx_buf->xdp)
+                       xsk_buff_free(tx_buf->xdp);
                else
                        xsk_frames++;