OSDN Git Service

net/mlx5e: IPSec, Add Innova IPSec offload RX data path
authorIlan Tayari <ilant@mellanox.com>
Mon, 19 Jun 2017 11:04:36 +0000 (14:04 +0300)
committerSaeed Mahameed <saeedm@mellanox.com>
Tue, 27 Jun 2017 13:36:47 +0000 (16:36 +0300)
In RX data path, the hardware prepends a special metadata ethertype
which indicates that the packet underwent decryption, and the result of
the authentication check.

Communicate this to the stack in skb->sp.

Make wqe_size large enough to account for the injected metadata.

Support only Linked-list RQ type.

IPSec offload RX packets may have useful CHECKSUM_COMPLETE information,
which the stack may not be able to use yet.

Signed-off-by: Ilan Tayari <ilant@mellanox.com>
Signed-off-by: Yossi Kuperman <yossiku@mellanox.com>
Signed-off-by: Yevgeny Kliteynik <kliteyn@mellanox.com>
Signed-off-by: Boris Pismenny <borisp@mellanox.com>
Signed-off-by: Saeed Mahameed <saeedm@mellanox.com>
drivers/net/ethernet/mellanox/mlx5/core/Makefile
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c [new file with mode: 0644]
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h [new file with mode: 0644]
drivers/net/ethernet/mellanox/mlx5/core/en_main.c
drivers/net/ethernet/mellanox/mlx5/core/en_rx.c

index 7e81084..23cb8ba 100644 (file)
@@ -20,4 +20,4 @@ mlx5_core-$(CONFIG_MLX5_CORE_EN_DCB) +=  en_dcbnl.o
 
 mlx5_core-$(CONFIG_MLX5_CORE_IPOIB) += ipoib/ipoib.o ipoib/ethtool.o
 
-mlx5_core-$(CONFIG_MLX5_EN_IPSEC) += en_accel/ipsec.o
+mlx5_core-$(CONFIG_MLX5_EN_IPSEC) += en_accel/ipsec.o en_accel/ipsec_rxtx.o
index 06d9d6a..bb69660 100644 (file)
@@ -40,6 +40,7 @@
 #include "en.h"
 #include "accel/ipsec.h"
 #include "en_accel/ipsec.h"
+#include "en_accel/ipsec_rxtx.h"
 
 struct mlx5e_ipsec_sa_entry {
        struct hlist_node hlist; /* Item in SADB_RX hashtable */
@@ -49,6 +50,24 @@ struct mlx5e_ipsec_sa_entry {
        void *context;
 };
 
+struct xfrm_state *mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec *ipsec,
+                                             unsigned int handle)
+{
+       struct mlx5e_ipsec_sa_entry *sa_entry;
+       struct xfrm_state *ret = NULL;
+
+       rcu_read_lock();
+       hash_for_each_possible_rcu(ipsec->sadb_rx, sa_entry, hlist, handle)
+               if (sa_entry->handle == handle) {
+                       ret = sa_entry->x;
+                       xfrm_state_hold(ret);
+                       break;
+               }
+       rcu_read_unlock();
+
+       return ret;
+}
+
 static int mlx5e_ipsec_sadb_rx_add(struct mlx5e_ipsec_sa_entry *sa_entry)
 {
        struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
index b9423a2..4d745d3 100644 (file)
 #include <linux/idr.h>
 
 #define MLX5E_IPSEC_SADB_RX_BITS 10
+#define MLX5E_METADATA_ETHER_TYPE (0x8CE4)
+#define MLX5E_METADATA_ETHER_LEN 8
 
 struct mlx5e_priv;
 
+struct mlx5e_ipsec_sw_stats {
+       atomic64_t ipsec_rx_drop_sp_alloc;
+       atomic64_t ipsec_rx_drop_sadb_miss;
+       atomic64_t ipsec_rx_drop_syndrome;
+};
+
 struct mlx5e_ipsec {
        struct mlx5e_priv *en_priv;
        DECLARE_HASHTABLE(sadb_rx, MLX5E_IPSEC_SADB_RX_BITS);
        spinlock_t sadb_rx_lock; /* Protects sadb_rx and halloc */
        struct ida halloc;
+       struct mlx5e_ipsec_sw_stats sw_stats;
 };
 
 int mlx5e_ipsec_init(struct mlx5e_priv *priv);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c
new file mode 100644 (file)
index 0000000..56ab2e8
--- /dev/null
@@ -0,0 +1,135 @@
+/*
+ * Copyright (c) 2017 Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#include <crypto/aead.h>
+#include <net/xfrm.h>
+
+#include "en_accel/ipsec_rxtx.h"
+#include "en_accel/ipsec.h"
+#include "en.h"
+
+enum {
+       MLX5E_IPSEC_RX_SYNDROME_DECRYPTED = 0x11,
+       MLX5E_IPSEC_RX_SYNDROME_AUTH_FAILED = 0x12,
+};
+
+struct mlx5e_ipsec_rx_metadata {
+       unsigned char   reserved;
+       __be32          sa_handle;
+} __packed;
+
+struct mlx5e_ipsec_metadata {
+       unsigned char syndrome;
+       union {
+               unsigned char raw[5];
+               /* from FPGA to host, on successful decrypt */
+               struct mlx5e_ipsec_rx_metadata rx;
+       } __packed content;
+       /* packet type ID field */
+       __be16 ethertype;
+} __packed;
+
+static inline struct xfrm_state *
+mlx5e_ipsec_build_sp(struct net_device *netdev, struct sk_buff *skb,
+                    struct mlx5e_ipsec_metadata *mdata)
+{
+       struct mlx5e_priv *priv = netdev_priv(netdev);
+       struct xfrm_offload *xo;
+       struct xfrm_state *xs;
+       u32 sa_handle;
+
+       skb->sp = secpath_dup(skb->sp);
+       if (unlikely(!skb->sp)) {
+               atomic64_inc(&priv->ipsec->sw_stats.ipsec_rx_drop_sp_alloc);
+               return NULL;
+       }
+
+       sa_handle = be32_to_cpu(mdata->content.rx.sa_handle);
+       xs = mlx5e_ipsec_sadb_rx_lookup(priv->ipsec, sa_handle);
+       if (unlikely(!xs)) {
+               atomic64_inc(&priv->ipsec->sw_stats.ipsec_rx_drop_sadb_miss);
+               return NULL;
+       }
+
+       skb->sp->xvec[skb->sp->len++] = xs;
+       skb->sp->olen++;
+
+       xo = xfrm_offload(skb);
+       xo->flags = CRYPTO_DONE;
+       switch (mdata->syndrome) {
+       case MLX5E_IPSEC_RX_SYNDROME_DECRYPTED:
+               xo->status = CRYPTO_SUCCESS;
+               break;
+       case MLX5E_IPSEC_RX_SYNDROME_AUTH_FAILED:
+               xo->status = CRYPTO_TUNNEL_ESP_AUTH_FAILED;
+               break;
+       default:
+               atomic64_inc(&priv->ipsec->sw_stats.ipsec_rx_drop_syndrome);
+               return NULL;
+       }
+       return xs;
+}
+
+struct sk_buff *mlx5e_ipsec_handle_rx_skb(struct net_device *netdev,
+                                         struct sk_buff *skb)
+{
+       struct mlx5e_ipsec_metadata *mdata;
+       struct ethhdr *old_eth;
+       struct ethhdr *new_eth;
+       struct xfrm_state *xs;
+       __be16 *ethtype;
+
+       /* Detect inline metadata */
+       if (skb->len < ETH_HLEN + MLX5E_METADATA_ETHER_LEN)
+               return skb;
+       ethtype = (__be16 *)(skb->data + ETH_ALEN * 2);
+       if (*ethtype != cpu_to_be16(MLX5E_METADATA_ETHER_TYPE))
+               return skb;
+
+       /* Use the metadata */
+       mdata = (struct mlx5e_ipsec_metadata *)(skb->data + ETH_HLEN);
+       xs = mlx5e_ipsec_build_sp(netdev, skb, mdata);
+       if (unlikely(!xs)) {
+               kfree_skb(skb);
+               return NULL;
+       }
+
+       /* Remove the metadata from the buffer */
+       old_eth = (struct ethhdr *)skb->data;
+       new_eth = (struct ethhdr *)(skb->data + MLX5E_METADATA_ETHER_LEN);
+       memmove(new_eth, old_eth, 2 * ETH_ALEN);
+       /* Ethertype is already in its new place */
+       skb_pull_inline(skb, MLX5E_METADATA_ETHER_LEN);
+
+       return skb;
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h
new file mode 100644 (file)
index 0000000..4e2fab8
--- /dev/null
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2017 Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#ifndef __MLX5E_IPSEC_RXTX_H__
+#define __MLX5E_IPSEC_RXTX_H__
+
+#include <linux/skbuff.h>
+#include "en.h"
+
+struct sk_buff *mlx5e_ipsec_handle_rx_skb(struct net_device *netdev,
+                                         struct sk_buff *skb);
+void mlx5e_ipsec_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
+
+#endif /* __MLX5E_IPSEC_RXTX_H__ */
index aa5a7aa..170a937 100644 (file)
@@ -40,6 +40,8 @@
 #include "en_tc.h"
 #include "en_rep.h"
 #include "en_accel/ipsec.h"
+#include "en_accel/ipsec_rxtx.h"
+#include "accel/ipsec.h"
 #include "vxlan.h"
 
 struct mlx5e_rq_param {
@@ -116,7 +118,7 @@ void mlx5e_set_rq_type_params(struct mlx5_core_dev *mdev,
 static void mlx5e_set_rq_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
 {
        u8 rq_type = mlx5e_check_fragmented_striding_rq_cap(mdev) &&
-                   !params->xdp_prog ?
+                   !params->xdp_prog && !MLX5_IPSEC_DEV(mdev) ?
                    MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ :
                    MLX5_WQ_TYPE_LINKED_LIST;
        mlx5e_set_rq_type_params(mdev, params, rq_type);
@@ -593,6 +595,13 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
                rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe;
 
                rq->handle_rx_cqe = c->priv->profile->rx_handlers.handle_rx_cqe_mpwqe;
+#ifdef CONFIG_MLX5_EN_IPSEC
+               if (MLX5_IPSEC_DEV(mdev)) {
+                       err = -EINVAL;
+                       netdev_err(c->netdev, "MPWQE RQ with IPSec offload not supported\n");
+                       goto err_rq_wq_destroy;
+               }
+#endif
                if (!rq->handle_rx_cqe) {
                        err = -EINVAL;
                        netdev_err(c->netdev, "RX handler of MPWQE RQ is not set, err %d\n", err);
@@ -625,7 +634,12 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
                rq->alloc_wqe = mlx5e_alloc_rx_wqe;
                rq->dealloc_wqe = mlx5e_dealloc_rx_wqe;
 
-               rq->handle_rx_cqe = c->priv->profile->rx_handlers.handle_rx_cqe;
+#ifdef CONFIG_MLX5_EN_IPSEC
+               if (c->priv->ipsec)
+                       rq->handle_rx_cqe = mlx5e_ipsec_handle_rx_cqe;
+               else
+#endif
+                       rq->handle_rx_cqe = c->priv->profile->rx_handlers.handle_rx_cqe;
                if (!rq->handle_rx_cqe) {
                        kfree(rq->wqe.frag_info);
                        err = -EINVAL;
@@ -636,6 +650,10 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
                rq->buff.wqe_sz = params->lro_en  ?
                                params->lro_wqe_sz :
                                MLX5E_SW2HW_MTU(c->priv, c->netdev->mtu);
+#ifdef CONFIG_MLX5_EN_IPSEC
+               if (MLX5_IPSEC_DEV(mdev))
+                       rq->buff.wqe_sz += MLX5E_METADATA_ETHER_LEN;
+#endif
                rq->wqe.page_reuse = !params->xdp_prog && !params->lro_en;
                byte_count = rq->buff.wqe_sz;
 
index 574a962..325b2c8 100644 (file)
@@ -41,6 +41,7 @@
 #include "eswitch.h"
 #include "en_rep.h"
 #include "ipoib/ipoib.h"
+#include "en_accel/ipsec_rxtx.h"
 
 static inline bool mlx5e_rx_hw_stamp(struct mlx5e_tstamp *tstamp)
 {
@@ -1183,3 +1184,43 @@ wq_free_wqe:
 }
 
 #endif /* CONFIG_MLX5_CORE_IPOIB */
+
+#ifdef CONFIG_MLX5_EN_IPSEC
+
+void mlx5e_ipsec_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
+{
+       struct mlx5e_wqe_frag_info *wi;
+       struct mlx5e_rx_wqe *wqe;
+       __be16 wqe_counter_be;
+       struct sk_buff *skb;
+       u16 wqe_counter;
+       u32 cqe_bcnt;
+
+       wqe_counter_be = cqe->wqe_counter;
+       wqe_counter    = be16_to_cpu(wqe_counter_be);
+       wqe            = mlx5_wq_ll_get_wqe(&rq->wq, wqe_counter);
+       wi             = &rq->wqe.frag_info[wqe_counter];
+       cqe_bcnt       = be32_to_cpu(cqe->byte_cnt);
+
+       skb = skb_from_cqe(rq, cqe, wi, cqe_bcnt);
+       if (unlikely(!skb)) {
+               /* a DROP, save the page-reuse checks */
+               mlx5e_free_rx_wqe(rq, wi);
+               goto wq_ll_pop;
+       }
+       skb = mlx5e_ipsec_handle_rx_skb(rq->netdev, skb);
+       if (unlikely(!skb)) {
+               mlx5e_free_rx_wqe(rq, wi);
+               goto wq_ll_pop;
+       }
+
+       mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
+       napi_gro_receive(rq->cq.napi, skb);
+
+       mlx5e_free_rx_wqe_reuse(rq, wi);
+wq_ll_pop:
+       mlx5_wq_ll_pop(&rq->wq, wqe_counter_be,
+                      &wqe->next.next_wqe_index);
+}
+
+#endif /* CONFIG_MLX5_EN_IPSEC */