net/mlx5e: IPsec: Add Connect-X IPsec Rx data path offload

On receive flow inspect received packets for IPsec offload indication
using the cqe, for IPsec offloaded packets propagate offload status
and stack handle to stack for further processing.

Supported statuses:
- Offload ok.
- Authentication failure.
- Bad trailer indication.

Connect-X IPsec does not use mlx5e_ipsec_handle_rx_cqe.

For RX only offload, we see the BW gain. Below is the iperf3
performance report on two server of 24 cores Intel(R) Xeon(R)
CPU E5-2620 v3 @ 2.40GHz with ConnectX6-DX.
We use one thread per IPsec tunnel.

---------------------------------------------------------------------
Mode          |  Num tunnel | BW     | Send CPU util | Recv CPU util
              |             | (Gbps) | (Average %)   | (Average %)
---------------------------------------------------------------------
Cryto offload | 1           | 4.6    | 4.2           | 14.5
---------------------------------------------------------------------
Cryto offload | 24          | 38     | 73            | 63
---------------------------------------------------------------------
Non-offload   | 1           | 4      | 4             | 13
---------------------------------------------------------------------
Non-offload   | 24          | 23     | 52            | 67

Signed-off-by: Raed Salem <raeds@mellanox.com>
Reviewed-by: Boris Pismenny <borisp@mellanox.com>
Reviewed-by: Tariq Toukan <tariqt@mellanox.com>
Signed-off-by: Saeed Mahameed <saeedm@mellanox.com>

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c
index 824b87ac8f9eece197550f24de0206085ca20a90..93a8d68815adeefb2c66bee05c1636ff3d7ff643 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c
@@ -360,6 +360,62 @@ struct sk_buff *mlx5e_ipsec_handle_rx_skb(struct net_device *netdev,
        return skb;
 }
 
+enum {
+       MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_DECRYPTED,
+       MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_AUTH_FAILED,
+       MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_BAD_TRAILER,
+};
+
+void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev,
+                                      struct sk_buff *skb,
+                                      struct mlx5_cqe64 *cqe)
+{
+       u32 ipsec_meta_data = be32_to_cpu(cqe->ft_metadata);
+       u8 ipsec_syndrome = ipsec_meta_data & 0xFF;
+       struct mlx5e_priv *priv;
+       struct xfrm_offload *xo;
+       struct xfrm_state *xs;
+       struct sec_path *sp;
+       u32  sa_handle;
+
+       sa_handle = MLX5_IPSEC_METADATA_HANDLE(ipsec_meta_data);
+       priv = netdev_priv(netdev);
+       sp = secpath_set(skb);
+       if (unlikely(!sp)) {
+               atomic64_inc(&priv->ipsec->sw_stats.ipsec_rx_drop_sp_alloc);
+               return;
+       }
+
+       xs = mlx5e_ipsec_sadb_rx_lookup(priv->ipsec, sa_handle);
+       if (unlikely(!xs)) {
+               atomic64_inc(&priv->ipsec->sw_stats.ipsec_rx_drop_sadb_miss);
+               return;
+       }
+
+       sp = skb_sec_path(skb);
+       sp->xvec[sp->len++] = xs;
+       sp->olen++;
+
+       xo = xfrm_offload(skb);
+       xo->flags = CRYPTO_DONE;
+
+       switch (ipsec_syndrome & MLX5_IPSEC_METADATA_SYNDROM_MASK) {
+       case MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_DECRYPTED:
+               xo->status = CRYPTO_SUCCESS;
+               if (WARN_ON_ONCE(priv->ipsec->no_trailer))
+                       xo->flags |= XFRM_ESP_NO_TRAILER;
+               break;
+       case MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_AUTH_FAILED:
+               xo->status = CRYPTO_TUNNEL_ESP_AUTH_FAILED;
+               break;
+       case MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_BAD_TRAILER:
+               xo->status = CRYPTO_INVALID_PACKET_SYNTAX;
+               break;
+       default:
+               atomic64_inc(&priv->ipsec->sw_stats.ipsec_rx_drop_syndrome);
+       }
+}
+
 bool mlx5e_ipsec_feature_check(struct sk_buff *skb, struct net_device *netdev,
                               netdev_features_t features)
 {

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h
index ba02643586a540593109951640daeea4ae037777..2a47673da5a4e081fd092bdbc58b4c7cacf585d5 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h
@@ -34,13 +34,17 @@
 #ifndef __MLX5E_IPSEC_RXTX_H__
 #define __MLX5E_IPSEC_RXTX_H__
 
-#ifdef CONFIG_MLX5_EN_IPSEC
-
 #include <linux/skbuff.h>
 #include <net/xfrm.h>
 #include "en.h"
 #include "en/txrx.h"
 
+#define MLX5_IPSEC_METADATA_MARKER_MASK      (0x80)
+#define MLX5_IPSEC_METADATA_SYNDROM_MASK     (0x7F)
+#define MLX5_IPSEC_METADATA_HANDLE(metadata) (((metadata) >> 8) & 0xFF)
+
+#ifdef CONFIG_MLX5_EN_IPSEC
+
 struct sk_buff *mlx5e_ipsec_handle_rx_skb(struct net_device *netdev,
                                          struct sk_buff *skb, u32 *cqe_bcnt);
 void mlx5e_ipsec_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
@@ -55,7 +59,21 @@ void mlx5e_ipsec_set_iv(struct sk_buff *skb, struct xfrm_state *x,
 bool mlx5e_ipsec_handle_tx_skb(struct mlx5e_priv *priv,
                               struct mlx5_wqe_eth_seg *eseg,
                               struct sk_buff *skb);
+void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev,
+                                      struct sk_buff *skb,
+                                      struct mlx5_cqe64 *cqe);
+static inline bool mlx5_ipsec_is_rx_flow(struct mlx5_cqe64 *cqe)
+{
+       return !!(MLX5_IPSEC_METADATA_MARKER_MASK & be32_to_cpu(cqe->ft_metadata));
+}
+#else
+static inline
+void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev,
+                                      struct sk_buff *skb,
+                                      struct mlx5_cqe64 *cqe)
+{}
 
+static inline bool mlx5_ipsec_is_rx_flow(struct mlx5_cqe64 *cqe) { return false; }
 #endif /* CONFIG_MLX5_EN_IPSEC */
 
 #endif /* __MLX5E_IPSEC_RXTX_H__ */

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index 4e5d83f6334a489c2fad499fde3bc6568cb009b0..88ea1908cb14ab97fa4fcbedc6e3281ef67a95d4 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -65,6 +65,7 @@
 #include "en/hv_vhca_stats.h"
 #include "en/devlink.h"
 #include "lib/mlx5.h"
+#include "fpga/ipsec.h"
 
 bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev)
 {
@@ -496,7 +497,8 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
                rq->dealloc_wqe = mlx5e_dealloc_rx_wqe;
 
 #ifdef CONFIG_MLX5_EN_IPSEC
-               if (c->priv->ipsec)
+               if ((mlx5_fpga_ipsec_device_caps(mdev) & MLX5_ACCEL_IPSEC_CAP_DEVICE) &&
+                   c->priv->ipsec)
                        rq->handle_rx_cqe = mlx5e_ipsec_handle_rx_cqe;
                else
 #endif

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
index 350f9c54e508f3d88ff9e8eaf249d7ac9aab3e6b..8b24e44f860a8e79b062f5f635f2250f9e773204 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -973,9 +973,14 @@ static inline void mlx5e_handle_csum(struct net_device *netdev,
                goto csum_unnecessary;
 
        if (likely(is_last_ethertype_ip(skb, &network_depth, &proto))) {
-               if (unlikely(get_ip_proto(skb, network_depth, proto) == IPPROTO_SCTP))
+               u8 ipproto = get_ip_proto(skb, network_depth, proto);
+
+               if (unlikely(ipproto == IPPROTO_SCTP))
                        goto csum_unnecessary;
 
+               if (unlikely(mlx5_ipsec_is_rx_flow(cqe)))
+                       goto csum_none;
+
                stats->csum_complete++;
                skb->ip_summed = CHECKSUM_COMPLETE;
                skb->csum = csum_unfold((__force __sum16)cqe->check_sum);
@@ -1021,6 +1026,9 @@ static inline void mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe,
 
        mlx5e_tls_handle_rx_skb(rq, skb, cqe, &cqe_bcnt);
 
+       if (unlikely(mlx5_ipsec_is_rx_flow(cqe)))
+               mlx5e_ipsec_offload_handle_rx_skb(netdev, skb, cqe);
+
        if (lro_num_seg > 1) {
                mlx5e_lro_update_hdr(skb, cqe, cqe_bcnt);
                skb_shinfo(skb)->gso_size = DIV_ROUND_UP(cqe_bcnt, lro_num_seg);