net/mlx5e: kTLS, Improve connection rate by using fast update encryption key

As the fast DEK update is fully implemented, use it for kTLS to get
better performance.
TIS pool was already supported to recycle the TISes. With this series
and TIS pool, TLS CPS is improved by 9x higher, from 11k/s to 101k/s.

Signed-off-by: Jianbo Liu <jianbol@nvidia.com>
Reviewed-by: Tariq Toukan <tariqt@nvidia.com>
Signed-off-by: Saeed Mahameed <saeedm@nvidia.com>

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c
index f80d6fce28d2d5c95a39258d3703dd51b402014f..cf704f106b7c295ce236fa63916f1644f49f7ec6 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c
@@ -9,9 +9,8 @@
 #include "en_accel/ktls_utils.h"
 #include "en_accel/fs_tcp.h"
 
-int mlx5_ktls_create_key(struct mlx5_core_dev *mdev,
-                        struct tls_crypto_info *crypto_info,
-                        u32 *p_key_id)
+struct mlx5_crypto_dek *mlx5_ktls_create_key(struct mlx5_crypto_dek_pool *dek_pool,
+                                            struct tls_crypto_info *crypto_info)
 {
        const void *key;
        u32 sz_bytes;
@@ -34,17 +33,16 @@ int mlx5_ktls_create_key(struct mlx5_core_dev *mdev,
                break;
        }
        default:
-               return -EINVAL;
+               return ERR_PTR(-EINVAL);
        }
 
-       return mlx5_create_encryption_key(mdev, key, sz_bytes,
-                                         MLX5_ACCEL_OBJ_TLS_KEY,
-                                         p_key_id);
+       return mlx5_crypto_dek_create(dek_pool, key, sz_bytes);
 }
 
-void mlx5_ktls_destroy_key(struct mlx5_core_dev *mdev, u32 key_id)
+void mlx5_ktls_destroy_key(struct mlx5_crypto_dek_pool *dek_pool,
+                          struct mlx5_crypto_dek *dek)
 {
-       mlx5_destroy_encryption_key(mdev, key_id);
+       mlx5_crypto_dek_destroy(dek_pool, dek);
 }
 
 static int mlx5e_ktls_add(struct net_device *netdev, struct sock *sk,
@@ -190,6 +188,7 @@ static void mlx5e_tls_debugfs_init(struct mlx5e_tls *tls,
 
 int mlx5e_ktls_init(struct mlx5e_priv *priv)
 {
+       struct mlx5_crypto_dek_pool *dek_pool;
        struct mlx5e_tls *tls;
 
        if (!mlx5e_is_ktls_device(priv->mdev))
@@ -198,9 +197,15 @@ int mlx5e_ktls_init(struct mlx5e_priv *priv)
        tls = kzalloc(sizeof(*tls), GFP_KERNEL);
        if (!tls)
                return -ENOMEM;
+       tls->mdev = priv->mdev;
 
+       dek_pool = mlx5_crypto_dek_pool_create(priv->mdev, MLX5_ACCEL_OBJ_TLS_KEY);
+       if (IS_ERR(dek_pool)) {
+               kfree(tls);
+               return PTR_ERR(dek_pool);
+       }
+       tls->dek_pool = dek_pool;
        priv->tls = tls;
-       priv->tls->mdev = priv->mdev;
 
        mlx5e_tls_debugfs_init(tls, priv->dfs_root);
 
@@ -217,6 +222,7 @@ void mlx5e_ktls_cleanup(struct mlx5e_priv *priv)
        debugfs_remove_recursive(tls->debugfs.dfs);
        tls->debugfs.dfs = NULL;
 
+       mlx5_crypto_dek_pool_destroy(tls->dek_pool);
        kfree(priv->tls);
        priv->tls = NULL;
 }

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h
index fccf995ee16d76752e21337d510300160bd0327c..f11075e67658621b9fc52da27c4239ae7d7cd716 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h
@@ -10,10 +10,12 @@
 #include "en.h"
 
 #ifdef CONFIG_MLX5_EN_TLS
-int mlx5_ktls_create_key(struct mlx5_core_dev *mdev,
-                        struct tls_crypto_info *crypto_info,
-                        u32 *p_key_id);
-void mlx5_ktls_destroy_key(struct mlx5_core_dev *mdev, u32 key_id);
+#include "lib/crypto.h"
+
+struct mlx5_crypto_dek *mlx5_ktls_create_key(struct mlx5_crypto_dek_pool *dek_pool,
+                                            struct tls_crypto_info *crypto_info);
+void mlx5_ktls_destroy_key(struct mlx5_crypto_dek_pool *dek_pool,
+                          struct mlx5_crypto_dek *dek);
 
 static inline bool mlx5e_is_ktls_device(struct mlx5_core_dev *mdev)
 {
@@ -83,6 +85,7 @@ struct mlx5e_tls {
        struct mlx5e_tls_sw_stats sw_stats;
        struct workqueue_struct *rx_wq;
        struct mlx5e_tls_tx_pool *tx_pool;
+       struct mlx5_crypto_dek_pool *dek_pool;
        struct mlx5e_tls_debugfs debugfs;
 };
 

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c
index 3e54834747ce6d697c28df34c7a9de66556c7550..4be770443b0cd2a22df63c07c34ebbec4847678a 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c
@@ -50,7 +50,7 @@ struct mlx5e_ktls_offload_context_rx {
        struct mlx5e_tls_sw_stats *sw_stats;
        struct completion add_ctx;
        struct mlx5e_tir tir;
-       u32 key_id;
+       struct mlx5_crypto_dek *dek;
        u32 rxq;
        DECLARE_BITMAP(flags, MLX5E_NUM_PRIV_RX_FLAGS);
 
@@ -148,7 +148,8 @@ post_static_params(struct mlx5e_icosq *sq,
        wqe = MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi);
        mlx5e_ktls_build_static_params(wqe, sq->pc, sq->sqn, &priv_rx->crypto_info,
                                       mlx5e_tir_get_tirn(&priv_rx->tir),
-                                      priv_rx->key_id, priv_rx->resync.seq, false,
+                                      mlx5_crypto_dek_get_id(priv_rx->dek),
+                                      priv_rx->resync.seq, false,
                                       TLS_OFFLOAD_CTX_DIR_RX);
        wi = (struct mlx5e_icosq_wqe_info) {
                .wqe_type = MLX5E_ICOSQ_WQE_UMR_TLS,
@@ -610,20 +611,22 @@ int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk,
        struct mlx5e_ktls_offload_context_rx *priv_rx;
        struct mlx5e_ktls_rx_resync_ctx *resync;
        struct tls_context *tls_ctx;
-       struct mlx5_core_dev *mdev;
+       struct mlx5_crypto_dek *dek;
        struct mlx5e_priv *priv;
        int rxq, err;
 
        tls_ctx = tls_get_ctx(sk);
        priv = netdev_priv(netdev);
-       mdev = priv->mdev;
        priv_rx = kzalloc(sizeof(*priv_rx), GFP_KERNEL);
        if (unlikely(!priv_rx))
                return -ENOMEM;
 
-       err = mlx5_ktls_create_key(mdev, crypto_info, &priv_rx->key_id);
-       if (err)
+       dek = mlx5_ktls_create_key(priv->tls->dek_pool, crypto_info);
+       if (IS_ERR(dek)) {
+               err = PTR_ERR(dek);
                goto err_create_key;
+       }
+       priv_rx->dek = dek;
 
        INIT_LIST_HEAD(&priv_rx->list);
        spin_lock_init(&priv_rx->lock);
@@ -673,7 +676,7 @@ int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk,
 err_post_wqes:
        mlx5e_tir_destroy(&priv_rx->tir);
 err_create_tir:
-       mlx5_ktls_destroy_key(mdev, priv_rx->key_id);
+       mlx5_ktls_destroy_key(priv->tls->dek_pool, priv_rx->dek);
 err_create_key:
        kfree(priv_rx);
        return err;
@@ -683,11 +686,9 @@ void mlx5e_ktls_del_rx(struct net_device *netdev, struct tls_context *tls_ctx)
 {
        struct mlx5e_ktls_offload_context_rx *priv_rx;
        struct mlx5e_ktls_rx_resync_ctx *resync;
-       struct mlx5_core_dev *mdev;
        struct mlx5e_priv *priv;
 
        priv = netdev_priv(netdev);
-       mdev = priv->mdev;
 
        priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_ctx);
        set_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags);
@@ -707,7 +708,7 @@ void mlx5e_ktls_del_rx(struct net_device *netdev, struct tls_context *tls_ctx)
                mlx5e_accel_fs_del_sk(priv_rx->rule.rule);
 
        mlx5e_tir_destroy(&priv_rx->tir);
-       mlx5_ktls_destroy_key(mdev, priv_rx->key_id);
+       mlx5_ktls_destroy_key(priv->tls->dek_pool, priv_rx->dek);
        /* priv_rx should normally be freed here, but if there is an outstanding
         * GET_PSV, deallocation will be delayed until the CQE for GET_PSV is
         * processed.

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c
index 6db27062b765faaa19fe65f8b9e021d67283e903..e80b43b7aac942bea1287c35de33c9bd09f336a5 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c
@@ -98,7 +98,7 @@ struct mlx5e_ktls_offload_context_tx {
        struct tls_offload_context_tx *tx_ctx;
        struct mlx5_core_dev *mdev;
        struct mlx5e_tls_sw_stats *sw_stats;
-       u32 key_id;
+       struct mlx5_crypto_dek *dek;
        u8 create_err : 1;
 };
 
@@ -457,6 +457,7 @@ int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk,
        struct mlx5e_ktls_offload_context_tx *priv_tx;
        struct mlx5e_tls_tx_pool *pool;
        struct tls_context *tls_ctx;
+       struct mlx5_crypto_dek *dek;
        struct mlx5e_priv *priv;
        int err;
 
@@ -468,9 +469,12 @@ int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk,
        if (IS_ERR(priv_tx))
                return PTR_ERR(priv_tx);
 
-       err = mlx5_ktls_create_key(pool->mdev, crypto_info, &priv_tx->key_id);
-       if (err)
+       dek = mlx5_ktls_create_key(priv->tls->dek_pool, crypto_info);
+       if (IS_ERR(dek)) {
+               err = PTR_ERR(dek);
                goto err_create_key;
+       }
+       priv_tx->dek = dek;
 
        priv_tx->expected_seq = start_offload_tcp_sn;
        switch (crypto_info->cipher_type) {
@@ -512,7 +516,7 @@ void mlx5e_ktls_del_tx(struct net_device *netdev, struct tls_context *tls_ctx)
        pool = priv->tls->tx_pool;
 
        atomic64_inc(&priv_tx->sw_stats->tx_tls_del);
-       mlx5_ktls_destroy_key(priv_tx->mdev, priv_tx->key_id);
+       mlx5_ktls_destroy_key(priv->tls->dek_pool, priv_tx->dek);
        pool_push(pool, priv_tx);
 }
 
@@ -551,8 +555,9 @@ post_static_params(struct mlx5e_txqsq *sq,
        pi = mlx5e_txqsq_get_next_pi(sq, num_wqebbs);
        wqe = MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi);
        mlx5e_ktls_build_static_params(wqe, sq->pc, sq->sqn, &priv_tx->crypto_info,
-                                      priv_tx->tisn, priv_tx->key_id, 0, fence,
-                                      TLS_OFFLOAD_CTX_DIR_TX);
+                                      priv_tx->tisn,
+                                      mlx5_crypto_dek_get_id(priv_tx->dek),
+                                      0, fence, TLS_OFFLOAD_CTX_DIR_TX);
        tx_fill_wi(sq, pi, num_wqebbs, 0, NULL);
        sq->pc += num_wqebbs;
 }