net/mlx5e: Don't prefill WQEs in XDP SQ in the multi buffer mode

When MPWQE is disabled, mlx5e_open_xdpsq() prefills the common fields of
WQEs in the XDP SQ to save time when sending packets.
mlx5e_xmit_xdp_frame() runs on the prefilled fields, however, sending
multi buffer XDP frames would require changing some of these fields on a
per-packet basis. Besides that, mlx5e_xmit_xdp_frame() will be used as a
fallback to send multi buffer XDP frames when MPWQE is enabled (MPWQE
can only handle linear packets).

In order to prepare for XDP multi buffer support, this commit introduces
a mode for mlx5e_xmit_xdp_frame() that fills all the fields itself.

Signed-off-by: Maxim Mikityanskiy <maximmi@nvidia.com>
Signed-off-by: Saeed Mahameed <saeedm@nvidia.com>

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h
index d084f93..cf935a7 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
@@ -405,6 +405,7 @@ enum {
        MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE,
        MLX5E_SQ_STATE_PENDING_XSK_TX,
        MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC,
+       MLX5E_SQ_STATE_XDP_MULTIBUF,
 };
 
 struct mlx5e_tx_mpwqe {

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
index 9646867..08fd137 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
@@ -843,6 +843,7 @@ static void mlx5e_build_async_icosq_param(struct mlx5_core_dev *mdev,
 
 void mlx5e_build_xdpsq_param(struct mlx5_core_dev *mdev,
                             struct mlx5e_params *params,
+                            struct mlx5e_xsk_param *xsk,
                             struct mlx5e_sq_param *param)
 {
        void *sqc = param->sqc;
@@ -851,6 +852,7 @@ void mlx5e_build_xdpsq_param(struct mlx5_core_dev *mdev,
        mlx5e_build_sq_param_common(mdev, param);
        MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size);
        param->is_mpw = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE);
+       param->is_xdp_mb = !mlx5e_rx_is_linear_skb(params, xsk);
        mlx5e_build_tx_cq_param(mdev, params, &param->cqp);
 }
 
@@ -870,7 +872,7 @@ int mlx5e_build_channel_param(struct mlx5_core_dev *mdev,
        async_icosq_log_wq_sz = mlx5e_build_async_icosq_log_wq_sz(mdev);
 
        mlx5e_build_sq_param(mdev, params, &cparam->txq_sq);
-       mlx5e_build_xdpsq_param(mdev, params, &cparam->xdp_sq);
+       mlx5e_build_xdpsq_param(mdev, params, NULL, &cparam->xdp_sq);
        mlx5e_build_icosq_param(mdev, icosq_log_wq_sz, &cparam->icosq);
        mlx5e_build_async_icosq_param(mdev, async_icosq_log_wq_sz, &cparam->async_icosq);
 

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h
index 47a3681..f5c46e7 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h
@@ -31,6 +31,7 @@ struct mlx5e_sq_param {
        struct mlx5_wq_param       wq;
        bool                       is_mpw;
        bool                       is_tls;
+       bool                       is_xdp_mb;
        u16                        stop_room;
 };
 
@@ -155,6 +156,7 @@ void mlx5e_build_tx_cq_param(struct mlx5_core_dev *mdev,
                             struct mlx5e_cq_param *param);
 void mlx5e_build_xdpsq_param(struct mlx5_core_dev *mdev,
                             struct mlx5e_params *params,
+                            struct mlx5e_xsk_param *xsk,
                             struct mlx5e_sq_param *param);
 int mlx5e_build_channel_param(struct mlx5_core_dev *mdev,
                              struct mlx5e_params *params,

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
index 9478df6..b220e61 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
@@ -323,6 +323,7 @@ mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd,
 
        dma_addr_t dma_addr = xdptxd->dma_addr;
        u32 dma_len = xdptxd->len;
+       u16 ds_cnt, inline_hdr_sz;
 
        struct mlx5e_xdpsq_stats *stats = sq->stats;
 
@@ -338,7 +339,8 @@ mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd,
        if (unlikely(check_result < 0))
                return false;
 
-       cseg->fm_ce_se = 0;
+       ds_cnt = MLX5E_TX_WQE_EMPTY_DS_COUNT + 1;
+       inline_hdr_sz = 0;
 
        /* copy the inline part if required */
        if (sq->min_inline_mode != MLX5_INLINE_MODE_NONE) {
@@ -347,7 +349,9 @@ mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd,
                       MLX5E_XDP_MIN_INLINE - sizeof(eseg->inline_hdr.start));
                dma_len  -= MLX5E_XDP_MIN_INLINE;
                dma_addr += MLX5E_XDP_MIN_INLINE;
+               inline_hdr_sz = MLX5E_XDP_MIN_INLINE;
                dseg++;
+               ds_cnt++;
        }
 
        /* write the dma part */
@@ -356,7 +360,31 @@ mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd,
 
        cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_SEND);
 
-       sq->pc++;
+       if (unlikely(test_bit(MLX5E_SQ_STATE_XDP_MULTIBUF, &sq->state))) {
+               u8 num_pkts = 1;
+               u8 num_wqebbs;
+
+               memset(&cseg->signature, 0, sizeof(*cseg) -
+                      sizeof(cseg->opmod_idx_opcode) - sizeof(cseg->qpn_ds));
+               memset(eseg, 0, sizeof(*eseg) - sizeof(eseg->trailer));
+
+               eseg->inline_hdr.sz = cpu_to_be16(inline_hdr_sz);
+               dseg->lkey = sq->mkey_be;
+
+               cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
+
+               num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
+               sq->db.wqe_info[pi] = (struct mlx5e_xdp_wqe_info) {
+                       .num_wqebbs = num_wqebbs,
+                       .num_pkts = num_pkts,
+               };
+
+               sq->pc += num_wqebbs;
+       } else {
+               cseg->fm_ce_se = 0;
+
+               sq->pc++;
+       }
 
        sq->doorbell_cseg = cseg;
 

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c
index 25eac9e..3ad7f13 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c
@@ -43,7 +43,7 @@ static void mlx5e_build_xsk_cparam(struct mlx5_core_dev *mdev,
                                   struct mlx5e_channel_param *cparam)
 {
        mlx5e_build_rq_param(mdev, params, xsk, q_counter, &cparam->rq);
-       mlx5e_build_xdpsq_param(mdev, params, &cparam->xdp_sq);
+       mlx5e_build_xdpsq_param(mdev, params, xsk, &cparam->xdp_sq);
 }
 
 static int mlx5e_init_xsk_rq(struct mlx5e_channel *c,

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index f21cae7..95cec28 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -1666,13 +1666,21 @@ int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params,
        csp.wq_ctrl         = &sq->wq_ctrl;
        csp.min_inline_mode = sq->min_inline_mode;
        set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
+
+       /* Don't enable multi buffer on XDP_REDIRECT SQ, as it's not yet
+        * supported by upstream, and there is no defined trigger to allow
+        * transmitting redirected multi-buffer frames.
+        */
+       if (param->is_xdp_mb && !is_redirect)
+               set_bit(MLX5E_SQ_STATE_XDP_MULTIBUF, &sq->state);
+
        err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn);
        if (err)
                goto err_free_xdpsq;
 
        mlx5e_set_xmit_fp(sq, param->is_mpw);
 
-       if (!param->is_mpw) {
+       if (!param->is_mpw && !test_bit(MLX5E_SQ_STATE_XDP_MULTIBUF, &sq->state)) {
                unsigned int ds_cnt = MLX5E_XDP_TX_DS_COUNT;
                unsigned int inline_hdr_sz = 0;
                int i;