drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 // Copyright (c) 2019 Mellanox Technologies.
   3
   4 #include <net/inet6_hashtables.h>
   5 #include "en_accel/en_accel.h"
   6 #include "en_accel/tls.h"
   7 #include "en_accel/ktls_txrx.h"
   8 #include "en_accel/ktls_utils.h"
   9 #include "en_accel/fs_tcp.h"
  10
  11 struct accel_rule {
  12         struct work_struct work;
  13         struct mlx5e_priv *priv;
  14         struct mlx5_flow_handle *rule;
  15 };
  16
  17 #define PROGRESS_PARAMS_WRITE_UNIT      64
  18 #define PROGRESS_PARAMS_PADDED_SIZE     \
  19                 (ALIGN(sizeof(struct mlx5_wqe_tls_progress_params_seg), \
  20                        PROGRESS_PARAMS_WRITE_UNIT))
  21
  22 struct mlx5e_ktls_rx_resync_buf {
  23         union {
  24                 struct mlx5_wqe_tls_progress_params_seg progress;
  25                 u8 pad[PROGRESS_PARAMS_PADDED_SIZE];
  26         } ____cacheline_aligned_in_smp;
  27         dma_addr_t dma_addr;
  28         struct mlx5e_ktls_offload_context_rx *priv_rx;
  29 };
  30
  31 enum {
  32         MLX5E_PRIV_RX_FLAG_DELETING,
  33         MLX5E_NUM_PRIV_RX_FLAGS,
  34 };
  35
  36 struct mlx5e_ktls_rx_resync_ctx {
  37         struct tls_offload_resync_async core;
  38         struct work_struct work;
  39         struct mlx5e_priv *priv;
  40         refcount_t refcnt;
  41         __be64 sw_rcd_sn_be;
  42         u32 seq;
  43 };
  44
  45 struct mlx5e_ktls_offload_context_rx {
  46         struct tls12_crypto_info_aes_gcm_128 crypto_info;
  47         struct accel_rule rule;
  48         struct sock *sk;
  49         struct mlx5e_rq_stats *stats;
  50         struct completion add_ctx;
  51         u32 tirn;
  52         u32 key_id;
  53         u32 rxq;
  54         DECLARE_BITMAP(flags, MLX5E_NUM_PRIV_RX_FLAGS);
  55
  56         /* resync */
  57         struct mlx5e_ktls_rx_resync_ctx resync;
  58 };
  59
  60 static bool mlx5e_ktls_priv_rx_put(struct mlx5e_ktls_offload_context_rx *priv_rx)
  61 {
  62         if (!refcount_dec_and_test(&priv_rx->resync.refcnt))
  63                 return false;
  64
  65         kfree(priv_rx);
  66         return true;
  67 }
  68
  69 static void mlx5e_ktls_priv_rx_get(struct mlx5e_ktls_offload_context_rx *priv_rx)
  70 {
  71         refcount_inc(&priv_rx->resync.refcnt);
  72 }
  73
  74 static int mlx5e_ktls_create_tir(struct mlx5_core_dev *mdev, u32 *tirn, u32 rqtn)
  75 {
  76         int err, inlen;
  77         void *tirc;
  78         u32 *in;
  79
  80         inlen = MLX5_ST_SZ_BYTES(create_tir_in);
  81         in = kvzalloc(inlen, GFP_KERNEL);
  82         if (!in)
  83                 return -ENOMEM;
  84
  85         tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
  86
  87         MLX5_SET(tirc, tirc, transport_domain, mdev->mlx5e_res.hw_objs.td.tdn);
  88         MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
  89         MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_INVERTED_XOR8);
  90         MLX5_SET(tirc, tirc, indirect_table, rqtn);
  91         MLX5_SET(tirc, tirc, tls_en, 1);
  92         MLX5_SET(tirc, tirc, self_lb_block,
  93                  MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST |
  94                  MLX5_TIRC_SELF_LB_BLOCK_BLOCK_MULTICAST);
  95
  96         err = mlx5_core_create_tir(mdev, in, tirn);
  97
  98         kvfree(in);
  99         return err;
 100 }
 101
 102 static void accel_rule_handle_work(struct work_struct *work)
 103 {
 104         struct mlx5e_ktls_offload_context_rx *priv_rx;
 105         struct accel_rule *accel_rule;
 106         struct mlx5_flow_handle *rule;
 107
 108         accel_rule = container_of(work, struct accel_rule, work);
 109         priv_rx = container_of(accel_rule, struct mlx5e_ktls_offload_context_rx, rule);
 110         if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags)))
 111                 goto out;
 112
 113         rule = mlx5e_accel_fs_add_sk(accel_rule->priv, priv_rx->sk,
 114                                      priv_rx->tirn, MLX5_FS_DEFAULT_FLOW_TAG);
 115         if (!IS_ERR_OR_NULL(rule))
 116                 accel_rule->rule = rule;
 117 out:
 118         complete(&priv_rx->add_ctx);
 119 }
 120
 121 static void accel_rule_init(struct accel_rule *rule, struct mlx5e_priv *priv,
 122                             struct sock *sk)
 123 {
 124         INIT_WORK(&rule->work, accel_rule_handle_work);
 125         rule->priv = priv;
 126 }
 127
 128 static void icosq_fill_wi(struct mlx5e_icosq *sq, u16 pi,
 129                           struct mlx5e_icosq_wqe_info *wi)
 130 {
 131         sq->db.wqe_info[pi] = *wi;
 132 }
 133
 134 static struct mlx5_wqe_ctrl_seg *
 135 post_static_params(struct mlx5e_icosq *sq,
 136                    struct mlx5e_ktls_offload_context_rx *priv_rx)
 137 {
 138         struct mlx5e_set_tls_static_params_wqe *wqe;
 139         struct mlx5e_icosq_wqe_info wi;
 140         u16 pi, num_wqebbs, room;
 141
 142         num_wqebbs = MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS;
 143         room = mlx5e_stop_room_for_wqe(num_wqebbs);
 144         if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, room)))
 145                 return ERR_PTR(-ENOSPC);
 146
 147         pi = mlx5e_icosq_get_next_pi(sq, num_wqebbs);
 148         wqe = MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi);
 149         mlx5e_ktls_build_static_params(wqe, sq->pc, sq->sqn, &priv_rx->crypto_info,
 150                                        priv_rx->tirn, priv_rx->key_id,
 151                                        priv_rx->resync.seq, false,
 152                                        TLS_OFFLOAD_CTX_DIR_RX);
 153         wi = (struct mlx5e_icosq_wqe_info) {
 154                 .wqe_type = MLX5E_ICOSQ_WQE_UMR_TLS,
 155                 .num_wqebbs = num_wqebbs,
 156                 .tls_set_params.priv_rx = priv_rx,
 157         };
 158         icosq_fill_wi(sq, pi, &wi);
 159         sq->pc += num_wqebbs;
 160
 161         return &wqe->ctrl;
 162 }
 163
 164 static struct mlx5_wqe_ctrl_seg *
 165 post_progress_params(struct mlx5e_icosq *sq,
 166                      struct mlx5e_ktls_offload_context_rx *priv_rx,
 167                      u32 next_record_tcp_sn)
 168 {
 169         struct mlx5e_set_tls_progress_params_wqe *wqe;
 170         struct mlx5e_icosq_wqe_info wi;
 171         u16 pi, num_wqebbs, room;
 172
 173         num_wqebbs = MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS;
 174         room = mlx5e_stop_room_for_wqe(num_wqebbs);
 175         if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, room)))
 176                 return ERR_PTR(-ENOSPC);
 177
 178         pi = mlx5e_icosq_get_next_pi(sq, num_wqebbs);
 179         wqe = MLX5E_TLS_FETCH_SET_PROGRESS_PARAMS_WQE(sq, pi);
 180         mlx5e_ktls_build_progress_params(wqe, sq->pc, sq->sqn, priv_rx->tirn, false,
 181                                          next_record_tcp_sn,
 182                                          TLS_OFFLOAD_CTX_DIR_RX);
 183         wi = (struct mlx5e_icosq_wqe_info) {
 184                 .wqe_type = MLX5E_ICOSQ_WQE_SET_PSV_TLS,
 185                 .num_wqebbs = num_wqebbs,
 186                 .tls_set_params.priv_rx = priv_rx,
 187         };
 188
 189         icosq_fill_wi(sq, pi, &wi);
 190         sq->pc += num_wqebbs;
 191
 192         return &wqe->ctrl;
 193 }
 194
 195 static int post_rx_param_wqes(struct mlx5e_channel *c,
 196                               struct mlx5e_ktls_offload_context_rx *priv_rx,
 197                               u32 next_record_tcp_sn)
 198 {
 199         struct mlx5_wqe_ctrl_seg *cseg;
 200         struct mlx5e_icosq *sq;
 201         int err;
 202
 203         err = 0;
 204         sq = &c->async_icosq;
 205         spin_lock_bh(&c->async_icosq_lock);
 206
 207         cseg = post_static_params(sq, priv_rx);
 208         if (IS_ERR(cseg))
 209                 goto err_out;
 210         cseg = post_progress_params(sq, priv_rx, next_record_tcp_sn);
 211         if (IS_ERR(cseg))
 212                 goto err_out;
 213
 214         mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg);
 215 unlock:
 216         spin_unlock_bh(&c->async_icosq_lock);
 217
 218         return err;
 219
 220 err_out:
 221         priv_rx->stats->tls_resync_req_skip++;
 222         err = PTR_ERR(cseg);
 223         complete(&priv_rx->add_ctx);
 224         goto unlock;
 225 }
 226
 227 static void
 228 mlx5e_set_ktls_rx_priv_ctx(struct tls_context *tls_ctx,
 229                            struct mlx5e_ktls_offload_context_rx *priv_rx)
 230 {
 231         struct mlx5e_ktls_offload_context_rx **ctx =
 232                 __tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_RX);
 233
 234         BUILD_BUG_ON(sizeof(struct mlx5e_ktls_offload_context_rx *) >
 235                      TLS_OFFLOAD_CONTEXT_SIZE_RX);
 236
 237         *ctx = priv_rx;
 238 }
 239
 240 static struct mlx5e_ktls_offload_context_rx *
 241 mlx5e_get_ktls_rx_priv_ctx(struct tls_context *tls_ctx)
 242 {
 243         struct mlx5e_ktls_offload_context_rx **ctx =
 244                 __tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_RX);
 245
 246         return *ctx;
 247 }
 248
 249 /* Re-sync */
 250 /* Runs in work context */
 251 static int
 252 resync_post_get_progress_params(struct mlx5e_icosq *sq,
 253                                 struct mlx5e_ktls_offload_context_rx *priv_rx)
 254 {
 255         struct mlx5e_get_tls_progress_params_wqe *wqe;
 256         struct mlx5e_ktls_rx_resync_buf *buf;
 257         struct mlx5e_icosq_wqe_info wi;
 258         struct mlx5_wqe_ctrl_seg *cseg;
 259         struct mlx5_seg_get_psv *psv;
 260         struct device *pdev;
 261         int err;
 262         u16 pi;
 263
 264         buf = kzalloc(sizeof(*buf), GFP_KERNEL);
 265         if (unlikely(!buf)) {
 266                 err = -ENOMEM;
 267                 goto err_out;
 268         }
 269
 270         pdev = mlx5_core_dma_dev(sq->channel->priv->mdev);
 271         buf->dma_addr = dma_map_single(pdev, &buf->progress,
 272                                        PROGRESS_PARAMS_PADDED_SIZE, DMA_FROM_DEVICE);
 273         if (unlikely(dma_mapping_error(pdev, buf->dma_addr))) {
 274                 err = -ENOMEM;
 275                 goto err_free;
 276         }
 277
 278         buf->priv_rx = priv_rx;
 279
 280         BUILD_BUG_ON(MLX5E_KTLS_GET_PROGRESS_WQEBBS != 1);
 281
 282         spin_lock_bh(&sq->channel->async_icosq_lock);
 283
 284         if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, 1))) {
 285                 spin_unlock_bh(&sq->channel->async_icosq_lock);
 286                 err = -ENOSPC;
 287                 goto err_dma_unmap;
 288         }
 289
 290         pi = mlx5e_icosq_get_next_pi(sq, 1);
 291         wqe = MLX5E_TLS_FETCH_GET_PROGRESS_PARAMS_WQE(sq, pi);
 292
 293 #define GET_PSV_DS_CNT (DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS))
 294
 295         cseg = &wqe->ctrl;
 296         cseg->opmod_idx_opcode =
 297                 cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_GET_PSV |
 298                             (MLX5_OPC_MOD_TLS_TIR_PROGRESS_PARAMS << 24));
 299         cseg->qpn_ds =
 300                 cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | GET_PSV_DS_CNT);
 301
 302         psv = &wqe->psv;
 303         psv->num_psv      = 1 << 4;
 304         psv->l_key        = sq->channel->mkey_be;
 305         psv->psv_index[0] = cpu_to_be32(priv_rx->tirn);
 306         psv->va           = cpu_to_be64(buf->dma_addr);
 307
 308         wi = (struct mlx5e_icosq_wqe_info) {
 309                 .wqe_type = MLX5E_ICOSQ_WQE_GET_PSV_TLS,
 310                 .num_wqebbs = 1,
 311                 .tls_get_params.buf = buf,
 312         };
 313         icosq_fill_wi(sq, pi, &wi);
 314         sq->pc++;
 315         mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg);
 316         spin_unlock_bh(&sq->channel->async_icosq_lock);
 317
 318         return 0;
 319
 320 err_dma_unmap:
 321         dma_unmap_single(pdev, buf->dma_addr, PROGRESS_PARAMS_PADDED_SIZE, DMA_FROM_DEVICE);
 322 err_free:
 323         kfree(buf);
 324 err_out:
 325         priv_rx->stats->tls_resync_req_skip++;
 326         return err;
 327 }
 328
 329 /* Function is called with elevated refcount.
 330  * It decreases it only if no WQE is posted.
 331  */
 332 static void resync_handle_work(struct work_struct *work)
 333 {
 334         struct mlx5e_ktls_offload_context_rx *priv_rx;
 335         struct mlx5e_ktls_rx_resync_ctx *resync;
 336         struct mlx5e_channel *c;
 337         struct mlx5e_icosq *sq;
 338
 339         resync = container_of(work, struct mlx5e_ktls_rx_resync_ctx, work);
 340         priv_rx = container_of(resync, struct mlx5e_ktls_offload_context_rx, resync);
 341
 342         if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) {
 343                 mlx5e_ktls_priv_rx_put(priv_rx);
 344                 return;
 345         }
 346
 347         c = resync->priv->channels.c[priv_rx->rxq];
 348         sq = &c->async_icosq;
 349
 350         if (resync_post_get_progress_params(sq, priv_rx))
 351                 mlx5e_ktls_priv_rx_put(priv_rx);
 352 }
 353
 354 static void resync_init(struct mlx5e_ktls_rx_resync_ctx *resync,
 355                         struct mlx5e_priv *priv)
 356 {
 357         INIT_WORK(&resync->work, resync_handle_work);
 358         resync->priv = priv;
 359         refcount_set(&resync->refcnt, 1);
 360 }
 361
 362 /* Function can be called with the refcount being either elevated or not.
 363  * It does not affect the refcount.
 364  */
 365 static int resync_handle_seq_match(struct mlx5e_ktls_offload_context_rx *priv_rx,
 366                                    struct mlx5e_channel *c)
 367 {
 368         struct tls12_crypto_info_aes_gcm_128 *info = &priv_rx->crypto_info;
 369         struct mlx5_wqe_ctrl_seg *cseg;
 370         struct mlx5e_icosq *sq;
 371         int err;
 372
 373         memcpy(info->rec_seq, &priv_rx->resync.sw_rcd_sn_be, sizeof(info->rec_seq));
 374         err = 0;
 375
 376         sq = &c->async_icosq;
 377         spin_lock_bh(&c->async_icosq_lock);
 378
 379         cseg = post_static_params(sq, priv_rx);
 380         if (IS_ERR(cseg)) {
 381                 priv_rx->stats->tls_resync_res_skip++;
 382                 err = PTR_ERR(cseg);
 383                 goto unlock;
 384         }
 385         /* Do not increment priv_rx refcnt, CQE handling is empty */
 386         mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg);
 387         priv_rx->stats->tls_resync_res_ok++;
 388 unlock:
 389         spin_unlock_bh(&c->async_icosq_lock);
 390
 391         return err;
 392 }
 393
 394 /* Function can be called with the refcount being either elevated or not.
 395  * It decreases the refcount and may free the kTLS priv context.
 396  * Refcount is not elevated only if tls_dev_del has been called, but GET_PSV was
 397  * already in flight.
 398  */
 399 void mlx5e_ktls_handle_get_psv_completion(struct mlx5e_icosq_wqe_info *wi,
 400                                           struct mlx5e_icosq *sq)
 401 {
 402         struct mlx5e_ktls_rx_resync_buf *buf = wi->tls_get_params.buf;
 403         struct mlx5e_ktls_offload_context_rx *priv_rx;
 404         struct mlx5e_ktls_rx_resync_ctx *resync;
 405         u8 tracker_state, auth_state, *ctx;
 406         struct device *dev;
 407         u32 hw_seq;
 408
 409         priv_rx = buf->priv_rx;
 410         resync = &priv_rx->resync;
 411         dev = mlx5_core_dma_dev(resync->priv->mdev);
 412         if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags)))
 413                 goto out;
 414
 415         dma_sync_single_for_cpu(dev, buf->dma_addr, PROGRESS_PARAMS_PADDED_SIZE,
 416                                 DMA_FROM_DEVICE);
 417
 418         ctx = buf->progress.ctx;
 419         tracker_state = MLX5_GET(tls_progress_params, ctx, record_tracker_state);
 420         auth_state = MLX5_GET(tls_progress_params, ctx, auth_state);
 421         if (tracker_state != MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_TRACKING ||
 422             auth_state != MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD) {
 423                 priv_rx->stats->tls_resync_req_skip++;
 424                 goto out;
 425         }
 426
 427         hw_seq = MLX5_GET(tls_progress_params, ctx, hw_resync_tcp_sn);
 428         tls_offload_rx_resync_async_request_end(priv_rx->sk, cpu_to_be32(hw_seq));
 429         priv_rx->stats->tls_resync_req_end++;
 430 out:
 431         mlx5e_ktls_priv_rx_put(priv_rx);
 432         dma_unmap_single(dev, buf->dma_addr, PROGRESS_PARAMS_PADDED_SIZE, DMA_FROM_DEVICE);
 433         kfree(buf);
 434 }
 435
 436 /* Runs in NAPI.
 437  * Function elevates the refcount, unless no work is queued.
 438  */
 439 static bool resync_queue_get_psv(struct sock *sk)
 440 {
 441         struct mlx5e_ktls_offload_context_rx *priv_rx;
 442         struct mlx5e_ktls_rx_resync_ctx *resync;
 443
 444         priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_get_ctx(sk));
 445         if (unlikely(!priv_rx))
 446                 return false;
 447
 448         if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags)))
 449                 return false;
 450
 451         resync = &priv_rx->resync;
 452         mlx5e_ktls_priv_rx_get(priv_rx);
 453         if (unlikely(!queue_work(resync->priv->tls->rx_wq, &resync->work)))
 454                 mlx5e_ktls_priv_rx_put(priv_rx);
 455
 456         return true;
 457 }
 458
 459 /* Runs in NAPI */
 460 static void resync_update_sn(struct mlx5e_rq *rq, struct sk_buff *skb)
 461 {
 462         struct ethhdr *eth = (struct ethhdr *)(skb->data);
 463         struct net_device *netdev = rq->netdev;
 464         struct sock *sk = NULL;
 465         unsigned int datalen;
 466         struct iphdr *iph;
 467         struct tcphdr *th;
 468         __be32 seq;
 469         int depth = 0;
 470
 471         __vlan_get_protocol(skb, eth->h_proto, &depth);
 472         iph = (struct iphdr *)(skb->data + depth);
 473
 474         if (iph->version == 4) {
 475                 depth += sizeof(struct iphdr);
 476                 th = (void *)iph + sizeof(struct iphdr);
 477
 478                 sk = inet_lookup_established(dev_net(netdev), &tcp_hashinfo,
 479                                              iph->saddr, th->source, iph->daddr,
 480                                              th->dest, netdev->ifindex);
 481 #if IS_ENABLED(CONFIG_IPV6)
 482         } else {
 483                 struct ipv6hdr *ipv6h = (struct ipv6hdr *)iph;
 484
 485                 depth += sizeof(struct ipv6hdr);
 486                 th = (void *)ipv6h + sizeof(struct ipv6hdr);
 487
 488                 sk = __inet6_lookup_established(dev_net(netdev), &tcp_hashinfo,
 489                                                 &ipv6h->saddr, th->source,
 490                                                 &ipv6h->daddr, ntohs(th->dest),
 491                                                 netdev->ifindex, 0);
 492 #endif
 493         }
 494
 495         depth += sizeof(struct tcphdr);
 496
 497         if (unlikely(!sk))
 498                 return;
 499
 500         if (unlikely(sk->sk_state == TCP_TIME_WAIT))
 501                 goto unref;
 502
 503         if (unlikely(!resync_queue_get_psv(sk)))
 504                 goto unref;
 505
 506         seq = th->seq;
 507         datalen = skb->len - depth;
 508         tls_offload_rx_resync_async_request_start(sk, seq, datalen);
 509         rq->stats->tls_resync_req_start++;
 510
 511 unref:
 512         sock_gen_put(sk);
 513 }
 514
 515 void mlx5e_ktls_rx_resync(struct net_device *netdev, struct sock *sk,
 516                           u32 seq, u8 *rcd_sn)
 517 {
 518         struct mlx5e_ktls_offload_context_rx *priv_rx;
 519         struct mlx5e_ktls_rx_resync_ctx *resync;
 520         struct mlx5e_priv *priv;
 521         struct mlx5e_channel *c;
 522
 523         priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_get_ctx(sk));
 524         if (unlikely(!priv_rx))
 525                 return;
 526
 527         resync = &priv_rx->resync;
 528         resync->sw_rcd_sn_be = *(__be64 *)rcd_sn;
 529         resync->seq = seq;
 530
 531         priv = netdev_priv(netdev);
 532         c = priv->channels.c[priv_rx->rxq];
 533
 534         resync_handle_seq_match(priv_rx, c);
 535 }
 536
 537 /* End of resync section */
 538
 539 void mlx5e_ktls_handle_rx_skb(struct mlx5e_rq *rq, struct sk_buff *skb,
 540                               struct mlx5_cqe64 *cqe, u32 *cqe_bcnt)
 541 {
 542         struct mlx5e_rq_stats *stats = rq->stats;
 543
 544         switch (get_cqe_tls_offload(cqe)) {
 545         case CQE_TLS_OFFLOAD_DECRYPTED:
 546                 skb->decrypted = 1;
 547                 stats->tls_decrypted_packets++;
 548                 stats->tls_decrypted_bytes += *cqe_bcnt;
 549                 break;
 550         case CQE_TLS_OFFLOAD_RESYNC:
 551                 stats->tls_resync_req_pkt++;
 552                 resync_update_sn(rq, skb);
 553                 break;
 554         default: /* CQE_TLS_OFFLOAD_ERROR: */
 555                 stats->tls_err++;
 556                 break;
 557         }
 558 }
 559
 560 void mlx5e_ktls_handle_ctx_completion(struct mlx5e_icosq_wqe_info *wi)
 561 {
 562         struct mlx5e_ktls_offload_context_rx *priv_rx = wi->tls_set_params.priv_rx;
 563         struct accel_rule *rule = &priv_rx->rule;
 564
 565         if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) {
 566                 complete(&priv_rx->add_ctx);
 567                 return;
 568         }
 569         queue_work(rule->priv->tls->rx_wq, &rule->work);
 570 }
 571
 572 static int mlx5e_ktls_sk_get_rxq(struct sock *sk)
 573 {
 574         int rxq = sk_rx_queue_get(sk);
 575
 576         if (unlikely(rxq == -1))
 577                 rxq = 0;
 578
 579         return rxq;
 580 }
 581
 582 int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk,
 583                       struct tls_crypto_info *crypto_info,
 584                       u32 start_offload_tcp_sn)
 585 {
 586         struct mlx5e_ktls_offload_context_rx *priv_rx;
 587         struct mlx5e_ktls_rx_resync_ctx *resync;
 588         struct tls_context *tls_ctx;
 589         struct mlx5_core_dev *mdev;
 590         struct mlx5e_priv *priv;
 591         int rxq, err;
 592         u32 rqtn;
 593
 594         tls_ctx = tls_get_ctx(sk);
 595         priv = netdev_priv(netdev);
 596         mdev = priv->mdev;
 597         priv_rx = kzalloc(sizeof(*priv_rx), GFP_KERNEL);
 598         if (unlikely(!priv_rx))
 599                 return -ENOMEM;
 600
 601         err = mlx5_ktls_create_key(mdev, crypto_info, &priv_rx->key_id);
 602         if (err)
 603                 goto err_create_key;
 604
 605         priv_rx->crypto_info  =
 606                 *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
 607
 608         rxq = mlx5e_ktls_sk_get_rxq(sk);
 609         priv_rx->rxq = rxq;
 610         priv_rx->sk = sk;
 611
 612         priv_rx->stats = &priv->channel_stats[rxq].rq;
 613         mlx5e_set_ktls_rx_priv_ctx(tls_ctx, priv_rx);
 614
 615         rqtn = priv->direct_tir[rxq].rqt.rqtn;
 616
 617         err = mlx5e_ktls_create_tir(mdev, &priv_rx->tirn, rqtn);
 618         if (err)
 619                 goto err_create_tir;
 620
 621         init_completion(&priv_rx->add_ctx);
 622
 623         accel_rule_init(&priv_rx->rule, priv, sk);
 624         resync = &priv_rx->resync;
 625         resync_init(resync, priv);
 626         tls_offload_ctx_rx(tls_ctx)->resync_async = &resync->core;
 627         tls_offload_rx_resync_set_type(sk, TLS_OFFLOAD_SYNC_TYPE_DRIVER_REQ_ASYNC);
 628
 629         err = post_rx_param_wqes(priv->channels.c[rxq], priv_rx, start_offload_tcp_sn);
 630         if (err)
 631                 goto err_post_wqes;
 632
 633         priv_rx->stats->tls_ctx++;
 634
 635         return 0;
 636
 637 err_post_wqes:
 638         mlx5_core_destroy_tir(mdev, priv_rx->tirn);
 639 err_create_tir:
 640         mlx5_ktls_destroy_key(mdev, priv_rx->key_id);
 641 err_create_key:
 642         kfree(priv_rx);
 643         return err;
 644 }
 645
 646 void mlx5e_ktls_del_rx(struct net_device *netdev, struct tls_context *tls_ctx)
 647 {
 648         struct mlx5e_ktls_offload_context_rx *priv_rx;
 649         struct mlx5e_ktls_rx_resync_ctx *resync;
 650         struct mlx5_core_dev *mdev;
 651         struct mlx5e_priv *priv;
 652
 653         priv = netdev_priv(netdev);
 654         mdev = priv->mdev;
 655
 656         priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_ctx);
 657         set_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags);
 658         mlx5e_set_ktls_rx_priv_ctx(tls_ctx, NULL);
 659         synchronize_net(); /* Sync with NAPI */
 660         if (!cancel_work_sync(&priv_rx->rule.work))
 661                 /* completion is needed, as the priv_rx in the add flow
 662                  * is maintained on the wqe info (wi), not on the socket.
 663                  */
 664                 wait_for_completion(&priv_rx->add_ctx);
 665         resync = &priv_rx->resync;
 666         if (cancel_work_sync(&resync->work))
 667                 mlx5e_ktls_priv_rx_put(priv_rx);
 668
 669         priv_rx->stats->tls_del++;
 670         if (priv_rx->rule.rule)
 671                 mlx5e_accel_fs_del_sk(priv_rx->rule.rule);
 672
 673         mlx5_core_destroy_tir(mdev, priv_rx->tirn);
 674         mlx5_ktls_destroy_key(mdev, priv_rx->key_id);
 675         /* priv_rx should normally be freed here, but if there is an outstanding
 676          * GET_PSV, deallocation will be delayed until the CQE for GET_PSV is
 677          * processed.
 678          */
 679         mlx5e_ktls_priv_rx_put(priv_rx);
 680 }